kernel: completion-event hints and GSP-RM state introspection

- re-post the completion event for a bounded window after a channel
  completes work (notifier consumed or semaphore release applied), so a
  client that starts waiting after the edge-triggered wake-up still gets
  a wake-up; the OS event classes carry no source object, so the hint
  broadcasts to the registered events
- add rmstate (dump msgq cursors, guest consumption position, event
  table) and rmpost (post an event record by index or to all) debug
  parameters, both executing in kthreads because the IOMMU read path
  only works with a NULL current->mm or the mapping owner's mm
- report the underlying error codes from the event post path

4 files changed, 274 insertions, 0 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 986132e..5120d4e 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -633,6 +633,146 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
 	return posted ? 0 : -ENOENT;
 }
 
+/*
+ * Debug: post the event record for one registered event (or all of them with
+ * idx 0xffffffff). Used to test the guest's event path on demand without
+ * waiting for a channel submission.
+ */
+static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx)
+{
+	struct gsp_rm_emu *rm = ctx->rm_emu;
+	u8 payload[32];
+
+	if (idx >= rm->events_n || !rm->events[idx].valid)
+		return -EINVAL;
+
+	memset(payload, 0, sizeof(payload));
+	memcpy(payload + 0, &rm->events[idx].h_client, 4);
+	memcpy(payload + 4, &rm->events[idx].h_event, 4);
+	memcpy(payload + 8, &rm->events[idx].notify_index, 4);
+	memcpy(payload + 12, &rm->events[idx].data, 4);
+
+	return gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+			   payload, sizeof(payload));
+}
+
+int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx)
+{
+	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
+	struct gsp_rm_emu *rm;
+	u32 i;
+	int n = 0, last = -ENOENT;
+
+	if (!ctx)
+		return -ENODEV;
+	rm = ctx->rm_emu;
+	if (!rm)
+		return -ENODEV;
+
+	if (idx != 0xffffffffu) {
+		int r = gsp_rm_post_event_idx(ctx, idx);
+
+		return r ? r : 1;
+	}
+
+	for (i = 0; i < rm->events_n; i++) {
+		int r = gsp_rm_post_event_idx(ctx, i);
+
+		if (r)
+			last = r;
+		else
+			n++;
+	}
+
+	return n ? 0 : last;
+}
+
+/*
+ * Post the completion event to every registered OS event (the classes the
+ * guest allocates carry no source object, so a per-channel association is
+ * not available in the ABI). Used by the service's completion hint so a
+ * client that starts waiting after the event fired still gets a wake-up.
+ */
+int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu)
+{
+	return gsp_rm_force_post(vgpu, 0xffffffffu);
+}
+
+/*
+ * Debug: dump the msgq cursors (ours and the guest's) and the event table.
+ * The guest's consumption cursor of the status queue lives in its own
+ * command-queue buffer at the rxHdrOff it advertised (SWAP_RX queues store
+ * the peer read pointer in the local buffer).
+ */
+int gsp_rm_dump_state(struct nv_vgpu_device *vgpu)
+{
+	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
+	struct gsp_rm_emu *rm;
+	u8 cmd_hdr[32] = {0}, stat_hdr[32] = {0};
+	u32 cmd_write = 0, cmd_flags = 0, cmd_rxoff = 0, cmd_entryoff = 0;
+	u32 stat_write = 0, stat_flags = 0, stat_rxoff = 0;
+	u32 guest_read = 0xdeadbeef, our_cmd_read = 0xdeadbeef;
+	u64 cmd_gpa_now = 0, stat_gpa_now = 0;
+	int cmd_ret = -1, stat_ret = -1, gr_ret = -1;
+	u32 i;
+
+	if (!ctx)
+		return -ENODEV;
+	rm = ctx->rm_emu;
+	if (!rm || !rm->parsed)
+		return -ENODEV;
+
+	cmd_gpa_now = gsp_rm_off_to_gpa(rm, rm->cmd_off);
+	stat_gpa_now = gsp_rm_off_to_gpa(rm, rm->stat_off);
+
+	cmd_ret = gsp_rm_read_buf(ctx, rm->cmd_off, cmd_hdr, sizeof(cmd_hdr));
+	if (cmd_ret == 0) {
+		memcpy(&cmd_write, cmd_hdr + 16, 4);
+		memcpy(&cmd_flags, cmd_hdr + 20, 4);
+		memcpy(&cmd_rxoff, cmd_hdr + 24, 4);
+		memcpy(&cmd_entryoff, cmd_hdr + 28, 4);
+	}
+	stat_ret = gsp_rm_read_buf(ctx, rm->stat_off, stat_hdr, sizeof(stat_hdr));
+	if (stat_ret == 0) {
+		memcpy(&stat_write, stat_hdr + 16, 4);
+		memcpy(&stat_flags, stat_hdr + 20, 4);
+		memcpy(&stat_rxoff, stat_hdr + 24, 4);
+	}
+
+	if (cmd_rxoff && cmd_rxoff < 0x1000)
+		gr_ret = gsp_rm_read_buf(ctx, rm->cmd_off + cmd_rxoff,
+					 &guest_read, 4);
+	if (stat_rxoff && stat_rxoff < 0x1000)
+		gsp_rm_read_buf(ctx, rm->stat_off + stat_rxoff, &our_cmd_read, 4);
+
+	nv_vgpu_info(vgpu,
+		     "GSPRM: rmstate cmd{gpa=0x%llx->0x%llx off=0x%llx size=0x%x mc=%u ptes=%u read=%d our_read=%u hdr{wr=%u fl=0x%x rxoff=0x%x entryoff=0x%x}} "
+		     "stat{gpa=0x%llx->0x%llx off=0x%llx size=0x%x mc=%u read=%d our_write=%u seq=%u hdr{wr=%u fl=0x%x rxoff=0x%x ours_written=%u} guest_read=%u gr_ret=%d}",
+		     rm->cmd_gpa, cmd_gpa_now, rm->cmd_off, rm->cmd_size,
+		     rm->cmd_msgcount, rm->mq.pageTableEntryCount, cmd_ret,
+		     rm->cmd_read, cmd_write, cmd_flags, cmd_rxoff, cmd_entryoff,
+		     rm->stat_gpa, stat_gpa_now, rm->stat_off, rm->stat_size,
+		     rm->stat_msgcount, stat_ret, rm->stat_write, rm->stat_seq,
+		     stat_write, stat_flags, stat_rxoff, our_cmd_read,
+		     guest_read, gr_ret);
+
+
+	nv_vgpu_info(vgpu, "GSPRM: rmstate events_n=%u posted=%llu",
+		     rm->events_n, rm->events_posted);
+
+	for (i = 0; i < rm->events_n && i < GSPRM_MAX_EVENTS; i++) {
+		if (!rm->events[i].valid)
+			continue;
+		nv_vgpu_info(vgpu,
+			     "GSPRM:   event[%u] hClient=0x%x hEvent=0x%x hSrc=0x%x notifyIndex=0x%x data=0x%x",
+			     i, rm->events[i].h_client, rm->events[i].h_event,
+			     rm->events[i].h_src, rm->events[i].notify_index,
+			     rm->events[i].data);
+	}
+
+	return 0;
+}
+
 /* ============================================================
  * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
  * ============================================================ */
diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
index 1ab0a83..9ff74e9 100644
--- a/kernel/gsp_rm_emulation.h
+++ b/kernel/gsp_rm_emulation.h
@@ -195,6 +195,9 @@ struct gsp_rm_emu {
 int  gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx);
 void gsp_rm_emu_fini(struct gsp_emu_context *ctx);
 int  gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src);
+int  gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx);
+int  gsp_rm_post_all_events(struct nv_vgpu_device *vgpu);
+int  gsp_rm_dump_state(struct nv_vgpu_device *vgpu);
 int  gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle,
 			   u64 start, u64 end, u32 max_hits);
 int  gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat,
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 10d6b80..b66cfd7 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -190,6 +190,15 @@ struct sema_svc {
 	u64			last_build;
 	u64			last_scan;
 
+	/*
+	 * Completion hint window: after a channel completes work the client is
+	 * waiting on, keep re-posting the event for a few seconds. A client
+	 * that starts waiting after the completion would otherwise lose the
+	 * (edge-triggered) wake-up.
+	 */
+	u64			post_hint_until;
+	u64			last_hint;
+
 	/* registered guest channels (GPFIFO-driven push execution) */
 	spinlock_t		chan_lock;
 	struct svc_channel	chan[SVC_MAX_CHANNELS];
@@ -1642,6 +1651,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						cur + payload : payload;
 					if (!svc_sem_write(vgpu, va, cur))
 						svc.releases_applied++;
+						svc.post_hint_until = jiffies + 10 * HZ;
 				}
 			}
 			off += 4 + count * 4;
@@ -1701,6 +1711,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					if (!svc_sem_write(vgpu, va,
 							   st.rs_payload))
 						svc.releases_applied++;
+						svc.post_hint_until = jiffies + 10 * HZ;
 					break;
 				}
 				default:
@@ -1825,6 +1836,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						cur = st.sem_payload;
 					if (!svc_sem_write(vgpu, sem_va, cur))
 						svc.releases_applied++;
+						svc.post_hint_until = jiffies + 10 * HZ;
 				}
 				/* release ends the push */
 				n_ops = 0;
@@ -1928,6 +1940,7 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 
 		if (svc_sem_write(vgpu, va, payload) == 0) {
 			svc.releases_applied++;
+			svc.post_hint_until = jiffies + 10 * HZ;
 			applied++;
 			if (payload != cur + 1) {
 				svc.releases_jumped++;
@@ -2212,6 +2225,10 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
 		 * thread waits on (a real GPU would raise an interrupt here).
 		 */
 		gsp_rm_post_channel_event(vgpu, ch->h_object);
+
+		/* Keep the completion visible to clients that start waiting
+		 * after the event fired. */
+		svc.post_hint_until = jiffies + 10 * HZ;
 	}
 }
 
@@ -2526,6 +2543,21 @@ static int svc_thread(void *arg)
 		svc_pump_channels(vgpu);
 
 		/*
+		 * Completion hint: re-post the events for a few seconds after a
+		 * completion, so a client that starts waiting after the
+		 * edge-triggered wake-up still observes it.
+		 */
+		if (time_before(jiffies, (unsigned long)svc.post_hint_until) &&
+		    time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) {
+			int rc;
+
+			svc.last_hint = jiffies;
+			rc = gsp_rm_post_all_events(vgpu);
+			pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
+					    rc);
+		}
+
+		/*
 		 * Fast path for the pushes themselves: every mapped page is a
 		 * candidate pushbuffer page, and there are far fewer of those
 		 * than there are pages of guest RAM.
diff --git a/kernel/main.c b/kernel/main.c
index 05a49b1..25bd5eb 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -3023,5 +3023,104 @@ static const struct kernel_param_ops nv_vgpu_inject_irq_ops = {
 module_param_cb(inject_irq, &nv_vgpu_inject_irq_ops, NULL, 0200);
 MODULE_PARM_DESC(inject_irq, "Deliver MSI-X vector N to the guest (debug)");
 
+/*
+ * Debug: dump the GSP-RM msgq cursors (ours and the guest's) plus the
+ * registered event objects:
+ *   echo 1 > .../parameters/rmstate
+ *
+ * Runs in a kthread: vfio_iommu_type1_dma_rw only works when current->mm is
+ * NULL (kthread) or matches the mapping's mm (QEMU).
+ */
+static int nv_vgpu_rmstate_thread(void *arg)
+{
+	struct completion *done = arg;
+	struct nv_vgpu_device *vgpu;
+
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+			continue;
+		gsp_rm_dump_state(vgpu);
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+
+	complete(done);
+	return 0;
+}
+
+static int nv_vgpu_rmstate_set(const char *val, const struct kernel_param *kp)
+{
+	DECLARE_COMPLETION_ONSTACK(done);
+	struct task_struct *task;
+
+	task = kthread_run(nv_vgpu_rmstate_thread, &done, "vgpu-rmstate");
+	if (IS_ERR(task))
+		return PTR_ERR(task);
+	wait_for_completion(&done);
+	return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_rmstate_ops = {
+	.set = nv_vgpu_rmstate_set,
+};
+
+module_param_cb(rmstate, &nv_vgpu_rmstate_ops, NULL, 0200);
+MODULE_PARM_DESC(rmstate, "Dump GSP-RM msgq cursors and event table (debug)");
+
+/*
+ * Debug: post an event record to the guest (idx < events_n, or 0xffffffff
+ * for every registered event):
+ *   echo 0 > .../parameters/rmpost
+ */
+struct nv_vgpu_rmpost_req {
+	u32 idx;
+	int rc;
+	struct completion done;
+};
+
+static int nv_vgpu_rmpost_thread(void *arg)
+{
+	struct nv_vgpu_rmpost_req *req = arg;
+	struct nv_vgpu_device *vgpu;
+
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) {
+		if (atomic_read(&vgpu->vfio_dev_open_count) <= 0)
+			continue;
+		req->rc = gsp_rm_force_post(vgpu, req->idx);
+		pr_info("gsp-resman-vfio: rmpost idx=%u -> %d\n",
+			req->idx, req->rc);
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+
+	complete(&req->done);
+	return 0;
+}
+
+static int nv_vgpu_rmpost_set(const char *val, const struct kernel_param *kp)
+{
+	static struct nv_vgpu_rmpost_req req;
+	struct task_struct *task;
+
+	memset(&req, 0, sizeof(req));
+	req.rc = -ENODEV;
+	if (kstrtou32(val, 0, &req.idx))
+		return -EINVAL;
+
+	init_completion(&req.done);
+	task = kthread_run(nv_vgpu_rmpost_thread, &req, "vgpu-rmpost");
+	if (IS_ERR(task))
+		return PTR_ERR(task);
+	wait_for_completion(&req.done);
+	return 0;
+}
+
+static const struct kernel_param_ops nv_vgpu_rmpost_ops = {
+	.set = nv_vgpu_rmpost_set,
+};
+
+module_param_cb(rmpost, &nv_vgpu_rmpost_ops, NULL, 0200);
+MODULE_PARM_DESC(rmpost, "Post a GSP-RM event to the guest by event index (debug)");
+
 module_init(nv_vgpu_init);
 module_exit(nv_vgpu_exit);