kernel: validate learned semaphore maps and post guest GPU events

- admit {GPU VA -> PA} pairs only when the preceding bytes form a valid
  client memory descriptor header (type 0/1, CPU+GPU mapped bits), prefer
  validated pairs over headerless candidates under first-wins dedup, and
  invalidate the translation cache on every rebuild; fixes stale identity
  entries shadowing live pool pages and lets the guest UVM channel
  bring-up complete on its own
- capture OS event allocations (class 0x79) and post event records
  (function 0x1003) into the guest status queue when a channel's busy
  notification is consumed, with a broadcast fallback, serialized with
  replies via a queue lock
- add BAR0 shadow introspection (dump_bar0/scan_bar0) and guest-side
  diagnostic scripts for the VM cycle

1 files changed, 273 insertions, 0 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index a21cf28..986132e 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -473,11 +473,17 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 	if (!buf)
 		return -ENOMEM;
 
+	/* The status queue is shared: RPC replies run on the GSP emulation
+	 * thread, channel-completion events are posted from the progress
+	 * service's thread. */
+	mutex_lock(&rm->stat_lock);
+
 	record_size = gsp_rm_build_record(rm, buf, buf_size, function,
 					   sequence, result, data, data_len,
 					   &elem_count);
 	if (!record_size) {
 		kfree(buf);
+		mutex_unlock(&rm->stat_lock);
 		return -EOVERFLOW;
 	}
 
@@ -486,6 +492,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 		nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u",
 			    elem_count, rm->stat_msgcount);
 		kfree(buf);
+		mutex_unlock(&rm->stat_lock);
 		return -EOVERFLOW;
 	}
 
@@ -507,6 +514,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 			nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d",
 				    rec_off, ret);
 			kfree(buf);
+			mutex_unlock(&rm->stat_lock);
 			return ret;
 		}
 
@@ -518,6 +526,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 				nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d",
 					    ret);
 				kfree(buf);
+				mutex_unlock(&rm->stat_lock);
 				return ret;
 			}
 		}
@@ -538,6 +547,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 			       &new_wp, sizeof(new_wp));
 	if (ret) {
 		nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret);
+		mutex_unlock(&rm->stat_lock);
 		return ret;
 	}
 
@@ -547,9 +557,82 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
 	 * drain while a poller waits for it (same as real GSP-RM). */
 	gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq);
 
+	mutex_unlock(&rm->stat_lock);
 	return 0;
 }
 
+/*
+ * Post the GPU event that tells the guest's CPU-side RM a channel's
+ * submitted work is complete. A real GPU raises an interrupt, the host GSP
+ * turns it into an NV_VGPU_MSG_EVENT_POST_EVENT record for the client's
+ * event object, and the client's RM signals the OS event (eventfd) the
+ * submitting thread polls - this is what makes libcuda's work waits finish.
+ *
+ * Safe to call from the progress service's thread: the status queue write is
+ * serialized by rm->stat_lock and the guest is woken with MSI-X vector 0.
+ */
+int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
+{
+	struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
+	struct gsp_rm_emu *rm;
+	u8 payload[32];
+	u32 i;
+	int posted = 0;
+
+	if (!ctx)
+		return -ENODEV;
+	rm = ctx->rm_emu;
+	if (!rm || !h_src)
+		return -ENODEV;
+
+	for (i = 0; i < rm->events_n; i++) {
+		if (!rm->events[i].valid || rm->events[i].h_src != h_src)
+			continue;
+
+		memset(payload, 0, sizeof(payload));
+		memcpy(payload + 0, &rm->events[i].h_client, 4);
+		memcpy(payload + 4, &rm->events[i].h_event, 4);
+		memcpy(payload + 8, &rm->events[i].notify_index, 4);
+		memcpy(payload + 12, &rm->events[i].data, 4);
+		/* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */
+		if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+				payload, sizeof(payload)))
+			continue;
+		posted++;
+	}
+
+	/*
+	 * Fallback when the event's source object could not be associated with
+	 * the channel: nudge every registered event. The client's dispatcher
+	 * re-reads each channel's state on any event, so a spurious wake-up is
+	 * harmless (and this only runs when a submission was consumed).
+	 */
+	if (!posted) {
+		for (i = 0; i < rm->events_n; i++) {
+			if (!rm->events[i].valid)
+				continue;
+			memset(payload, 0, sizeof(payload));
+			memcpy(payload + 0, &rm->events[i].h_client, 4);
+			memcpy(payload + 4, &rm->events[i].h_event, 4);
+			memcpy(payload + 8, &rm->events[i].notify_index, 4);
+			memcpy(payload + 12, &rm->events[i].data, 4);
+			if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+					payload, sizeof(payload)))
+				continue;
+			posted++;
+		}
+	}
+
+	if (posted) {
+		rm->events_posted += posted;
+		if (rm->events_posted <= 32 || (rm->events_posted % 5000) == 0)
+			nv_vgpu_info(vgpu,
+				     "GSPRM: post_event hSrc=0x%x posted=%d total=%llu",
+				     h_src, posted, rm->events_posted);
+	}
+	return posted ? 0 : -ENOENT;
+}
+
 /* ============================================================
  * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED)
  * ============================================================ */
@@ -714,6 +797,8 @@ static const char *gsp_rm_rpc_name(u32 function)
 	case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO:	return "GSP_SET_SYSTEM_INFO";
 	case GSPRM_RPC_FUNC_SET_REGISTRY:		return "SET_REGISTRY";
 	case GSPRM_RPC_FUNC_CONTINUATION_RECORD:	return "CONTINUATION_RECORD";
+	case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:		return "SET_PAGE_DIRECTORY";
+	case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY:	return "UNSET_PAGE_DIRECTORY";
 	case GSPRM_RPC_FUNC_GSP_RM_CONTROL:		return "GSP_RM_CONTROL";
 	default:					return "UNKNOWN";
 	}
@@ -1949,6 +2034,7 @@ static bool gsp_rm_is_channel_class(u32 class_id)
 #define GSPRM_CHAN_GPFIFO_OFFSET	8
 #define GSPRM_CHAN_GPFIFO_ENTRIES	16
 #define GSPRM_CHAN_ENGINE_TYPE		128
+#define GSPRM_CHAN_CID			132
 #define GSPRM_CHAN_INSTANCE_MEM		144
 #define GSPRM_CHAN_USERD_MEM		168
 #define GSPRM_CHAN_MEM_APERTURE		16	/* offset of addressSpace in the desc */
@@ -1967,11 +2053,14 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
 	memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
 	memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
 	memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4);
+	memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4);
 	memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
 	memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
 	memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM +
 	       GSPRM_CHAN_MEM_APERTURE, 4);
 	memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8);
+	memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM +
+	       GSPRM_CHAN_MEM_APERTURE, 4);
 
 	gsp_sema_service_channel_add(&desc);
 }
@@ -2038,6 +2127,58 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
 	if (gsp_rm_is_channel_class(class_id))
 		gsp_rm_register_channel(ctx, ho, params, params_size);
 
+	/*
+	 * NV01_EVENT (allocated with class 0x79 in the guest ABI; params are
+	 * NV0005_ALLOC_PARAMETERS {hParentClient, hSrcResource, hClass,
+	 * notifyIndex, data}). The driver allocates one per channel and waits
+	 * for the GPU to post the event before it considers submitted work
+	 * complete - remember them so completed work can be signalled back.
+	 */
+	if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) {
+		u32 h_src = 0, notify_index = 0, hclass = 0;
+		u64 data64 = 0;
+		int hc_off = -1, i;
+
+		/*
+		 * Locate hClass (0x79) inside the params instead of trusting a
+		 * fixed offset: NV0005_ALLOC_PARAMETERS is
+		 * {hParentClient, hSrcResource, hClass, notifyIndex, data},
+		 * with hSrcResource immediately before hClass.
+		 */
+		for (i = 0; i + 4 <= (int)params_size; i += 4) {
+			u32 v;
+
+			memcpy(&v, params + i, 4);
+			if (v == 0x79) {
+				hc_off = i;
+				break;
+			}
+		}
+		if (hc_off >= 4) {
+			memcpy(&h_src, params + hc_off - 4, 4);
+			memcpy(&hclass, params + hc_off, 4);
+			if (hc_off + 8 <= (int)params_size)
+				memcpy(&notify_index, params + hc_off + 4, 4);
+			if (hc_off + 16 <= (int)params_size)
+				memcpy(&data64, params + hc_off + 8, 8);
+		} else {
+			memcpy(&h_src, params + 4, 4);
+			memcpy(&notify_index, params + 12, 4);
+			memcpy(&data64, params + 16, 8);
+		}
+		rm->events[rm->events_n].h_src = h_src;
+		rm->events[rm->events_n].h_event = ho;
+		rm->events[rm->events_n].h_client = hc;
+		rm->events[rm->events_n].notify_index = notify_index;
+		rm->events[rm->events_n].data = (u32)data64;
+		rm->events[rm->events_n].valid = true;
+		rm->events_n++;
+		nv_vgpu_info(ctx->vgpu,
+			     "GSPRM: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN",
+			     ho, h_src, notify_index, hc_off, (int)min(params_size, (u32)24),
+			     params);
+	}
+
 	/* Echo the fixed alloc header + as many param bytes as are contiguous
 	 * in the first queue element. */
 	echo_len = min(params_size,
@@ -2336,6 +2477,129 @@ static bool gsp_rm_alloc_interesting(u32 class_id)
 	}
 }
 
+/*
+ * SET_PAGE_DIRECTORY (54) / UNSET_PAGE_DIRECTORY (79)
+ *
+ * The driver builds the page tables of the GPU VA spaces it owns and hands the
+ * top-level directory to the GPU here. The parameters are
+ *   { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
+ *     u32 subDeviceId; u32 pasid; }
+ * (flags bits 1:0: 0 = vidmem aperture, 1 = sysmem coherent,
+ * 2 = sysmem non-coherent).
+ *
+ * Recording the directory lets the progress service translate that VA space's
+ * addresses directly (four-level walk), which is how the client's semaphore
+ * waits are completed; it also removes the need for heuristic discovery.
+ */
+static void gsp_rm_handle_set_page_directory(struct gsp_emu_context *ctx,
+					     const u8 *elem, u32 sequence,
+					     u32 length)
+{
+	const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+		      sizeof(struct gsprm_rpc_header);
+	struct sema_vaspace_desc desc;
+	u64 phys = 0;
+	u32 num_entries = 0, flags = 0, h_vas = 0, ch_id = 0, subdev = 0, pasid = 0;
+
+	/*
+	 * Wire layout: a 16-byte message header (hClient, hObject, ...)
+	 * followed by the control's own parameters
+	 * { u64 physAddress; u32 numEntries; u32 flags; u32 hVASpace; u32 chId;
+	 *   u32 subDeviceId; u32 pasid; }.
+	 */
+	if (length >= 44) {
+		memcpy(&phys, p + 16, 8);
+		memcpy(&num_entries, p + 24, 4);
+		memcpy(&flags, p + 28, 4);
+		memcpy(&h_vas, p + 32, 4);
+		memcpy(&ch_id, p + 36, 4);
+		memcpy(&subdev, p + 40, 4);
+		memcpy(&pasid, p + 44, 4);
+	}
+
+	{
+		char dump[200];
+		int o = 0, i;
+		u32 dlen = min_t(u32, length, 80);
+
+		for (i = 0; i < (int)dlen; i += 4) {
+			u32 v = 0;
+			memcpy(&v, p + i, 4);
+			o += snprintf(dump + o, sizeof(dump) - o, "%08x ",
+				      v);
+		}
+		nv_vgpu_info(ctx->vgpu,
+			     "GSPRM: SET_PAGE_DIRECTORY len=%u payload: %s",
+			     length, dump);
+	}
+	nv_vgpu_info(ctx->vgpu,
+		     "GSPRM: SET_PAGE_DIRECTORY pdb=0x%llx entries=%u flags=0x%x hVASpace=0x%x chId=0x%x subDev=%u pasid=%u len=%u",
+		     phys, num_entries, flags, h_vas, ch_id, subdev, pasid,
+		     length);
+
+	if (phys && num_entries) {
+		memset(&desc, 0, sizeof(desc));
+		desc.h_vaspace = h_vas;
+		desc.pdb = phys;
+		desc.num_entries = num_entries;
+		desc.b_vidmem = (flags & 0x3) == 0;
+		gsp_sema_service_vaspace_add(&desc);
+
+		/* Show the first entries of the directory: they are the ground
+		 * truth for the walk. */
+		{
+			u64 peek[4] = {0};
+			char b[128];
+			int i, o = 0, rc = -ENODEV;
+
+			if (desc.b_vidmem) {
+				u8 *base = ctx->vgpu->bars[1].vaddr;
+
+				if (base && phys + sizeof(peek) <=
+					    ctx->vgpu->bars[1].size) {
+					memcpy(peek, base + phys, sizeof(peek));
+					rc = 0;
+				}
+			} else {
+				rc = gsp_rm_read_dev(ctx->vgpu, phys, peek,
+						     sizeof(peek));
+			}
+			if (!rc) {
+				for (i = 0; i < 4; i++)
+					o += snprintf(b + o, sizeof(b) - o,
+						      "%016llx ", peek[i]);
+				nv_vgpu_info(ctx->vgpu,
+					     "GSPRM:   directory[0..3]: %s", b);
+			}
+		}
+	}
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY, sequence, 0,
+			NULL, 0))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: SET_PAGE_DIRECTORY reply send failed");
+}
+
+static void gsp_rm_handle_unset_page_directory(struct gsp_emu_context *ctx,
+					       const u8 *elem, u32 sequence,
+					       u32 length)
+{
+	const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+		      sizeof(struct gsprm_rpc_header);
+	u32 h_vas = 0;
+
+	if (length >= 4)
+		memcpy(&h_vas, p, 4);
+
+	nv_vgpu_info(ctx->vgpu,
+		     "GSPRM: UNSET_PAGE_DIRECTORY hVASpace=0x%x len=%u",
+		     h_vas, length);
+	gsp_sema_service_vaspace_del(h_vas);
+
+	if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY, sequence, 0,
+			NULL, 0))
+		nv_vgpu_err(ctx->vgpu, "GSPRM: UNSET_PAGE_DIRECTORY reply send failed");
+}
+
 static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
 {
 	struct gsp_rm_emu *rm = ctx->rm_emu;
@@ -2376,6 +2640,14 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
 		gsp_rm_handle_free(ctx, elem, sequence);
 		return;
 
+	case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:
+		gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length);
+		return;
+
+	case GSPRM_RPC_FUNC_UNSET_PAGE_DIRECTORY:
+		gsp_rm_handle_unset_page_directory(ctx, elem, sequence, hdr->length);
+		return;
+
 	default:
 		/* Synchronous RPC: echo function+sequence with success so the
 		 * driver's rpcRecvPoll completes. */
@@ -2499,6 +2771,7 @@ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx)
 		return -ENOMEM;
 
 	rm->vgpu = vgpu;
+	mutex_init(&rm->stat_lock);
 	ctx->rm_emu = rm;
 	return 0;
 }