kernel: aperture-aware pinning and UVM CE channel USERD discovery

Debug injections could only teach sysmem translations, but the UVM CE
channel rings and USERDs live in the frame buffer window, and the
progress service had no candidate for their USERD (the allocation
parameters carry an offset relative to a client pool, not the FB
offset). Add an aperture field to the pinned translations and to the
sema_map_add debug parameter (va,pa[,len[,aperture]]), and probe the
observed UVM layout as well: the USERD sits one 0x10000 block after the
ring for the data-path channels.

With the rings pinned, the pump now resolves and decodes the CE rings
(the entries point at 1 MB pushbuffer regions); only the pushbuffer
VA->PA translation is still missing, so the entries are kept pending.

3 files changed, 40 insertions, 17 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index cb1255b..3b009ef 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -161,6 +161,7 @@ static u32 svc_map_hash[SEMA_MAP_MAX * 2];
 struct svc_pin {
 	u64 va_page;
 	u64 pa_page;
+	int ap;		/* SVC_APERTURE_* of the mapped page */
 };
 
 struct svc_vaspace {
@@ -862,7 +863,7 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
 	l->valid = true;
 }
 
-static int svc_pin_lookup(u64 va, u64 *pa)
+static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
 {
 	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	u32 mask = SVC_PIN_SLOTS - 1;
@@ -876,13 +877,14 @@ static int svc_pin_lookup(u64 va, u64 *pa)
 			return -ENOENT;
 		if (p->va_page == page) {
 			*pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
+			*ap = p->ap;
 			return 0;
 		}
 	}
 	return -ENOENT;
 }
 
-static void svc_pin_store(u64 va, u64 pa)
+static void svc_pin_store(u64 va, u64 pa, int ap)
 {
 	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	u32 mask = SVC_PIN_SLOTS - 1;
@@ -897,6 +899,7 @@ static void svc_pin_store(u64 va, u64 pa)
 				svc.pin_n++;
 			p->va_page = page;
 			p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+			p->ap = ap;
 			return;
 		}
 	}
@@ -915,8 +918,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 {
 	u64 pa = 0;
 
-	if (!svc_pin_lookup(va, addr)) {
-		*ap = SVC_APERTURE_SYS;
+	if (!svc_pin_lookup(va, addr, ap)) {
 		return 0;
 	}
 	if (!svc_lat_lookup(va, addr, ap))
@@ -2207,6 +2209,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	if (ring_ap == SVC_APERTURE_FB && ring_pa) {
 		cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB;
 		cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB;
+		/*
+		 * UVM's channel allocations place the USERD one 0x10000 block
+		 * after the ring (observed for all eight CE channels: ring at
+		 * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The
+		 * allocation parameters' userdMem base is relative to a client
+		 * pool and does not describe this layout.
+		 */
+		if (ch->engine_type != 0) {
+			cand[nc] = ring_pa + 0x10000;
+			as[nc++] = SVC_APERTURE_FB;
+		}
 	}
 	cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;
 
@@ -2518,10 +2531,12 @@ static struct completion map_add_done;
 static u64 map_add_va;
 static u64 map_add_pa;
 static u64 map_add_len;
+static u32 map_add_ap;
 static bool map_add_pending;
 
 static void svc_handle_map_add(void)
 {
+	u32 ap = map_add_ap ? map_add_ap : SVC_APERTURE_SYS;
 	u64 off;
 
 	if (!map_add_pending)
@@ -2529,28 +2544,29 @@ static void svc_handle_map_add(void)
 
 	for (off = 0; off < (map_add_len ? map_add_len : 1);
 	     off += SEMA_PAGE_SIZE) {
-		svc_pin_store(map_add_va + off, map_add_pa + off);
-		svc_lat_store(map_add_va + off, map_add_pa + off,
-			      SVC_APERTURE_SYS);
+		svc_pin_store(map_add_va + off, map_add_pa + off, ap);
+		svc_lat_store(map_add_va + off, map_add_pa + off, ap);
 	}
-	pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (sysmem)\n",
-		map_add_va, map_add_pa, map_add_len);
+	pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n",
+		map_add_va, map_add_pa, map_add_len,
+		ap == SVC_APERTURE_FB ? "vidmem" : "sysmem");
 	map_add_pending = false;
 	complete(&map_add_done);
 }
 
-int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len)
+int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap)
 {
 	if (!svc.task)
 		return -ENODEV;
 
-	if (len > (64ULL << 20))
+	if (len > (64ULL << 20) || ap > SVC_APERTURE_FB)
 		return -EINVAL;
 
 	init_completion(&map_add_done);
 	map_add_va = gpu_va;
 	map_add_pa = pa;
 	map_add_len = len;
+	map_add_ap = ap;
 	map_add_pending = true;
 
 	if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) {
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index a6893bd..6b8dc6a 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -57,7 +57,7 @@ void gsp_sema_service_vaspace_del(u32 h_vaspace);
 int  gsp_sema_service_start(void);
 void gsp_sema_service_stop(void);
 int  gsp_sema_service_probe(u64 gpu_va, u32 count);
-int  gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len);
+int  gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap);
 void gsp_sema_service_doorbell(u32 token);
 
 #endif /* _GSP_SEMA_SERVICE_H_ */
diff --git a/kernel/main.c b/kernel/main.c
index 25bd5eb..63657f3 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2256,8 +2256,8 @@ MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA")
 static int nv_vgpu_sema_map_add_set(const char *val, const struct kernel_param *kp)
 {
 	char buf[64];
-	char *comma, *comma2;
-	u64 va = 0, pa = 0, len = 0;
+	char *comma, *comma2, *comma3;
+	u64 va = 0, pa = 0, len = 0, ap = 0;
 
 	strscpy(buf, val, sizeof(buf));
 	comma = strchr(buf, ',');
@@ -2269,10 +2269,17 @@ static int nv_vgpu_sema_map_add_set(const char *val, const struct kernel_param *
 		*comma2 = '\0';
 	if (kstrtoull(buf, 0, &va) || kstrtoull(comma + 1, 0, &pa))
 		return -EINVAL;
-	if (comma2 && kstrtoull(comma2 + 1, 0, &len))
-		return -EINVAL;
+	if (comma2) {
+		comma3 = strchr(comma2 + 1, ',');
+		if (comma3)
+			*comma3 = '\0';
+		if (kstrtoull(comma2 + 1, 0, &len))
+			return -EINVAL;
+		if (comma3 && kstrtoull(comma3 + 1, 0, &ap))
+			return -EINVAL;
+	}
 
-	return gsp_sema_service_add_map(va, pa, len);
+	return gsp_sema_service_add_map(va, pa, len, (u32)ap);
 }
 
 static const struct kernel_param_ops nv_vgpu_sema_map_add_ops = {