sema-svc: decode GRCE inline-data DMA (small HtoD path)

The 256-byte cuMemcpyHtoD payload never appears in any CE push: CUDA
submits it on the compute channel as a compute-class DMA whose source is
the push's own inline data (LINE_LENGTH_IN=256, OFFSET_OUT=<device VA>,
LAUNCH_DMA=0x41, LOAD_INLINE_DATA 64 dwords, then the channel's
SET_REPORT_SEMAPHORE release). The CE-shaped decoder skipped the data run
(opcode 3) and no GR work ran at all.

- Decode NVC5C0 GRCE methods on subchannel 1 (LINE_LENGTH_IN, LINE_COUNT,
  OFFSET_OUT/UPPER, LAUNCH_DMA) and the opcode-3 LOAD_INLINE_DATA run.
- svc_grce_inline_write resolves the destination VA through trusted
  sources (pins, map, published page directories, targeted rm_mem
  resolve), pins the page and writes into vidmem or guest RAM; budgeted
  sema-grce logs name the target or report UNRESOLVED.
- cdt2 test binary gains CDT_CP_PAUSE / CDT_HP_PAUSE for live inspection
  of the copy path; pushed by push_guest_tools.sh.

3 files changed, 284 insertions, 1 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 9b03cc5..db52299 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -347,6 +347,7 @@ struct sema_svc {
 	u64			copy_pat_log;		/* sema_copy_pat log budget */
 	u64			push_pat_log;		/* push-content pattern budget */
 	u64			dma_skip_log;		/* unrecognized-DMA budget */
+	u64			grce_log;		/* GRCE inline-write budget */
 	u32			cur_ch;			/* channel of the op being run */
 	u64			entry_log;		/* per-push summary budget */
 	u64			ops_denied;		/* work refused: implausible stream */
@@ -1844,6 +1845,22 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 #define CE_M_SET_REMAP_CONST_B	0x704
 #define CE_M_SET_REMAP_COMPONENTS 0x708
 
+/*
+ * NVC5C0 (compute class) GRCE methods: the compute engine also has a DMA
+ * path whose source is the push's own inline data. CUDA uses it for small
+ * HtoD copies (observed: 256-byte cuMemcpyHtoD on the GR channel as
+ * LINE_LENGTH_IN=256, OFFSET_OUT=<device VA>, LAUNCH_DMA=0x41, then
+ * LOAD_INLINE_DATA with the 64 payload dwords, then the channel's
+ * SET_REPORT_SEMAPHORE release). The CE-shaped decoder never runs GR work;
+ * these methods are decoded so the inline-data write lands.
+ */
+#define GRCE_M_LINE_LENGTH_IN	0x180
+#define GRCE_M_LINE_COUNT	0x184
+#define GRCE_M_OFFSET_OUT_UPPER	0x188
+#define GRCE_M_OFFSET_OUT	0x18c
+#define GRCE_M_LAUNCH_DMA	0x1b0
+#define GRCE_M_LOAD_INLINE_DATA	0x1b4
+
 /* LAUNCH_DMA fields. */
 #define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
 #define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
@@ -1886,6 +1903,7 @@ struct svc_push_state {
 	u64	offset_out, offset_in;
 	u32	have_in, have_out;
 	u32	line_length;
+	u32	gr_line_count;	/* GRCE LINE_COUNT */
 	u32	sem_va_hi, sem_va_lo, sem_payload;
 	u32	have_sem;
 	u32	have_remap;
@@ -2133,6 +2151,58 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	return 0;
 }
 
+/*
+ * GRCE inline-data write: the compute engine's DMA path takes its source
+ * from the method stream itself (LOAD_INLINE_DATA). Resolve the destination
+ * VA through the trusted sources (pins, discovered map, published page
+ * directories, targeted rm_mem resolve), pin the page and write the bytes.
+ */
+static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
+				 const u8 *data, u32 bytes, u32 ch_h)
+{
+	u64 pa = 0;
+	int ap = SVC_APERTURE_SYS;
+
+	if (!bytes || bytes > (1u << 20))
+		return -EINVAL;
+	if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) {
+		if (svc.grce_log < 16) {
+			svc.grce_log++;
+			pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n",
+				dst_va, bytes, ch_h);
+		}
+		return -ENOENT;
+	}
+	svc_pin_store(dst_va & ~0xfffULL, pa, ap);
+	pa += dst_va & 0xfff;
+	if (svc.grce_log < 64) {
+		u32 n = bytes >= 8 ? 8 : bytes;
+
+		svc.grce_log++;
+		pr_info("sema-grce: inline write dst=0x%llx -> pa=0x%llx ap=%s bytes=%u data8=%*phN h=0x%x\n",
+			dst_va, pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
+			bytes, n, data, ch_h);
+	}
+	if (ap == SVC_APERTURE_FB) {
+		u8 *base = vgpu->bars[1].vaddr;
+
+		if (!base || pa + bytes > vgpu->bars[1].size)
+			return -EINVAL;
+		memcpy(base + pa, data, bytes);
+	} else {
+		u32 o;
+
+		for (o = 0; o < bytes; o += 4096) {
+			u32 chunk = bytes - o > 4096 ? 4096 : bytes - o;
+
+			if (svc_write(vgpu, pa + o, (void *)(data + o), chunk))
+				return -EIO;
+		}
+	}
+	svc.memsets_done++;
+	return 0;
+}
+
 /* Execute a decoded operation list (fills and copies). */
 static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			u32 n_ops, u32 ch_h)
@@ -2530,6 +2600,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		subch = (hdr >> 13) & 0x7;
 		addr = (hdr & 0x1fff) << 2;
 
+		/*
+		 * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries
+		 * the DMA source bytes in the push itself. Run the write to
+		 * the destination the preceding methods selected.
+		 */
+		if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA &&
+		    count && off + 4 + count * 4 <= len) {
+			if ((trusted || allow_ops) && !gsp_sema_skip_ops &&
+			    st.have_out && st.line_length) {
+				u32 bytes = st.line_length *
+					(st.gr_line_count ? st.gr_line_count : 1);
+				u32 have = count * 4;
+
+				if (bytes > have)
+					bytes = have;
+				svc_grce_inline_write(vgpu, st.offset_out,
+						      buf + off + 4, bytes,
+						      ch_h);
+			} else if (svc.grce_log < 16) {
+				svc.grce_log++;
+				pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
+					trusted, allow_ops, st.have_out,
+					st.line_length, ch_h);
+			}
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (op != 1 || !count || count > 64 ||
 		    off + 4 + count * 4 > len) {
 			off += 4;	/* resync */
@@ -2762,6 +2860,45 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * carry the same method groups (the work-submission pushes that
 		 * release the client tracker semaphores). Accept both.
 		 */
+		/*
+		 * GRCE (compute class) operand methods on subchannel 1. State
+		 * is collected here and consumed by the LOAD_INLINE_DATA run
+		 * above; the channel's SET_REPORT_SEMAPHORE release is handled
+		 * by the shared branch before this point.
+		 */
+		if (subch == 1) {
+			switch (addr) {
+			case GRCE_M_LINE_LENGTH_IN:
+				st.line_length = v0;
+				if (count >= 2)
+					st.gr_line_count = v1;
+				break;
+			case GRCE_M_LINE_COUNT:
+				st.gr_line_count = v0;
+				break;
+			case GRCE_M_OFFSET_OUT_UPPER:
+				st.offset_out =
+					(st.offset_out & 0xffffffffULL) |
+					((u64)(v0 & 0x1ffff) << 32);
+				st.have_out = 1;
+				if (count >= 2) {
+					st.offset_out =
+						(st.offset_out &
+						 ~0xffffffffULL) | v1;
+				}
+				break;
+			case GRCE_M_OFFSET_OUT:
+				st.offset_out =
+					(st.offset_out & ~0xffffffffULL) | v0;
+				st.have_out = 1;
+				break;
+			default:
+				break;
+			}
+			off += 4 + count * 4;
+			continue;
+		}
+
 		if (subch != 0 && subch != 4) {
 			off += 4 + count * 4;
 			continue;
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
index 18db05c..9a45fd3 100755
--- a/tools/guest-diag/push_guest_tools.sh
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -24,6 +24,7 @@ push_b64 "$HERE/guest_engines.c" /tmp/ge.c
 push_b64 "$HERE/rmq_probe.c"     /tmp/rmqp.c
 push_b64 "$HERE/ioctl_trace.c"   /tmp/iot.c
 push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
+push_b64 "$REPO/tools/test/cuda_drv_test2.c" /tmp/cdt2.c
 push_b64 "$HERE/dumpctx.py"     /tmp/dumpctx.py
 push_b64 "$HERE/trkinfo.py"     /tmp/trkinfo.py
 push_b64 "$HERE/bp999.py"       /tmp/bp999.py
@@ -66,4 +67,5 @@ echo "[*] compiling in guest..."
 gcc -O1 -w -o /tmp/ge /tmp/ge.c && \
 gcc -O1 -w -o /tmp/rmqp /tmp/rmqp.c && \
 gcc -O1 -w -o /tmp/iot /tmp/iot.c && \
-gcc -O1 -o /tmp/cdt /tmp/cdt.c -ldl && echo BUILD_OK"
+gcc -O1 -o /tmp/cdt /tmp/cdt.c -ldl && \
+gcc -O1 -o /tmp/cdt2 /tmp/cdt2.c -ldl && echo BUILD_OK"
diff --git a/tools/test/cuda_drv_test2.c b/tools/test/cuda_drv_test2.c
new file mode 100644
index 0000000..a146bc2
--- /dev/null
+++ b/tools/test/cuda_drv_test2.c
@@ -0,0 +1,144 @@
+/* cuda_drv_test2.c - minimal CUDA driver-API smoke test (dlopen, no toolkit)
+ * with copy-path pause points for live inspection.
+ *
+ * Same sequence as cuda_drv_test.c, plus:
+ *   CDT_CP_PAUSE=<s>  sleep before cuMemcpyHtoD (after cuMemAlloc): the
+ *                     context is live, the device pointer is known - probe
+ *                     the GPU VA translation while parked.
+ *   CDT_HP_PAUSE=<s>  sleep after cuMemcpyHtoD (before DtoH): inspect what
+ *                     the HtoD left behind.
+ *   CDT_WAIT=<s>      park after the test (existing behaviour).
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <stdint.h>
+
+typedef int CUresult;
+typedef int CUdevice;
+typedef void *CUcontext;
+typedef uint64_t CUdeviceptr;
+
+static void fail_info(CUresult r)
+{
+    const char *(*errname)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorName");
+    const char *(*errstr)(CUresult, const char **) = dlsym(RTLD_DEFAULT, "cuGetErrorString");
+    const char *s = NULL;
+    const char *hold = getenv("CDT_HOLD");
+
+    if (errname && errname(r, &s) == 0 && s)
+        printf("cuGetErrorName: %s\n", s);
+    if (errstr && errstr(r, &s) == 0 && s)
+        printf("cuGetErrorString: %s\n", s);
+    if (hold && atoi(hold) > 0) {
+        printf("CDT_HOLD: sleeping %d s for inspection\n", atoi(hold));
+        fflush(stdout);
+        sleep((unsigned)atoi(hold));
+    }
+}
+
+#define CHECK(expr) do { \
+    CUresult r = (expr); \
+    if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); fail_info(r); return 1; } \
+    printf("%-24s OK\n", #expr); \
+} while (0)
+
+#define MARK(name) do { printf("-> %s\n", name); } while (0)
+
+static void pause_env(const char *name)
+{
+    const char *v = getenv(name);
+
+    if (v && atoi(v) > 0) {
+        printf("%s: parked %d s\n", name, atoi(v));
+        fflush(stdout);
+        sleep((unsigned)atoi(v));
+    }
+}
+
+int main(void)
+{
+    setvbuf(stdout, NULL, _IONBF, 0);
+
+    void *h = dlopen("libcuda.so.1", RTLD_NOW);
+    if (!h) { printf("dlopen libcuda.so.1 failed: %s\n", dlerror()); return 1; }
+
+    CUresult (*cuInit)(unsigned int) = dlsym(h, "cuInit");
+    CUresult (*cuDeviceGetCount)(int *) = dlsym(h, "cuDeviceGetCount");
+    CUresult (*cuDeviceGet)(CUdevice *, int) = dlsym(h, "cuDeviceGet");
+    CUresult (*cuCtxCreate)(CUcontext *, unsigned int, CUdevice) = dlsym(h, "cuCtxCreate_v2");
+    CUresult (*cuMemAlloc)(CUdeviceptr *, size_t) = dlsym(h, "cuMemAlloc_v2");
+    CUresult (*cuMemcpyHtoD)(CUdeviceptr, const void *, size_t) = dlsym(h, "cuMemcpyHtoD_v2");
+    CUresult (*cuMemcpyDtoH)(void *, CUdeviceptr, size_t) = dlsym(h, "cuMemcpyDtoH_v2");
+    CUresult (*cuMemFree)(CUdeviceptr) = dlsym(h, "cuMemFree_v2");
+    CUresult (*cuCtxDestroy)(CUcontext) = dlsym(h, "cuCtxDestroy_v2");
+
+    if (!cuInit || !cuDeviceGetCount || !cuDeviceGet || !cuCtxCreate ||
+        !cuMemAlloc || !cuMemcpyHtoD || !cuMemcpyDtoH || !cuMemFree || !cuCtxDestroy) {
+        printf("dlsym failed\n");
+        return 1;
+    }
+
+    int count = 0;
+    CUdevice dev;
+    CUcontext ctx = NULL;
+    CUdeviceptr dptr = 0;
+    unsigned int buf[64];
+    unsigned int back[64];
+    size_t i;
+
+    MARK("cuInit");
+    CHECK(cuInit(0));
+    MARK("cuDeviceGetCount");
+    CHECK(cuDeviceGetCount(&count));
+    printf("device count = %d\n", count);
+    if (count < 1) return 1;
+
+    MARK("cuDeviceGet");
+    CHECK(cuDeviceGet(&dev, 0));
+    printf("device 0 = %d\n", dev);
+
+    MARK("cuCtxCreate");
+    {
+        const char *pause = getenv("CDT_PAUSE");
+        if (pause && atoi(pause) > 0) {
+            printf("CDT_PAUSE: sleeping %d s before cuCtxCreate\n", atoi(pause));
+            fflush(stdout);
+            sleep((unsigned)atoi(pause));
+        }
+    }
+    CHECK(cuCtxCreate(&ctx, 0, dev));
+    printf("context = %p\n", ctx);
+
+    MARK("cuMemAlloc");
+    CHECK(cuMemAlloc(&dptr, sizeof(buf)));
+    printf("device ptr = 0x%llx (%zu bytes)\n", (unsigned long long)dptr, sizeof(buf));
+
+    for (i = 0; i < 64; i++) buf[i] = (unsigned int)(i * 3 + 1);
+    pause_env("CDT_CP_PAUSE");
+    MARK("cuMemcpyHtoD");
+    CHECK(cuMemcpyHtoD(dptr, buf, sizeof(buf)));
+    pause_env("CDT_HP_PAUSE");
+    memset(back, 0, sizeof(back));
+    MARK("cuMemcpyDtoH");
+    CHECK(cuMemcpyDtoH(back, dptr, sizeof(buf)));
+    {
+        size_t matched = 0;
+        for (i = 0; i < 64; i++)
+            if (buf[i] == back[i]) matched++;
+        printf("roundtrip %s matched=%zu/64 back[0..7]=%u,%u,%u,%u,%u,%u,%u,%u expected 1,4,7,10,13,16,19,22\n",
+               matched == 64 ? "match" : "MISMATCH", matched,
+               back[0], back[1], back[2], back[3],
+               back[4], back[5], back[6], back[7]);
+    }
+
+    MARK("cuMemFree");
+    CHECK(cuMemFree(dptr));
+    MARK("cuCtxDestroy");
+    CHECK(cuCtxDestroy(ctx));
+    printf("ALL OK\n");
+    pause_env("CDT_WAIT");
+    return 0;
+}