gsp-phys: H1 GR ladder - GR0 channel, compute object, GR sem push (default off)

New load-time gsp_phys_rmcall_engine (0=COPY0 default, 1=GR0) parameterizes
the physical ladder: TSG and channel engine_type GR0, one-attempt channel
alloc, TURING_COMPUTE_A engine object (class 0xc5c0, handle 0xcaf0002c),
BIND engineType=1, and a GR method push (SET_OBJECT 0xc5c0 on subchannel 1
plus SET_REPORT_SEMAPHORE) submitted through the existing GPFIFO/USERD
doorbell tail, polling the module semaphore. CE work-forwarding refuses
cleanly while a GR ladder is active. A debugfs trigger re-runs the GR push.
With the parameter at its default the ladder emits byte-identical COPY0
requests (only the state line gains engine=COPY0).

Live (first GR context on the live server): every GR RPC accepted -
channel variant V5, TURING_COMPUTE_A alloc status=0x0 -> hObject=0xcaf0002c,
BIND status=0x0, token=0x1 - and the engine fetched the push (GPGet 0->1),
but raised a GR exception instead of executing the release:
exceptType=0x20, NOCAT faultingEngine="GR", intr1=0x80000000, no rpc_result
errors. The failure is downstream of all RPCs; no mutating retry was made
and the device released cleanly (WPR2 down, rebound, 5 locks, pstore empty,
same boot). A default-engine control cycle stayed green (REAL COPY OK).
Follow-up: bisect the GR alloc params, the release method D word, and the
method order/context init using the drained GR fault records as the oracle.

1 files changed, 464 insertions, 21 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index a033022..4409a29 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -236,6 +236,25 @@ bool gsp_phys_boot_requested(void)
 #define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u
 
 /*
+ * H1 ladder engine selection (gsp_phys_rmcall_engine). GR0 is engine type 1
+ * (NV2080_ENGINE_TYPE_GRAPHICS / ENG_GR(0)); the guest compute-object path
+ * allocates TURING_COMPUTE_A 0xc5c0 on the channel
+ * (generated/g_gpu_class_list.c: { TURING_COMPUTE_A, ENG_GR(0) }) and its
+ * methods live on the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1,
+ * class/cla06fsubch.h:29). The local handle 0xcaf0002c is free in the
+ * ladder's client handle range (0xcaf00000..0xcaf7ffff). The compute
+ * class's alloc params are NV_GR_ALLOCATION_PARAMETERS
+ * (common/sdk/nvidia/inc/nvos.h:2322-2328): version 2, flags 0,
+ * size sizeof (0x10), caps is an output.
+ */
+#define GSP_PHYS_RMCALL_ENGINE_GR0		0x1u
+#define GSP_PHYS_RMCALL_CLASS_COMPUTE		0xc5c0u
+#define GSP_PHYS_RMCALL_HANDLE_GR_OBJ		0xcaf0002cu
+#define GSP_PHYS_RMCALL_GR_SUBCHANNEL		1u
+#define GSP_PHYS_RMCALL_GR_PARAM_VERSION	0x2u
+#define GSP_PHYS_RMCALL_GR_PARAM_SIZE		0x10u
+
+/*
  * Work-submit token notifier slot. The default index is
  * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1
  * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex
@@ -532,6 +551,22 @@ enum gsp_phys_rmcall_buf {
 #define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN	0x418u
 #define GSP_PHYS_RMCALL_CE_LINE_COUNT		0x41cu
 
+/*
+ * H1 GR (TURING_COMPUTE_A 0xc5c0) release methods on the compute subchannel:
+ * SET_OBJECT 0x0000 (class/clc5c0.h:34) and SET_REPORT_SEMAPHORE_A..D at
+ * 0x1b00/0x1b04/0x1b08/0x1b0c (class/clc5c0.h:622-631). A/B are the
+ * semaphore GPU VA upper/lower, C the payload and D the operation word:
+ * OPERATION=RELEASE (bits 1:0 = 0), FLUSH_DISABLE=0, REDUCTION_ENABLE=0,
+ * AWAKEN_ENABLE=1 (bit 20) and STRUCTURE_SIZE=ONE_WORD (bit 28), i.e.
+ * 0x10100000 (class/clc5c0.h:632-644).
+ */
+#define GSP_PHYS_RMCALL_GR_SET_OBJECT			0x000u
+#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A	0x1b00u
+#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B	0x1b04u
+#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C	0x1b08u
+#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D	0x1b0cu
+#define GSP_PHYS_RMCALL_GR_SEM_RELEASE_D		0x10100000u
+
 /* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3=
  * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE VIRTUAL
  * bits12/13=0 (clc5b5.h:69-108); exactly the UVM user-channel pitch copy
@@ -1010,6 +1045,17 @@ static struct gsp_phys_boot_stage {
 	u32 rmcall_channel_variant;	/* winning stage-3 variant (0=none; see
 					 * the variant table for its number) */
 	bool rmcall_channel_server_mem;	/* winning variant had server-allocated mem */
+	/*
+	 * H1 engine snapshot: gsp_phys_rmcall_engine (0=COPY0, 1=GR0) is
+	 * snapshotted when stage 3 starts and every later step uses the
+	 * snapshot, so a runtime sysfs change can never mix engines. A later
+	 * change is logged once and ignored.
+	 */
+	u32 rmcall_engine;		/* COPY0 or GR0 (stage-3 snapshot) */
+	bool rmcall_engine_valid;	/* the stage-3 snapshot was taken */
+	bool rmcall_engine_change_logged;	/* mid-session change noted once */
+	bool rmcall_gr_ce_refused_logged;	/* CE-on-GR refusal noted once */
+	u32 rmcall_gr_object;		/* TURING_COMPUTE_A handle (0=none) */
 	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
 	u32 rmcall_notifier_mem;	/* FBMEM error-notifier object handle */
 	u64 rmcall_notifier_base;	/* its returned frame-buffer address */
@@ -1586,6 +1632,11 @@ static void gsp_phys_rmcall_reset(void)
 	boot_stage.rmcall_channel = 0;
 	boot_stage.rmcall_channel_variant = 0;
 	boot_stage.rmcall_channel_server_mem = false;
+	boot_stage.rmcall_engine = 0;
+	boot_stage.rmcall_engine_valid = false;
+	boot_stage.rmcall_engine_change_logged = false;
+	boot_stage.rmcall_gr_ce_refused_logged = false;
+	boot_stage.rmcall_gr_object = 0;
 	boot_stage.rmcall_token = 0;
 	boot_stage.rmcall_notifier_mem = 0;
 	boot_stage.rmcall_notifier_base = 0;
@@ -5413,6 +5464,22 @@ struct gsp_phys_rmcall_channel_params {
 	__le32 tpc_config_id;
 };
 
+/*
+ * NV_GR_ALLOCATION_PARAMETERS for the H1 TURING_COMPUTE_A engine object
+ * (common/sdk/nvidia/inc/nvos.h:2322-2328): version must be 2, flags has no
+ * defined bits, size is sizeof (16) and caps is an output. The server's
+ * class-constructor path for 0xc5c0 consumes exactly this 16-byte shape
+ * (the guest driver's own wire request carries paramsSize=16 for the class).
+ * If a live server rejects {2,0,0x10,0}, the bisection order recorded in the
+ * H1 notes is: {2,0,0x10,0} -> {1,0,0x10,0} -> {2,0,0x0,0}.
+ */
+struct gsp_phys_rmcall_gr_alloc_params {
+	__le32 version;
+	__le32 flags;
+	__le32 size;
+	__le32 caps;
+};
+
 /* Bounded little-endian read of a reply payload word: a read past the
  * received payload is never performed (off+4 <= payload_len). */
 static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec,
@@ -6024,13 +6091,87 @@ gsp_phys_rmcall_channel_variants[] = {
 	  GSP_PHYS_RMCALL_HANDLE_CHANNEL },
 };
 
+/*
+ * H1 engine snapshot. gsp_phys_rmcall_engine is a load-time module
+ * parameter (0=COPY0, 1=GR0); the ladder snapshots it when stage 3 starts
+ * and every later step uses the snapshot, so a runtime sysfs change can
+ * never mix a GR0 TSG/channel with a COPY0 BIND or the CE method block. A
+ * later change is logged once and ignored. With the default COPY0 value the
+ * snapshot emits no log and every request is byte-identical to before.
+ */
+static void gsp_phys_rmcall_engine_snapshot(void)
+{
+	int v;
+
+	if (boot_stage.rmcall_engine_valid)
+		return;
+	v = gsp_phys_rmcall_engine_value();
+	if (v == 1) {
+		boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_GR0;
+	} else {
+		if (v != 0)
+			pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine=%d is neither 0 (COPY0) nor 1 (GR0); using COPY0\n",
+				v);
+		boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_COPY0;
+	}
+	boot_stage.rmcall_engine_valid = true;
+	if (boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0)
+		pr_info("gsp-phys-boot: rm-call: engine snapshot: gsp_phys_rmcall_engine=1 -> GR0 (TSG/channel engineType=0x1, TURING_COMPUTE_A 0xc5c0, BIND 0x1, GR semaphore release)\n");
+}
+
+static bool gsp_phys_rmcall_engine_is_gr(void)
+{
+	return boot_stage.rmcall_engine_valid &&
+	       boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0;
+}
+
+static const char *gsp_phys_rmcall_engine_name(void)
+{
+	return gsp_phys_rmcall_engine_is_gr() ? "GR0" : "COPY0";
+}
+
+static void gsp_phys_rmcall_engine_check_change(void)
+{
+	int v = gsp_phys_rmcall_engine_value();
+	u32 live = (v == 1) ? GSP_PHYS_RMCALL_ENGINE_GR0 :
+			     GSP_PHYS_RMCALL_ENGINE_COPY0;
+
+	if (!boot_stage.rmcall_engine_valid ||
+	    live == boot_stage.rmcall_engine ||
+	    boot_stage.rmcall_engine_change_logged)
+		return;
+	boot_stage.rmcall_engine_change_logged = true;
+	pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine changed to %d after the stage-3 snapshot; the ladder keeps engine=%s for the rest of this staging (mid-session changes are ignored)\n",
+		v, gsp_phys_rmcall_engine_name());
+}
+
+/*
+ * H1 CE-refusal gate: true when the current staging's ladder channel speaks
+ * GR, i.e. when a CE method block must never be submitted on it. Logs once
+ * per staging (the callers are the lock-free readiness gates, so only the
+ * first attempt is noisy).
+ */
+static bool gsp_phys_rmcall_gr_ce_refuse(const char *what)
+{
+	if (!gsp_phys_rmcall_engine_is_gr())
+		return false;
+	if (!boot_stage.rmcall_gr_ce_refused_logged) {
+		boot_stage.rmcall_gr_ce_refused_logged = true;
+		pr_info("gsp-phys-work: %s refused: the ladder channel is a %s engine channel (gsp_phys_rmcall_engine snapshot); CE methods are never submitted on a GR channel (logged once per staging)\n",
+			what, gsp_phys_rmcall_engine_name());
+	}
+	return true;
+}
+
 static void
 gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
 				    const struct gsp_phys_rmcall_channel_variant *v)
 {
 	memset(ch, 0, sizeof(*ch));
 	ch->flags = cpu_to_le32(v->flags);
-	ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
+	ch->engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ?
+				      GSP_PHYS_RMCALL_ENGINE_GR0 :
+				      GSP_PHYS_RMCALL_ENGINE_COPY0);
 	ch->cid = cpu_to_le32(1);	/* client hint; server assigns its own */
 	ch->sub_device_id = cpu_to_le32(0);
 	ch->internal_flags = cpu_to_le32(v->internal_flags);
@@ -6078,17 +6219,25 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
  * state. Failed variants log one compact line each and no payload dump; an
  * all-failed walk ends in one summary line with every variant's
  * rpc_result/status. Kept out of stage 3 so the inline stack stays small.
+ *
+ * H1: on a GR0 ladder only the first variant (V5 tsg-vas-pdb-skip-scrubber)
+ * is tried. A single rpc_result != 0 aborts the walk with no retry and no
+ * fallback variants: mixing a failed GR channel attempt with the historical
+ * COPY0-shaped fallbacks could leave an unknown server state, so the whole
+ * ladder stops at the first failure.
  */
 static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 {
 	struct gsp_phys_rmcall_channel_params ch;
 	u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
 	u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
+	u32 attempts = gsp_phys_rmcall_engine_is_gr() ? 1u :
+		       (u32)ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
 	u32 i;
 	int ret;
 
 	ret = -EIO;
-	for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) {
+	for (i = 0; i < attempts; i++) {
 		const struct gsp_phys_rmcall_channel_variant *v =
 			&gsp_phys_rmcall_channel_variants[i];
 		u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
@@ -6118,6 +6267,14 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 		}
 	}
 
+	if (attempts == 1) {
+		pr_err("gsp-phys-boot: rm-call: stage 3: the single GR0 channel attempt (V%u %s) failed (ret=%d rpc_result=0x%x status=0x%x); the GR ladder aborts with no fallback variants and no retry\n",
+		       gsp_phys_rmcall_channel_variants[0].id,
+		       gsp_phys_rmcall_channel_variants[0].name, ret,
+		       v_rpc[0], v_status[0]);
+		return ret;
+	}
+
 	{
 		char summary[256];
 		u32 pos = 0;
@@ -6136,6 +6293,48 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
 }
 
 /*
+ * H1 stage-3 step (GR0 snapshot only): allocate the TURING_COMPUTE_A
+ * (0xc5c0) engine object the GR method block's SET_OBJECT binds to. The
+ * client chooses the local handle 0xcaf0002c (free in the ladder's resource
+ * range; the server validates it with clientAssignResourceHandle) and the
+ * parent is the ladder channel: the GR class constructor routes to the
+ * engine from the parent channel handle (kgrobjConstruct_IMPL:
+ * kgrmgrCtrlSetChannelHandle(pParams->hParent)). The params are
+ * NV_GR_ALLOCATION_PARAMETERS {version=2, flags=0, size=0x10, caps=0}; if
+ * the server rejects them the alloc helper dumps the failed reply and the
+ * ladder aborts with no retry (the H1 notes carry the bisection order).
+ */
+static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
+{
+	struct gsp_phys_rmcall_gr_alloc_params gr;
+	u32 status = 0;
+	int ret;
+
+	gr.version = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_VERSION);
+	gr.flags = cpu_to_le32(0);
+	gr.size = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_SIZE);
+	gr.caps = cpu_to_le32(0);
+
+	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+				    boot_stage.rmcall_channel,
+				    GSP_PHYS_RMCALL_HANDLE_GR_OBJ,
+				    GSP_PHYS_RMCALL_CLASS_COMPUTE,
+				    &gr, sizeof(gr), false, NULL,
+				    "TURING_COMPUTE_A", true, NULL, &status);
+	pr_info("gsp-phys-boot: rm-call: TURING_COMPUTE_A alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n",
+		GSP_PHYS_RMCALL_CLASS_COMPUTE, status,
+		GSP_PHYS_RMCALL_HANDLE_GR_OBJ, boot_stage.rmcall_channel,
+		GSP_PHYS_RMCALL_GR_PARAM_VERSION, 0u,
+		GSP_PHYS_RMCALL_GR_PARAM_SIZE, 0u,
+		ret ? " (rejected)" : "");
+	if (ret)
+		return ret;
+
+	boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_GR_OBJ;
+	return 0;
+}
+
+/*
  * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The
  * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and
  * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE
@@ -6384,6 +6583,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	int ret;
 
 	/*
+	 * H1: freeze the ladder engine for the rest of this staging before the
+	 * first engine-dependent request (TSG/channel alloc params below).
+	 */
+	gsp_phys_rmcall_engine_snapshot();
+
+	/*
 	 * The TSG below carries this externally-owned FERMI_VASPACE_A, whose
 	 * root page directory is committed from the zeroed VASPDB chunk before
 	 * any channel alloc (the instance-block setup programs that base and
@@ -6439,7 +6644,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 
 	memset(&tsg, 0, sizeof(tsg));
 	tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
-	tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
+	tsg.engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ?
+				      GSP_PHYS_RMCALL_ENGINE_GR0 :
+				      GSP_PHYS_RMCALL_ENGINE_COPY0);
 	/*
 	 * The TSG's hParent must be the DEVICE, not the subdevice: the server
 	 * resolves it with deviceGetByHandle (kernel_channel_group_api.c:150)
@@ -6474,11 +6681,28 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
 	if (ret)
 		return ret;
 
-	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
+	/*
+	 * H1: bind the compute engine object to the just-allocated GR channel
+	 * before stage 3 returns. A rejection aborts the ladder (the GR
+	 * method block's SET_OBJECT would fail too) and is never retried. On a
+	 * COPY0 ladder this step is skipped entirely: no RPC, no log.
+	 */
+	if (gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_alloc_gr_object(bar0);
+		if (ret) {
+			pr_err("gsp-phys-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n",
+			       ret);
+			return ret;
+		}
+	}
+
+	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
 		boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
 		boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
 		boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
-		boot_stage.rmcall_notifier_base);
+		boot_stage.rmcall_notifier_base,
+		gsp_phys_rmcall_engine_name(),
+		gsp_phys_rmcall_engine_is_gr() ? " hGr=0xcaf0002c" : "");
 	return 0;
 }
 
@@ -6538,6 +6762,45 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va,
 	return (u32)(p - pb);
 }
 
+/* H1 GR method header: the same immediate-pair shape as the CE helper but on
+ * the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1). */
+static u32 *gsp_phys_rmcall_push_gr(u32 *p, u32 method, u32 data)
+{
+	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+	       (1u << 16) |
+	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
+	       (method >> 2);
+	*p++ = data;
+	return p;
+}
+
+/*
+ * H1 GR method block: five immediate pairs (10 dwords) that bind the compute
+ * class to the channel's compute subchannel and release one word into the
+ * semaphore GPU VA. This is not a copy block: no src/dst VAs, offsets,
+ * pitches or LAUNCH_DMA. The semaphore VA is the same channel-PDB VA the CE
+ * ladder armed, so the release lands in the module semaphore page. Used by
+ * stage 4's GR verify and by the debugfs GR re-run trigger; only called when
+ * the stage-3 snapshot selected GR0.
+ */
+static u32 gsp_phys_rmcall_build_gr_methods(u32 *pb, u64 sem_va,
+					    u32 sem_payload)
+{
+	u32 *p = pb;
+
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT,
+				    GSP_PHYS_RMCALL_CLASS_COMPUTE);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
+				    (u32)(sem_va >> 32));
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
+				    (u32)sem_va);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
+				    sem_payload);
+	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
+				    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
+	return (u32)(p - pb);
+}
+
 /*
  * Result of one ladder CE submit, valid only when the submit helper returned 0.
  * Kept for the caller's log line and the GP_GET/GPPut context.
@@ -6572,10 +6835,16 @@ struct gsp_phys_rmcall_ce_submit {
  * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
  * still cleared even when the caller polls a guest page: the clear only
  * guards the module-sem poll path and is a harmless non-guest write.
+ *
+ * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
+ * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
+ * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
+ * site passes false, so the default COPY0 ladder emits the same bytes as
+ * before.
  */
 static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 					    u64 dst_va, u64 sem_va, u32 bytes,
-					    u32 sem_payload,
+					    u32 sem_payload, bool gr,
 					    struct gsp_phys_rmcall_ce_submit *out)
 {
 	struct gsp_phys_boot_chunk *pb =
@@ -6623,8 +6892,13 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
 	wmb();
 
-	words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va,
-					      bytes, sem_payload);
+	if (gr)
+		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
+							 sem_payload);
+	else
+		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
+						      dst_va, bytes,
+						      sem_payload);
 
 	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
 	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
@@ -6704,6 +6978,80 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
 }
 
 /*
+ * H1 stage-4 GR verification: one GR semaphore release on the ladder channel,
+ * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller
+ * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll;
+ * the trigger takes the same lock), and this helper polls the module
+ * semaphore while the lock is held, exactly like the CE stage-4 tail. The
+ * submission goes through the shared guarded helper with gr=true, so the
+ * server-memory/BAR0/token/GP_PUT/GPFIFO-GET guards apply unchanged. There is
+ * no src/dst copy to compare: the GR method block only releases the
+ * semaphore. On timeout the existing status-queue drain runs and the helper
+ * returns -ETIMEDOUT; it never retries.
+ */
+static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
+{
+	struct gsp_phys_boot_chunk *userd_c =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+	struct gsp_phys_boot_chunk *sem =
+		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+	struct gsp_phys_rmcall_ce_submit sub;
+	u32 *userd;
+	u32 poll, gpget, gpput;
+	unsigned long deadline;
+	int ret;
+
+	ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC,
+					       GSP_PHYS_RMCALL_VA_DST,
+					       GSP_PHYS_RMCALL_VA_SEM,
+					       GSP_PHYS_RMCALL_COPY_SIZE,
+					       GSP_PHYS_RMCALL_SEM_PAYLOAD, true,
+					       &sub);
+	if (ret) {
+		pr_err("gsp-phys-boot: rm-call: GR submission refused before the doorbell (%d)\n",
+		       ret);
+		return ret;
+	}
+	userd = gsp_phys_chunk_page_va(userd_c, 0);
+
+	pr_info("gsp-phys-boot: rm-call: GR submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSem=0x%llx vaSrc=0x%llx vaDst=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
+		sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
+		GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_VA_SRC,
+		GSP_PHYS_RMCALL_VA_DST, sub.words, sub.slot, sub.gpget,
+		sub.gpput, sub.e0, sub.e1, boot_stage.rmcall_token,
+		sub.runlist_id, sub.ch_id, sub.doorbell,
+		GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma,
+		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);
+
+	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
+	for (;;) {
+		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			gpget = get_unaligned_le32((u8 *)userd +
+						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+			gpput = get_unaligned_le32((u8 *)userd +
+						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
+			pr_err("gsp-phys-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n",
+			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
+			       GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput,
+			       gsp_phys_rmcall_engine_name());
+			gsp_phys_status_drain(NULL);
+			return -ETIMEDOUT;
+		}
+		msleep(GSP_PHYS_RMCALL_POLL_MS);
+	}
+
+	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
+	pr_info("gsp-phys-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n",
+		poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget,
+		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
+		sub.words, boot_stage.rmcall_token);
+	return 0;
+}
+
+/*
  * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on
  * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock
  * (the same lock discipline as the P5 shadow API, so a guest copy can never
@@ -6728,11 +7076,15 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va,
 	u32 poll;
 	int ret;
 
+	/* H1: never submit CE methods on a GR0 ladder channel. */
+	if (gsp_phys_rmcall_gr_ce_refuse("guest CE copy (trigger 8 selftest)"))
+		return -EOPNOTSUPP;
+
 	mutex_lock(&gsp_phys_rm_lock);
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
 					       bytes,
 					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
-					       &sub);
+					       false, &sub);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret)
 		return ret;
@@ -6773,6 +7125,10 @@ static u64 gsp_phys_work_refused;
  * P5/P6b forwarding APIs additionally require gsp_phys_work_forward (see
  * gsp_phys_backend_work_ready below); triggers 6/7 must run with forwarding
  * off, so their guest-copy variant uses this gate directly.
+ *
+ * H1: a GR0 ladder channel has no CE engine bound, so every CE forwarding
+ * path refuses here. The refusal is logged once per staging by
+ * gsp_phys_rmcall_gr_ce_refuse(); the software release path keeps working.
  */
 static bool gsp_phys_backend_ladder_ready(void)
 {
@@ -6781,6 +7137,8 @@ static bool gsp_phys_backend_ladder_ready(void)
 	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
 	    boot_stage.rm_unload_submitted)
 		return false;
+	if (gsp_phys_rmcall_gr_ce_refuse("CE forwarding"))
+		return false;
 	if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem ||
 	    !boot_stage.rmcall_token)
 		return false;
@@ -6868,7 +7226,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va,
 	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
 					       bytes,
 					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
-					       &sub);
+					       false, &sub);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret) {
 		gsp_phys_work_refused++;
@@ -7331,7 +7689,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 					       bytes,
 					       relay ? sem_payload :
 						       GSP_PHYS_RMCALL_SEM_PAYLOAD,
-					       NULL);
+					       false, NULL);
 	mutex_unlock(&gsp_phys_rm_lock);
 	if (ret) {
 		if (relay)
@@ -7526,7 +7884,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
 	struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
 	struct gsp_phys_rmcall_ce_submit sub;
-	u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0;	/* NVA06F engineType */
+	u32 bind_params = gsp_phys_rmcall_engine_is_gr() ?
+		GSP_PHYS_RMCALL_ENGINE_GR0 :	/* NVA06F engineType */
+		GSP_PHYS_RMCALL_ENGINE_COPY0;
 	u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX;	/* NVC36F index */
 	u8 sched_params[2] = { 1, 0 };	/* NVA06F bEnable=1, bSkipSubmit=0 */
 	resource_size_t bar0_len;
@@ -7536,6 +7896,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	unsigned long deadline;
 	int ret;
 
+	/* H1: note (once) a mid-session engine-parameter change, keep snapshot. */
+	gsp_phys_rmcall_engine_check_change();
+
 	/*
 	 * The winning variant requested server-allocated memdescs: there is no
 	 * client GPFIFO, USERD or pushbuffer to fill, so no work item can be
@@ -7663,6 +8026,21 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 	}
 	boot_stage.rmcall_token = token;
 
+	/*
+	 * H1: a GR0 ladder releases a semaphore on the compute engine instead
+	 * of running a CE copy. The dedicated helper performs the same guarded
+	 * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM
+	 * OK / REAL GR SEM FAILED and never compares src/dst; the CE tail
+	 * below is untouched and byte-identical for the default COPY0 ladder.
+	 */
+	if (gsp_phys_rmcall_engine_is_gr()) {
+		ret = gsp_phys_rmcall_gr_sem_push_locked(bar0);
+		if (ret)
+			pr_err("gsp-phys-boot: rm-call: stage 4 GR path failed (%d); no retry\n",
+			       ret);
+		return ret;
+	}
+
 	userd = gsp_phys_chunk_page_va(userd_c, 0);
 
 	/*
@@ -7677,7 +8055,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 					       GSP_PHYS_RMCALL_VA_SEM,
 					       GSP_PHYS_RMCALL_COPY_SIZE,
 					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
-					       &sub);
+					       false, &sub);
 	if (ret) {
 		pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n",
 		       ret);
@@ -7736,6 +8114,39 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
 }
 
 /*
+ * H1 debugfs trigger 9: re-run one GR semaphore release on the already-built
+ * GR ladder without a full stage cycle. Valid only when the stage-3 snapshot
+ * selected GR0, the GR engine object was allocated and stage 4 already
+ * produced a work-submit token (a failed GR poll still leaves the token set,
+ * so the trigger is useful for bisection). Takes gsp_phys_rm_lock like
+ * gsp_phys_rmcall_run() and performs no RPC: only the method-block submit,
+ * GPFIFO/USERD publish, doorbell and bounded semaphore poll. Never retries;
+ * every gate miss is a logged no-op.
+ */
+static int gsp_phys_rmcall_gr_sem_rerun(void __iomem *bar0)
+{
+	int ret;
+
+	mutex_lock(&gsp_phys_rm_lock);
+	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
+	    boot_stage.rm_unload_submitted ||
+	    !gsp_phys_rmcall_engine_is_gr() ||
+	    !boot_stage.rmcall_channel || !boot_stage.rmcall_gr_object ||
+	    !boot_stage.rmcall_token || boot_stage.rmcall_stage < 3) {
+		mutex_unlock(&gsp_phys_rm_lock);
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 no-op: requires the GR ladder through stage 3 (gr=%u claimed=%u init_done=%u unload=%u hChannel=0x%x hGr=0x%x token=0x%x stage=%u)\n",
+			gsp_phys_rmcall_engine_is_gr(), gsp_phys_gpu_claimed(),
+			boot_stage.rm_init_done, boot_stage.rm_unload_submitted,
+			boot_stage.rmcall_channel, boot_stage.rmcall_gr_object,
+			boot_stage.rmcall_token, boot_stage.rmcall_stage);
+		return -EINVAL;
+	}
+	ret = gsp_phys_rmcall_gr_sem_push_locked(bar0);
+	mutex_unlock(&gsp_phys_rm_lock);
+	return ret;
+}
+
+/*
  * Stage 5: adopt the server's internal client/device/subdevice handles from
  * this session's decoded GspStaticConfigInfo instead of allocating our own
  * NV01_ROOT/DEVICE/SUBDEVICE. If the rm-probe cycle has not produced them
@@ -8236,7 +8647,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rm_probe_last_function,
 		   boot_stage.rm_probe_last_payload_len);
 	seq_printf(m,
-		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x notifier=%s base=0x%llx\n",
+		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u engine=%s hGr=0x%x token=0x%x notifier=%s base=0x%llx\n",
 		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
 		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
 		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
@@ -8244,6 +8655,8 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		   boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
 		   boot_stage.rmcall_channel,
 		   boot_stage.rmcall_channel_variant,
+		   gsp_phys_rmcall_engine_name(),
+		   boot_stage.rmcall_gr_object,
 		   boot_stage.rmcall_token,
 		   boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
 		   boot_stage.rmcall_notifier_base);
@@ -8867,11 +9280,15 @@ static const struct file_operations gsp_phys_rm_probe_fops = {
  * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs
  * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the
  * fresh-line PTE self-test (no GPA needed); all three require
- * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every
- * other value or state (parameter off, GPU unclaimed, stage 3d not run,
- * init-done not observed, unload already submitted, BAR0 unmapped) is a
- * logged no-op. The whole ladder is serialised with the unload/probe triggers
- * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched.
+ * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Writing 9
+ * re-runs one GR semaphore release on an already-built GR ladder
+ * (gsp_phys_rmcall_engine=1 snapshot through stage 3 with the GR object and
+ * work-submit token) without a full stage cycle; it needs no guest-memory
+ * parameter. Every other value or state (parameter off, GPU unclaimed,
+ * stage 3d not run, init-done not observed, unload already submitted, BAR0
+ * unmapped) is a logged no-op. The whole ladder is serialised with the
+ * unload/probe triggers by gsp_phys_rm_lock; at most one run per write; WPR2
+ * is never touched.
  */
 static ssize_t gsp_phys_rm_call_write(struct file *file,
 				      const char __user *ubuf, size_t len,
@@ -8890,8 +9307,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file,
 
 	if (kstrtoul(strim(buf), 0, &value))
 		return -EINVAL;
-	if (value < 1 || value > 8) {
-		pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n");
+	if (value < 1 || value > 9) {
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, 8 for the guest PTE self-test, or 9 to re-run one GR semaphore release on a GR ladder; 6-8 need gsp_phys_guest_mem=1)\n");
 		return len;
 	}
 	if (!gsp_phys_boot_enabled()) {
@@ -8932,6 +9349,21 @@ static ssize_t gsp_phys_rm_call_write(struct file *file,
 		return len;
 	}
 
+	/*
+	 * H1 trigger 9: re-run one GR semaphore release without a full ladder
+	 * cycle. Not a stage and not cumulative: it issues no RPC, only the
+	 * method-block submit, GPFIFO/USERD publish, doorbell and bounded
+	 * module-sem poll on the already-built GR channel. All gates live in
+	 * the helper (under gsp_phys_rm_lock); 6-8 below keep their existing
+	 * gsp_phys_guest_mem gating.
+	 */
+	if (value == 9) {
+		ret = gsp_phys_rmcall_gr_sem_rerun(bar0);
+		pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 returned %d (GR semaphore re-run; WPR2 untouched)\n",
+			ret);
+		return len;
+	}
+
 	if (value >= 6) {
 		/*
 		 * P6a guest-memory directions and self-test: strictly opt-in,
@@ -9505,6 +9937,17 @@ static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32,
 	      "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes");
 static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20,
 	      "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes");
+static_assert(sizeof(struct gsp_phys_rmcall_gr_alloc_params) ==
+	      GSP_PHYS_RMCALL_GR_PARAM_SIZE,
+	      "NV_GR_ALLOCATION_PARAMETERS must be 16 bytes (reference ABI)");
+static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, version) == 0,
+	      "NV_GR_ALLOCATION_PARAMETERS.version must be at offset 0");
+static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, flags) == 4,
+	      "NV_GR_ALLOCATION_PARAMETERS.flags must be at offset 4");
+static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, size) == 8,
+	      "NV_GR_ALLOCATION_PARAMETERS.size must be at offset 8");
+static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, caps) == 12,
+	      "NV_GR_ALLOCATION_PARAMETERS.caps must be at offset 12");
 static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368,
 	      "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)");
 static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8,