vgsp: full-frame strip assembly (origins+per-channel CE state, delta push fold), FERMI_TWOD executor (screen fills rc=0), RT-as-screen surface alias + win-drop auto, client replay retires (faulted=0, GR forward failed=0) - SESSION 153 r984-r997

1 files changed, 816 insertions, 15 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 3bb883d..4701331 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -5685,6 +5685,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 #define CE_M_LINE_COUNT		0x41c
 #define CE_M_PITCH_IN		0x410
 #define CE_M_PITCH_OUT		0x414
+#define CE_M_SRC_ORIGIN_X	0x744
+#define CE_M_SRC_ORIGIN_Y	0x748
+#define CE_M_DST_ORIGIN_X	0x74c
+#define CE_M_DST_ORIGIN_Y	0x750
 #define CE_M_SET_REMAP_CONST_A	0x700
 #define CE_M_SET_REMAP_CONST_B	0x704
 #define CE_M_SET_REMAP_COMPONENTS 0x708
@@ -5792,6 +5796,18 @@ struct svc_push_state {
 	u32	pitch_in, pitch_out;
 	u32	have_pitch_in, have_pitch_out;
 	u32	line_count;
+	/*
+	 * SESSION 153 CONT.25: C5B5 SRC_ORIGIN_X/Y (0x744/0x748) and
+	 * DST_ORIGIN_X/Y (0x74c/0x750). The X DDX's read/present paths
+	 * position each strip with SRC_ORIGIN (x in bytes, y in rows for
+	 * compSizeFactor == Bpp == 4; class C5B5 has no null remapper)
+	 * instead of folding it into OFFSET_IN. The linear shadow holds
+	 * the pixels linearly, so the effective operand becomes
+	 * base + origin_y * pitch + origin_x.
+	 */
+	u32	src_origin_x, src_origin_y;
+	u32	dst_origin_x, dst_origin_y;
+	u32	have_src_origin, have_dst_origin;
 	u32	sem_va_hi, sem_va_lo, sem_payload;
 	u32	have_sem;
 	u32	have_remap;
@@ -5815,6 +5831,66 @@ struct svc_push_state {
 	u32	qmd[SVC_QMD_WORDS];
 };
 
+/*
+ * SESSION 153 CONT.25: CE register state persists across pushes on the
+ * same channel/subchannel in hardware. The X DDX's read-image loop pushes
+ * the full setup once (SET_SRC_BLOCK_SIZE / OFFSET_IN / PITCH /
+ * LINE_LENGTH, a 112-byte push) and then per-strip DELTA pushes that only
+ * carry SRC_ORIGIN, SET_SEMAPHORE_A/PAYLOAD, OFFSET_OUT, (LINE_COUNT) and
+ * LAUNCH_DMA (a 52-byte push; raw witness: the delta launch decodes
+ * line=0/in=0/have_out=1/have_sem=1 on h=0x1000013). Decoding each push
+ * in isolation left line_length 0, so the transfer-shaped delta launch
+ * built NO op: every other 9-row strip never executed and the readback
+ * kept half the rows black (the visible even/odd banding). Remember the
+ * last full setup per channel and fill a delta launch's missing fields
+ * from it.
+ */
+struct svc_ce_saved {
+	u64	h_object;
+	u64	offset_in;
+	u32	pitch_in, pitch_out, line_length, line_count;
+	u32	src_phys_mode, dst_phys_mode;
+	u32	have_in, have_pitch_in, have_pitch_out;
+	u32	have_src_mode, have_dst_mode;
+};
+#define SVC_CE_SAVED_N	8
+static struct svc_ce_saved svc_ce_saved[SVC_CE_SAVED_N];
+static u32 svc_ce_saved_next;
+static u32 svc_cedelta_log;
+
+static struct svc_ce_saved *svc_ce_saved_slot(u64 h)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_CE_SAVED_N; i++)
+		if (svc_ce_saved[i].h_object == h)
+			return &svc_ce_saved[i];
+	return NULL;
+}
+
+static void svc_ce_saved_store(const struct svc_push_state *st, u64 h)
+{
+	struct svc_ce_saved *s = svc_ce_saved_slot(h);
+
+	if (!s) {
+		s = &svc_ce_saved[svc_ce_saved_next++ % SVC_CE_SAVED_N];
+		memset(s, 0, sizeof(*s));
+		s->h_object = h;
+	}
+	s->offset_in = st->offset_in;
+	s->pitch_in = st->pitch_in;
+	s->pitch_out = st->pitch_out;
+	s->line_length = st->line_length;
+	s->line_count = st->line_count;
+	s->src_phys_mode = st->src_phys_mode;
+	s->dst_phys_mode = st->dst_phys_mode;
+	s->have_in = st->have_in;
+	s->have_pitch_in = st->have_pitch_in;
+	s->have_pitch_out = st->have_pitch_out;
+	s->have_src_mode = st->have_src_mode;
+	s->have_dst_mode = st->have_dst_mode;
+}
+
 /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook
  * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */
 
@@ -12914,6 +12990,291 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu,
 	return true;
 }
 
+/* ------------------------------------------------------------------ */
+/* FERMI_TWOD (0x902d) executor                                        */
+/* ------------------------------------------------------------------ */
+/*
+ * SESSION 153-CONTINUED-26: the X DDX paints its surfaces (the screen
+ * 0x120400000 1280x720 X8R8G8B8 and the cursor 0x120370000 256x256
+ * A8R8G8B8) exclusively through subchannel-3 FERMI_TWOD methods: the
+ * SET_DST_* group, then RENDER_SOLID_PRIM (mode RECTS, color format,
+ * solid color, RENDER_SOLID_PRIM_POINT_X_Y packed pairs). Without this
+ * executor every 2D surface paint was a no-op and the surfaces the CE
+ * readback reads stayed unwritten (the screen byte pattern is all
+ * 0x00000000 - "the X server's drawing never executes").
+ *
+ * Method map: cl902d.h (FERMI_TWOD_A); encodings: NVA06F_DMA_*
+ * (1 INC / 3 NON_INC / 4 IMMD_DATA, data in header bits 28:16 / 5
+ * ONE_INC). Live-decoded per-frame dialog (guest RAM dump at
+ * 0x8d30c5f8): NON_INC 0x100 cnt17 (MME/global-render-enable window),
+ * INC 0x280 clip {0,0,0x7fff,0x7fff}, IMMD 0x204 layout, INC 0x214
+ * pitch, INC 0x218 {w,h,upper,lower}, IMMD 0x200 fmt, IMMD 0x2ac
+ * operation, INC 0x580 {mode,fmt,color}, NON_INC 0x5e0 {packed x/y
+ * point pairs}, IMMD 0x10c NOTIFY, IMMD 0x100 NOP.
+ *
+ * The DST VA is (upper<<32)|lower resolved through the SAME operand
+ * machinery every other engine op uses (svc_fill), so the 2D painting
+ * and the DDX's own CE readback of the same VA always agree by
+ * construction.
+ */
+struct svc_twod_state {
+	u64	h_object;
+	u32	dst_fmt, dst_layout, dst_bsize, dst_pitch, dst_w, dst_h;
+	u64	dst_off;
+	u32	have_dst;
+	u32	operation, rop;
+	u32	prim_mode, prim_fmt, prim_color;
+	u32	pt_x, pt_y, have_pt;
+	u64	fills, fill_bytes, misses;
+	u64	last_fill_hash, last_fill_va;
+	u32	fill_skips;
+	u32	dst_log, fill_log, unk_log;
+};
+
+#define SVC_TWOD_STATES	4
+static struct svc_twod_state svc_twod_tab[SVC_TWOD_STATES];
+
+unsigned gsp_sema_twod = 1;
+module_param_named(sema_twod, gsp_sema_twod, uint, 0644);
+MODULE_PARM_DESC(sema_twod,
+	"Execute FERMI_TWOD (0x902d) fills on subchannel 3 (default 1)");
+
+static struct svc_twod_state *svc_twod_get(struct svc_channel *ch)
+{
+	u32 i, free_i = SVC_TWOD_STATES;
+
+	for (i = 0; i < SVC_TWOD_STATES; i++) {
+		if (svc_twod_tab[i].h_object == ch->h_object)
+			return &svc_twod_tab[i];
+		if (!svc_twod_tab[i].h_object && free_i == SVC_TWOD_STATES)
+			free_i = i;
+	}
+	if (free_i == SVC_TWOD_STATES)
+		free_i = 0;	/* bounded table: steal slot 0 */
+	memset(&svc_twod_tab[free_i], 0, sizeof(svc_twod_tab[0]));
+	svc_twod_tab[free_i].h_object = ch->h_object;
+	return &svc_twod_tab[free_i];
+}
+
+static u32 svc_twod_bpp(u32 fmt)
+{
+	switch (fmt) {
+	case 0xe8: case 0xf8: case 0xe9: case 0xfb: case 0xfc: case 0xee:
+		return 2;
+	case 0xf3: case 0xf4: case 0xf7: case 0x1c:
+		return 1;
+	default:
+		return 4;
+	}
+}
+
+/* One RECTS primitive: fill the (xa,ya)-(xb,yb) axis-aligned rectangle. */
+static void svc_twod_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			  struct svc_twod_state *tw, u32 xa, u32 ya, u32 xb,
+			  u32 yb, u64 push_va)
+{
+	u32 bpp = svc_twod_bpp(tw->dst_fmt);
+	u32 pitch, x0, x1, y0, y1, w, h;
+	u64 done = 0;
+	int rc;
+
+	if (!tw->have_dst) {
+		if (tw->fill_log < 32) {
+			tw->fill_log++;
+			pr_info("sema-twod-fill: h=0x%x push=0x%llx SKIP (no DST set)\n",
+				ch->h_object, push_va);
+		}
+		return;
+	}
+	/*
+	 * Pitch: the PITCH layout carries it explicitly (the cursor dialog
+	 * pushes 0x400 for a 256-wide 4-bpp surface). The BLOCKLINEAR
+	 * screen dialog sets only w/h; the DDX's own CE readback of the
+	 * same surface rides pitch = w*bpp (5120 = 1280*4), so the linear
+	 * approximation uses w*bpp there - painting and readback agree.
+	 */
+	pitch = (tw->dst_layout == 1 && tw->dst_pitch) ? tw->dst_pitch :
+		tw->dst_w * bpp;
+	if (!bpp || !pitch || !tw->dst_w || !tw->dst_h) {
+		if (tw->fill_log < 32) {
+			tw->fill_log++;
+			pr_info("sema-twod-fill: h=0x%x SKIP (bpp=%u pitch=%u w=%u h=%u)\n",
+				ch->h_object, bpp, pitch, tw->dst_w,
+				tw->dst_h);
+		}
+		return;
+	}
+	x0 = xa < xb ? xa : xb;
+	x1 = xa < xb ? xb : xa;
+	y0 = ya < yb ? ya : yb;
+	y1 = ya < yb ? yb : ya;
+	/* Surface-bound clamp (the DDX keeps CLIP at 0x7fff). */
+	if (tw->dst_w >= 4 && tw->dst_w <= 0x8000) {
+		if (x0 >= tw->dst_w)
+			return;
+		if (x1 > tw->dst_w)
+			x1 = tw->dst_w;
+	}
+	if (tw->dst_h >= 4 && tw->dst_h <= 0x8000) {
+		if (y0 >= tw->dst_h)
+			return;
+		if (y1 > tw->dst_h)
+			y1 = tw->dst_h;
+	}
+	w = x1 - x0;
+	h = y1 - y0;
+	if (!w || !h)
+		return;
+	/*
+	 * Replay dedupe: the DDX re-submits the same dialog push buffer
+	 * for every repaint (the pump re-decodes ring entries), and the
+	 * first pass of a reused slot is often a replay (allow=0). The
+	 * fill is idempotent, so re-running is safe - but re-running on
+	 * every replay pass would write megabytes per second for zero
+	 * change. Execute once per (push VA, content hash); a changed
+	 * push (new points/colors/surface) hashes differently and runs.
+	 */
+	if (tw->fills && push_va == tw->last_fill_va &&
+	    tw->last_fill_hash == svc_pass_hash) {
+		tw->fill_skips++;
+		return;
+	}
+	tw->last_fill_va = push_va;
+	tw->last_fill_hash = svc_pass_hash;
+	/*
+	 * Row-wise fill through the shared operand machinery: the same
+	 * resolution chain (shadow -> pin -> landing -> walk -> scan) the
+	 * CE pitched fill uses, so the operand guards (LOWFILL /
+	 * FILL-PT / FILL-HOLD) apply to 2D painting as well.
+	 */
+	rc = svc_fill(vgpu, tw->dst_off + (u64)y0 * pitch + (u64)x0 * bpp,
+		      tw->prim_color, bpp, w, 0, CE_APERTURE_SYS, &done,
+		      h, pitch);
+	tw->fills++;
+	if (!rc)
+		tw->fill_bytes += (u64)w * bpp * h;
+	else
+		tw->misses++;
+	if (tw->fill_log < 256) {
+		tw->fill_log++;
+		pr_info("sema-twod-fill: h=0x%x va=0x%llx lay=%u fmt=0x%x pitch=%u rect=%u,%u-%u,%u color=0x%x bpp=%u rows=%u done=%llu rc=%d push=0x%llx\n",
+			ch->h_object, tw->dst_off, tw->dst_layout,
+			tw->dst_fmt, pitch, x0, y0, x1, y1, tw->prim_color,
+			bpp, h, done, rc, push_va);
+	}
+}
+
+static void svc_twod_method(struct nv_vgpu_device *vgpu,
+			    struct svc_channel *ch, u32 m, u32 v, bool exec,
+			    u64 push_va)
+{
+	struct svc_twod_state *tw = svc_twod_get(ch);
+
+	switch (m) {
+	case 0x200:
+		tw->dst_fmt = v;
+		break;
+	case 0x204:
+		tw->dst_layout = v;
+		break;
+	case 0x208:
+		tw->dst_bsize = v;
+		break;
+	case 0x214:
+		tw->dst_pitch = v;
+		break;
+	case 0x218:
+		tw->dst_w = v;
+		break;
+	case 0x21c:
+		tw->dst_h = v;
+		break;
+	case 0x220:
+		tw->dst_off = (tw->dst_off & 0xffffffffULL) |
+			      ((u64)(v & 0x1ffffff) << 32);
+		break;
+	case 0x224:
+		tw->dst_off = (tw->dst_off & ~0xffffffffULL) | v;
+		tw->have_dst = 1;
+		if (tw->dst_log < 64) {
+			tw->dst_log++;
+			pr_info("sema-twod-dst: h=0x%x va=0x%llx pitch=%u w=%u h=%u fmt=0x%x lay=%u bsz=0x%x\n",
+				ch->h_object, tw->dst_off, tw->dst_pitch,
+				tw->dst_w, tw->dst_h, tw->dst_fmt,
+				tw->dst_layout, tw->dst_bsize);
+		}
+		break;
+	case 0x2ac:
+		tw->operation = v;
+		break;
+	case 0x2a0:
+		tw->rop = v;
+		break;
+	case 0x580:
+		tw->prim_mode = v;
+		break;
+	case 0x584:
+		tw->prim_fmt = v;
+		break;
+	case 0x588:
+		tw->prim_color = v;
+		break;
+	case 0x5e0: {
+		u32 x = v & 0xffff, y = (v >> 16) & 0xffff;
+
+		if (tw->prim_mode == 4 /* RECTS: point pairs */) {
+			if (!tw->have_pt) {
+				tw->pt_x = x;
+				tw->pt_y = y;
+				tw->have_pt = 1;
+			} else {
+				tw->have_pt = 0;
+				if (exec && gsp_sema_twod)
+					svc_twod_fill(vgpu, ch, tw,
+						      tw->pt_x, tw->pt_y,
+						      x, y, push_va);
+			}
+		} else if (tw->unk_log < 256) {
+			tw->unk_log++;
+			pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x (point mode=%u)\n",
+				ch->h_object, m, v, tw->prim_mode);
+		}
+		break;
+	}
+	default:
+		if (tw->unk_log < 256) {
+			tw->unk_log++;
+			pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x\n",
+				ch->h_object, m, v);
+		}
+		break;
+	}
+}
+
+/*
+ * Dispatch a whole method run's data words. `inc` selects the NVA06F
+ * DMA semantics: INC_METHOD/ONE_INC (op 1/5) advance the method address
+ * per word (a contiguous register run); NON_INC (op 3) does NOT - every
+ * data word goes to the SAME method, which is how the DDX streams the
+ * RECTS point list into RENDER_SOLID_PRIM_POINT_X_Y (0x5e0; the 2-point
+ * rect and the 4-point double-band fills are one NON_INC run each, and
+ * its 17-word 0x100 run is a NO_OPERATION padding block).
+ */
+static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			 u32 addr, bool inc, u32 count, const u8 *data,
+			 u32 avail, bool exec, u64 push_va)
+{
+	u32 k;
+
+	for (k = 0; k < count && (u64)(k + 1) * 4 <= avail; k++) {
+		u32 v;
+
+		memcpy(&v, data + 4 * k, 4);
+		svc_twod_method(vgpu, ch, inc ? addr + 4 * k : addr, v, exec,
+				push_va);
+	}
+}
+
 /*
  * SESSION 148 perf (r862/r864): the RC watchdog's class2d notify cycle
  * keep-alive that needs no push decode and no captured VA. Under the
@@ -13624,6 +13985,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * NVC597_BEGIN (0x1618, an IMMD method - its immediate is
 		 * the topology in the count field) on the attached
 		 * subchannel of the attached channel opens the section.
+		 * SESSION 153-CONTINUED-26: the glxgears-era client draw
+		 * pushes carry no 0x1618 marker - their first run is the
+		 * C597 GL render-target setup (INC 0x840, {1, rtVA, w, h,
+		 * fmt, blksz}) followed by the 0x1b0 draw list - so the RT
+		 * setup opens the section too. Both shapes are gated on
+		 * the sticky attachment, the channel class, and a closed
+		 * section.
 		 * The window starts at gl_win_start (the first C597 state
 		 * method after the last foreign-subchannel run); the state
 		 * methods before the marker are re-scanned so their operand
@@ -13636,8 +14004,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		 * (the section close at the END method / push tail), which
 		 * svc_gl_flush slices into the verbatim 3D replay.
 		 */
-		if (gsp_sema_gl_xlate && !ch->engine_type && op == 4 &&
-		    addr == 0x1618 && !svc_gl.seen &&
+		if (gsp_sema_gl_xlate && !ch->engine_type &&
+		    ((op == 4 && addr == 0x1618) ||
+		     (op == 1 && addr == 0x840 && count >= 4)) &&
+		    !svc_gl.seen &&
 		    svc_gl_att_has(ch->h_object, subch)) {
 			u64 keep_rel = svc_gl.rel_va;
 			u32 keep_pay = svc_gl.rel_payload;
@@ -13649,6 +14019,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			svc_gl.start_off = wstart;
 			svc_gl.rel_va = keep_rel;
 			svc_gl.rel_payload = keep_pay;
+			/*
+			 * SESSION 153-CONTINUED-26: when the RT-setup run is
+			 * the opener, its address word IS the draw's render
+			 * target - put it into the candidate page set, or the
+			 * forwarded replay's shadow relocation cannot rewrite
+			 * the RT VA and the physical launch faults/times out
+			 * (r990: `GR forward failed rc=-110` on every client
+			 * draw while the X section's 7-pointer capture ran).
+			 * Must run AFTER the section reset above.
+			 */
+			if (op == 1 && addr == 0x840 && off + 12 <= len) {
+				/*
+				 * SESSION 153-CONTINUED-27: the run is
+				 * [upper, lower, w, h, fmt, blksz] - the VA
+				 * is ((upper & 0xff) << 32) | lower exactly
+				 * like every other pair recognizer in
+				 * svc_gl_field. Reading only the low word
+				 * added 0x20400000 for a surface whose true
+				 * VA is 0x120400000 (the X screen), so the
+				 * backend's shadow relocation MISSed it and
+				 * the physical replay wrote the raw VA with
+				 * no PTE chain (r992 FAULT_PDE @ +0xa000).
+				 */
+				u32 v0 = 0, v1 = 0;
+
+				memcpy(&v0, buf + off + 4, 4);
+				memcpy(&v1, buf + off + 8, 4);
+				svc_gl_add_ptr(((u64)(v0 & 0xff) << 32) |
+					       (u64)(v1 & 0xfffffffc));
+			}
 			if (wstart < off)
 				svc_gl_scan_window(wstart, off, buf, len,
 						   subch);
@@ -13677,6 +14077,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				svc_gl_shader_type[j] =
 					(u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf);
 		}
+		/*
+		 * SESSION 153-CONTINUED-26: FERMI_TWOD (0x902d) IMMD_DATA
+		 * methods on subchannel 3 (SET_DST_FORMAT 0x200, layout,
+		 * SET_OPERATION 0x2ac, NOTIFY 0x10c, ...). The value rides
+		 * in the header's bits 28:16. State only; no execution.
+		 */
+		if (gsp_sema_twod && !ch->engine_type &&
+		    subch == SVC_2D_SUBCH && op == 4 &&
+		    addr >= 0x0100 && addr <= 0x1400)
+			svc_twod_method(vgpu, ch, addr,
+					(hdr >> 16) & 0xfff,
+					allow_ops || trusted, push_va);
+
 		if (op == 4 || op == 7) {
 			/* Header-only methods (immediate data in the header
 			 * for op 4; end-of-push segment for op 7). */
@@ -13867,6 +14280,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			}
 		}
 
+		/*
+		 * SESSION 153-CONTINUED-26: FERMI_TWOD NON_INC/ONE_INC runs
+		 * on subchannel 3. The dialogs' whole body is NON_INC (the
+		 * 17-word context window and the point pairs); decode the
+		 * data words as cl902d methods before the generic
+		 * whole-run skip below consumes them.
+		 */
+		if (gsp_sema_twod && !ch->engine_type &&
+		    subch == SVC_2D_SUBCH && (op == 3 || op == 5) &&
+		    addr >= 0x0100 && addr <= 0x1400)
+			svc_twod_run(vgpu, ch, addr, op == 5 ? true : false,
+				     count, buf + off + 4, len - off - 4,
+				     allow_ops || trusted, push_va);
+
 		if (op == 3 || op == 5) {
 			/*
 			 * NON_INC / ONE_INC methods carry their own data
@@ -13899,6 +14326,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			memcpy(&v4, buf + off + 20, 4);
 
 		/*
+		 * SESSION 153-CONTINUED-26: FERMI_TWOD INC runs on
+		 * subchannel 3 (the RENDER_SOLID_PRIM mode/format/color
+		 * triplet at 0x580, the DST group, clip). The three
+		 * notify shapes (0x100 cnt<=1, 0x104, 0x108, 0x10c) stay
+		 * with the watchdog handler below; everything else in the
+		 * cl902d space is decoded here and must not fall into the
+		 * CE method chain.
+		 */
+		if (gsp_sema_twod && !ch->engine_type &&
+		    subch == SVC_2D_SUBCH && op == 1 &&
+		    addr >= 0x0100 && addr <= 0x1400 &&
+		    !(addr == 0x0100 && count <= 1) &&
+		    addr != 0x0104 && addr != 0x0108 && addr != 0x010c) {
+			svc_twod_run(vgpu, ch, addr, true, count,
+				     buf + off + 4, len - off - 4,
+				     allow_ops || trusted, push_va);
+			off += 4 + count * 4;
+			continue;
+		}
+
+		/*
 		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
 		 * contiguous (0x400..0x40c) and the driver pushes them as a
 		 * single INC_METHOD run (count 2 or 4). Apply every word of
@@ -14601,12 +15049,118 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			st.pitch_out = v0 & 0xffffff;
 			st.have_pitch_out = 1;
 			break;
+		/*
+		 * SESSION 153 CONT.25: SRC_ORIGIN_X/Y (0x744/0x748) and
+		 * DST_ORIGIN_X/Y (0x74c/0x750), pushed as a count-2 (X,Y)
+		 * or count-4 (X,Y,X,Y) INC run (nv_xaa_kepler-style
+		 * SetSrcOrigin: x already scaled by compSizeFactor == Bpp
+		 * for this class). No other class uses these method
+		 * offsets (cl902d/cla06f clear).
+		 */
+		case CE_M_SRC_ORIGIN_X:
+		case CE_M_SRC_ORIGIN_Y:
+		case CE_M_DST_ORIGIN_X:
+		case CE_M_DST_ORIGIN_Y:
+			{
+				u32 vv[4] = { v0, v1, v2, v3 };
+				u32 k, n = count > 4 ? 4 : count;
+
+				for (k = 0; k < n; k++) {
+					switch (addr + 4 * k) {
+					case CE_M_SRC_ORIGIN_X:
+						st.src_origin_x = vv[k];
+						st.have_src_origin = 1;
+						break;
+					case CE_M_SRC_ORIGIN_Y:
+						st.src_origin_y = vv[k];
+						st.have_src_origin = 1;
+						break;
+					case CE_M_DST_ORIGIN_X:
+						st.dst_origin_x = vv[k];
+						st.have_dst_origin = 1;
+						break;
+					case CE_M_DST_ORIGIN_Y:
+						st.dst_origin_y = vv[k];
+						st.have_dst_origin = 1;
+						break;
+					}
+				}
+			}
+			break;
 		case CE_M_LAUNCH_DMA: {
 			u32 dma = v0;
 			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);
 			u32 n_ops_before = n_ops;
 			bool dma_served = false;
 
+			/*
+			 * SESSION 153 CONT.25: remember a full setup (this is
+			 * the push that carried LINE_LENGTH + OFFSET_IN) as
+			 * the channel's persistent CE state, then fill a
+			 * transfer-shaped DELTA launch (the driver's per-strip
+			 * follow-up: no LINE_LENGTH/OFFSET_IN of its own,
+			 * OFFSET_OUT and the semaphore release present - the
+			 * witnessed 52-byte pushes on h=0x1000013) from it so
+			 * it builds its op instead of being silently dropped.
+			 */
+			if (allow_ops && st.line_length && st.have_in)
+				svc_ce_saved_store(&st, ch->h_object);
+			if (transfer != 0 && allow_ops && st.have_out &&
+			    st.have_sem &&
+			    (!st.line_length || !st.have_in)) {
+				struct svc_ce_saved *s =
+					svc_ce_saved_slot(ch->h_object);
+
+				if (s && s->line_length && s->have_in) {
+					if (!st.line_length) {
+						st.line_length =
+							s->line_length;
+						st.line_count =
+							s->line_count;
+					}
+					if (!st.have_in) {
+						st.offset_in =
+							s->offset_in;
+						st.have_in = 1;
+					}
+					if (!st.have_pitch_in &&
+					    s->have_pitch_in) {
+						st.pitch_in = s->pitch_in;
+						st.have_pitch_in = 1;
+					}
+					if (!st.have_pitch_out &&
+					    s->have_pitch_out) {
+						st.pitch_out = s->pitch_out;
+						st.have_pitch_out = 1;
+					}
+					if (!st.have_src_mode &&
+					    s->have_src_mode) {
+						st.src_phys_mode =
+							s->src_phys_mode;
+						st.have_src_mode = 1;
+					}
+					if (!st.have_dst_mode &&
+					    s->have_dst_mode) {
+						st.dst_phys_mode =
+							s->dst_phys_mode;
+						st.have_dst_mode = 1;
+					}
+					if (gsp_sema_dbg &&
+					    svc_cedelta_log < 64) {
+						svc_cedelta_log++;
+						pr_info("sema-cedelta: h=0x%x push=0x%llx dma=0x%08x filled line=%u lc=%u in=0x%llx pi=%u po=%u sx=%u sy=%u (engine state inherited)\n",
+							ch->h_object, push_va,
+							dma, st.line_length,
+							st.line_count,
+							st.offset_in,
+							st.pitch_in,
+							st.pitch_out,
+							st.src_origin_x,
+							st.src_origin_y);
+					}
+				}
+			}
+
 			if (gsp_sema_dbg &&
 			    svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) {
 				svc.lda_log++;
@@ -14708,6 +15262,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 
 					m->type = 0;
 					m->dst = st.offset_out;
+					/* SESSION 153 CONT.25: blocklinear
+					 * destination strip position. */
+					if (st.have_dst_origin) {
+						u32 dp = (st.have_pitch_out &&
+							  st.pitch_out) ?
+							st.pitch_out :
+							st.line_length;
+
+						m->dst +=
+							(u64)st.dst_origin_y *
+							dp +
+							st.dst_origin_x;
+					}
 					/*
 					 * r367: the driver's own memset push
 					 * (channel_utils.c) proves the
@@ -14856,6 +15423,38 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					m->src_aperture = st.have_src_mode ?
 						st.src_phys_mode : CE_APERTURE_VID;
 					/*
+					 * SESSION 153 CONT.25: a BLOCKLINEAR
+					 * operand positions each strip with
+					 * SRC/DST_ORIGIN (x in bytes, y in
+					 * rows) instead of folding it into
+					 * OFFSET_IN/OUT. The linear shadow
+					 * holds the pixels linearly, so the
+					 * effective operand is
+					 * base + origin_y*pitch + origin_x.
+					 */
+					if (st.have_src_origin) {
+						u32 sp = (st.have_pitch_in &&
+							  st.pitch_in) ?
+							st.pitch_in :
+							st.line_length;
+
+						m->src +=
+							(u64)st.src_origin_y *
+							sp +
+							st.src_origin_x;
+					}
+					if (st.have_dst_origin) {
+						u32 dp = (st.have_pitch_out &&
+							  st.pitch_out) ?
+							st.pitch_out :
+							st.line_length;
+
+						m->dst +=
+							(u64)st.dst_origin_y *
+							dp +
+							st.dst_origin_x;
+					}
+					/*
 					 * r402: a 2D copy (cudaMemcpy2D):
 					 * LINE_COUNT rows of
 					 * LINE_LENGTH_IN bytes with
@@ -14911,7 +15510,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					if (gsp_sema_dbg &&
 					    (st.have_pitch_out ||
 					     st.have_pitch_in) &&
-					    svc.op2d_raw_log < 12) {
+					    svc.op2d_raw_log < 32) {
 						u32 dl = len < 220 ? len : 220;
 
 						svc.op2d_raw_log++;
@@ -14926,8 +15525,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 							st.pitch_in,
 							m->dst, m->src,
 							(u32)m->bytes, dma);
-						pr_info("sema-op2d-raw: raw %*phN\n",
-							dl, buf);
+						/* %*phN caps at 64 bytes;
+						 * print the push whole in
+						 * 64-byte halves. */
+						pr_info("sema-op2d-raw: raw[0] %*phN\n",
+							dl > 64 ? 64 : dl,
+							buf);
+						if (dl > 64)
+							pr_info("sema-op2d-raw: raw[1] %*phN\n",
+								dl - 64,
+								buf + 64);
 					}
 					n_ops++;
 					ops_seq++;
@@ -15377,7 +15984,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * in one op"). Bounded.
 					 */
 					if (lm->type == 1)
-						pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n",
+						pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%llu rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u so=%u/%u do=%u/%u\n",
 							ch->h_object, lm->dst,
 							lm->src, lm->bytes,
 							lm->rows, lm->dpitch,
@@ -15387,7 +15994,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 							st.have_pitch_out,
 							st.pitch_out,
 							st.have_pitch_in,
-							st.pitch_in);
+							st.pitch_in,
+							st.have_src_origin,
+							st.src_origin_y,
+							st.have_dst_origin,
+							st.dst_origin_y);
 				}
 			}
 
@@ -24214,7 +24825,7 @@ static void svc_fwd_hold_direct_note(void)
  * A bounded fallback run (sweep, when no forward is owed) keeps the list
  * from stalling when the launch was refused.
  */
-#define SVC_GR_COPY_PEND	8
+#define SVC_GR_COPY_PEND	64
 struct svc_gr_copy_pend {
 	bool valid;
 	u32 h_object;
@@ -24234,6 +24845,55 @@ static u32 svc_gr_copy_pend_next;
 static u64 svc_gr_copy_run_total;
 static u64 svc_gr_copy_run_log;
 
+/*
+ * SESSION 153 CONT.25: recently-executed copy identities. The sweep
+ * re-decodes the ring continuation every pass, so an entry it just ran
+ * would be re-added (the valid-dedup above only covers live entries) and
+ * re-run forever - with a bounded per-pass run count that pins the runner
+ * on the same first slots. The content hash changes whenever the driver
+ * rewrites the push (new frame: the semaphore payload advances), so a
+ * (push_va, hash, src, dst, bytes, rows) match means "this exact strip
+ * already ran this frame".
+ */
+#define SVC_GR_COPY_DONE_N	512
+struct svc_gr_copy_done_rec {
+	u64 push_va, hash, src, dst;
+	u32 bytes, rows;
+};
+static struct svc_gr_copy_done_rec svc_gr_copy_done[SVC_GR_COPY_DONE_N];
+static u32 svc_gr_copy_done_next;
+
+static bool svc_gr_copy_done_lookup(u64 push_va, u64 hash, u64 src,
+				    u64 dst, u32 bytes, u32 rows)
+{
+	u32 i;
+
+	for (i = 0; i < SVC_GR_COPY_DONE_N; i++)
+		if (svc_gr_copy_done[i].push_va == push_va &&
+		    svc_gr_copy_done[i].hash == hash &&
+		    svc_gr_copy_done[i].src == src &&
+		    svc_gr_copy_done[i].dst == dst &&
+		    svc_gr_copy_done[i].bytes == bytes &&
+		    svc_gr_copy_done[i].rows == rows)
+			return true;
+	return false;
+}
+
+static void svc_gr_copy_done_store(u64 push_va, u64 hash, u64 src,
+				   u64 dst, u32 bytes, u32 rows)
+{
+	struct svc_gr_copy_done_rec *d =
+		&svc_gr_copy_done[svc_gr_copy_done_next++ %
+				  SVC_GR_COPY_DONE_N];
+
+	d->push_va = push_va;
+	d->hash = hash;
+	d->src = src;
+	d->dst = dst;
+	d->bytes = bytes;
+	d->rows = rows;
+}
+
 static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash,
 				 u64 src, u64 dst, u32 bytes, u32 rows,
 				 u32 dpitch, u32 spitch, u32 dma, u32 remap)
@@ -24242,6 +24902,8 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash,
 
 	if (!bytes || !rows || !src || !dst)
 		return;
+	if (svc_gr_copy_done_lookup(push_va, hash, src, dst, bytes, rows))
+		return;	/* this exact strip already ran for this content */
 	for (i = 0; i < SVC_GR_COPY_PEND; i++) {
 		struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i];
 
@@ -24291,17 +24953,33 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash,
  * when no forward is owed. The executor is the same svc_copy() the CE/GCE
  * op path uses (handles vidmem shadow sources and sysmem destinations).
  */
+/*
+ * SESSION 153 CONT.25: bounded per-call run count for the pending
+ * copy-out list. The X DDX readback keeps at most two strips in flight
+ * (the two scratch halves), so running the whole queue in one sweep pass
+ * overruns the server's consumption and every frame reads the newest
+ * strips only. Pace at a few per pass; 0 = unlimited (historical).
+ */
+static uint gsp_sema_gr_copy_run_max;
+module_param_named(sema_gr_copy_run_max, gsp_sema_gr_copy_run_max, uint, 0644);
+MODULE_PARM_DESC(sema_gr_copy_run_max,
+	"Max pending GRCE copy-outs executed per sweep pass (0 = unlimited)");
+
 static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu)
 {
-	u32 i;
+	u32 i, ran = 0;
 
 	for (i = 0; i < SVC_GR_COPY_PEND; i++) {
+		if (gsp_sema_gr_copy_run_max &&
+		    ran >= gsp_sema_gr_copy_run_max)
+			break;
 		struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i];
 		u64 done = 0;
 		int rc;
 
 		if (!p->valid)
 			continue;
+		ran++;
 		rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0,
 			      SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done,
 			      p->rows, p->dpitch, p->spitch, p->h_object, 0,
@@ -24309,6 +24987,8 @@ static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu)
 		svc_gr_copy_run_total++;
 		if (rc == 0 || rc == SVC_COPY_FORWARDED) {
 			p->valid = false;
+			svc_gr_copy_done_store(p->push_va, p->hash, p->src,
+					       p->dst, p->bytes, p->rows);
 			if (svc_gr_copy_run_log < 64) {
 				svc_gr_copy_run_log++;
 				pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n",
@@ -29892,6 +30572,9 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 	u32 have_in = 0, have_out = 0;
 	u32 pitch_in = 0, pitch_out = 0;
 	u32 line_len = 0, line_count = 0;
+	u32 src_origin_x = 0, src_origin_y = 0;
+	u32 dst_origin_x = 0, dst_origin_y = 0;
+	u32 have_src_origin = 0, have_dst_origin = 0;
 	u32 last_dma = 0, last_remap = 0;
 	u32 cp_subch = ~0u;
 	bool used_tail = false;
@@ -30205,6 +30888,38 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 							     ((vv[k] & 0xffffu)
 							      << 16);
 				}
+			} else if (addr >= CE_M_SRC_ORIGIN_X &&
+				   addr <= CE_M_DST_ORIGIN_Y && n) {
+				/*
+				 * SESSION 153 CONT.25: the X DDX readback
+				 * pair positions each 6-row strip with
+				 * SRC_ORIGIN (x bytes / y rows) instead of
+				 * folding it into OFFSET_IN - the same
+				 * methods the main walk now decodes. The
+				 * strip y steps 6 per push (A: 120, B: 126,
+				 * A: 132, ...) and the copy must read
+				 * OFFSET_IN + origin_y*PITCH_IN.
+				 */
+				for (k = 0; k < n; k++) {
+					switch (addr + 4 * k) {
+					case CE_M_SRC_ORIGIN_X:
+						src_origin_x = vv[k];
+						have_src_origin = 1;
+						break;
+					case CE_M_SRC_ORIGIN_Y:
+						src_origin_y = vv[k];
+						have_src_origin = 1;
+						break;
+					case CE_M_DST_ORIGIN_X:
+						dst_origin_x = vv[k];
+						have_dst_origin = 1;
+						break;
+					case CE_M_DST_ORIGIN_Y:
+						dst_origin_y = vv[k];
+						have_dst_origin = 1;
+						break;
+					}
+				}
 			} else if (addr == 0x300 && count) {
 				u32 dma = w0;
 
@@ -30212,28 +30927,104 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 				 * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a
 				 * release-only / semaphore op - no data
 				 * movement, nothing to record).
+				 *
+				 * SESSION 153 CONT.25: remember a full setup
+				 * (LINE_LENGTH + OFFSET_IN present) as the
+				 * channel's persistent CE state and fill a
+				 * DELTA launch (the X DDX readback pair's
+				 * second push: no setup of its own, only
+				 * SRC_ORIGIN + SEM_A/PAYLOAD + OFFSET_OUT +
+				 * LAUNCH_DMA) from it, exactly like the main
+				 * walk - without this every odd strip of the
+				 * frame is dropped.
 				 */
 				last_dma = dma;
+				if (have_in && off_in && line_len) {
+					struct svc_push_state tst;
+
+					memset(&tst, 0, sizeof(tst));
+					tst.offset_in = off_in;
+					tst.pitch_in = pitch_in;
+					tst.pitch_out = pitch_out;
+					tst.line_length = line_len;
+					tst.line_count = line_count;
+					svc_ce_saved_store(&tst,
+							   ch->h_object);
+				}
+				if ((dma & 0x3) && have_out && off_out &&
+				    (!have_in || !line_len)) {
+					struct svc_ce_saved *sv =
+						svc_ce_saved_slot(
+							ch->h_object);
+
+					if (sv && sv->line_length &&
+					    sv->have_in) {
+						if (!line_len) {
+							line_len =
+								sv->line_length;
+							line_count =
+								sv->line_count;
+						}
+						if (!have_in) {
+							off_in =
+								sv->offset_in;
+							have_in = 1;
+						}
+						if (!pitch_in)
+							pitch_in =
+								sv->pitch_in;
+						if (!pitch_out)
+							pitch_out =
+								sv->pitch_out;
+					}
+				}
 				if ((dma & 0x3) && have_in && have_out &&
 				    line_len && line_count &&
 				    off_in && off_out) {
+					/*
+					 * SESSION 153 CONT.25: fold the
+					 * per-strip SRC/DST_ORIGIN into the
+					 * operands (bytes on X, rows on Y;
+					 * the linear shadow makes
+					 * base + y*pitch + x the truth).
+					 * line_len is BYTES for C5B5
+					 * (LINE_LENGTH_IN semantics), not
+					 * dwords: the old *4 inflated every
+					 * strip 4x.
+					 */
+					u64 src_f = off_in, dst_f = off_out;
+
+					if (have_src_origin)
+						src_f +=
+							(u64)src_origin_y *
+							(pitch_in ?
+							 pitch_in :
+							 line_len) +
+							src_origin_x;
+					if (have_dst_origin)
+						dst_f +=
+							(u64)dst_origin_y *
+							(pitch_out ?
+							 pitch_out :
+							 line_len) +
+							dst_origin_x;
 					if (inline_only)
 						svc_ahead_copy_in(vgpu,
 							ch->h_object, push_va,
 							svc_push_hash(buf,
 								      len),
-							off_in, off_out,
-							line_len * 4,
+							src_f, dst_f,
+							line_len,
 							line_count, dma);
 					else if (forward) {
 						svc_gr_copy_pend_add(ch->h_object,
 							push_va, svc_push_hash(buf,
 									       len),
-							off_in, off_out,
-							line_len * 4,
+							src_f, dst_f,
+							line_len,
 							line_count,
 							pitch_out ? pitch_out :
-								line_len * 4,
+								line_len,
 							pitch_in, dma,
 							last_remap);
 						cp_subch = subch;
@@ -31551,8 +32342,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			 * backend is off), run any pending GRCE copies so
 			 * they cannot sit forever (the completion edge is
 			 * the primary path).
+			 *
+			 * SESSION 153 CONT.25: also run when a forward IS
+			 * owed - the X DDX readback pair's copies have no
+			 * forwarded launch behind them, and a stuck
+			 * unoffered candidate (valid && !offered) blocked
+			 * this fallback forever: 1443 copy pends, 0 runs,
+			 * the server's per-copy completion wait timing out
+			 * on every strip. The completion edge still runs
+			 * them first when it fires; the runner retires
+			 * each entry once, so running early is bounded.
 			 */
-			if (gsp_sema_gr_xlate && !svc_gr_forward_pending())
+			if (gsp_sema_gr_xlate)
 				svc_gr_copy_pend_run(vgpu);
 		}