vgsp: BAR0-shadow aperture userd probe + full PCFG-window (0x88000-0x89000) doorbell dispatch - built/live r1011; measurement falsified BAR0-USERD premise for client GP_PUT (shadows static, only pos=0x88704 traps); X channel sub=60 via kick-family RAM watch - SESSION 153 r1011

2 files changed, 276 insertions, 29 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4701331..9c7d92c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -47,6 +47,7 @@
 #include <asm/unaligned.h>
 
 #include "nvidia-vgpu-vfio.h"
+#include "bar_emulation.h"
 #include "gsp_sema_service.h"
 #include "gsp_rm_emulation.h"
 #include "gsp_phys_booter.h"
@@ -642,6 +643,21 @@ struct svc_channel {
 	u8	gr_ram_userd_watch_n;
 	u32	gr_ram_userd_watch_log;		/* bounded add-log counter */
 	/*
+	 * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the GL client's channel
+	 * allocation declares its USERD in the FB aperture (0x18fc00-class
+	 * offsets) but the guest's GP_PUT writes land in the emulated BAR0
+	 * shadow at that offset. Watch that block across passes: a nonzero PUT
+	 * (or a PUT step) is the client's own live write; adopt it as the
+	 * channel's USERD in the BAR0 aperture so the pump iterates the ring.
+	 * bar0_userd_at is the first-sighting stamp; a stable nonzero PUT is
+	 * adopted after a short patience window so a single-submission client
+	 * (waiting on that submission) cannot deadlock the watch.
+	 */
+	u64	bar0_userd_watch_pa;	/* watched offset (0 = none) */
+	u32	bar0_userd_watch_put;	/* PUT at the first sighting */
+	unsigned long bar0_userd_watch_at;	/* jiffies at the first sighting */
+	u8	bar0_userd_log;		/* bounded per-channel log counter */
+	/*
 	 * SESSION 153-CONTINUED-20: bounded budget re-arms for the
 	 * kick-family watch sweep. When every watch entry has gone stale
 	 * (frozen pool) and the slice budget is spent, one re-arm keeps the
@@ -909,12 +925,27 @@ static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session st
 
 /* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */
 #define SVC_MAX_GUESTS		8
+/*
+ * SESSION 153-CONTINUED-34: PCFG doorbell window in the emulated BAR0 shadow.
+ * Work-submission doorbells the guest pokes after writing a GPFIFO entry can
+ * land anywhere in the PCFG window (0x88000..0x89000), not only at the
+ * virtual-function doorbell the trap path used to dispatch (0x88704). Because
+ * the BAR0 shadow is direct-mapped, most writes never reach the trap path at
+ * all, so the service thread also watches the window for changed words and
+ * feeds each new value to gsp_sema_service_doorbell (token format unchanged;
+ * the call only wakes the pump earlier - the pump itself is per-pass).
+ */
+#define SVC_BAR0_PCFG_LO	0x88000u
+#define SVC_BAR0_PCFG_HI	0x89000u
+#define SVC_BAR0_PCFG_WORDS	((SVC_BAR0_PCFG_HI - SVC_BAR0_PCFG_LO) / 4u)
 struct svc_guest_sess {
 	struct nv_vgpu_device	*vgpu;		/* NULL = free slot */
 	bool			active;		/* a channel of this guest is
 						 * registered until this guest's
 						 * last channel closes */
 	u64			last_build;	/* per-guest svc_rebuild freshness */
+	u32			pcfg_snap[SVC_BAR0_PCFG_WORDS];
+	bool			pcfg_have;	/* snapshot taken for this guest */
 };
 static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS];
 static DEFINE_SPINLOCK(svc_guest_lock);
@@ -1560,6 +1591,17 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len
 /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
 #define SVC_APERTURE_SYS	1
 #define SVC_APERTURE_FB		2
+/*
+ * SESSION 153-CONTINUED-34: the emulated BAR0 (register) shadow as a third
+ * aperture. The GL client's channel allocation reports its USERD block in the
+ * FB aperture (NV_ADDR_FBMEM, offset 0x18fc00/0x18fe00/0xfbfc00...) but the
+ * writes the guest actually makes to its GP_PUT land in the emulated BAR0
+ * shadow at that same offset (the vGPU guest driver programs its usermode
+ * block through the register window). Every other source reads the FB shadow
+ * at those offsets and finds nothing, so the pump never iterates the
+ * channel's GPFIFO and the channel retires with submissions=0.
+ */
+#define SVC_APERTURE_BAR0	3
 
 /*
  * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom
@@ -1571,7 +1613,31 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len
  */
 static inline u32 svc_ap_of(int ap)
 {
-	return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+	if (ap == SVC_APERTURE_FB)
+		return SVC_APERTURE_FB;
+	/* SESSION 153-CONT-34: the BAR0 selector must survive normalisation. */
+	if (ap == SVC_APERTURE_BAR0)
+		return SVC_APERTURE_BAR0;
+	return SVC_APERTURE_SYS;
+}
+
+/*
+ * SESSION 153-CONT-34: the emulated BAR0 shadow (the buffer the guest's
+ * register writes actually land in - it is mmap'd into QEMU and also the
+ * trap-path write target). NULL when the device has no emulation shadow.
+ */
+static void *svc_bar0_shadow(struct nv_vgpu_device *vgpu, size_t *size)
+{
+	struct nv_vgpu_bar_emul_ctx *bctx;
+
+	if (!vgpu)
+		return NULL;
+	bctx = vgpu->bar_emul_ctx;
+	if (!bctx || !bctx->bar0_shadow)
+		return NULL;
+	if (size)
+		*size = bctx->bar0_shadow_size;
+	return bctx->bar0_shadow;
 }
 
 /*
@@ -1591,6 +1657,16 @@ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
 		memcpy(buf, base + addr, len);
 		return 0;
 	}
+	/* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */
+	if (aperture == SVC_APERTURE_BAR0) {
+		size_t size = 0;
+		u8 *base = svc_bar0_shadow(vgpu, &size);
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(buf, base + addr, len);
+		return 0;
+	}
 	return svc_read(vgpu, addr, buf, len);
 }
 
@@ -1606,6 +1682,16 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
 		memcpy(base + addr, buf, len);
 		return 0;
 	}
+	/* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */
+	if (aperture == SVC_APERTURE_BAR0) {
+		size_t size = 0;
+		u8 *base = svc_bar0_shadow(vgpu, &size);
+
+		if (!base || addr + len > size)
+			return -EINVAL;
+		memcpy(base + addr, buf, len);
+		return 0;
+	}
 	return svc_write(vgpu, addr, buf, len);
 }
 
@@ -17457,6 +17543,52 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
 	}
 }
 
+/*
+ * SESSION 153-CONT-34: watch the emulated BAR0 shadow's PCFG window
+ * (0x88000..0x89000) for doorbell writes the guest made directly to the
+ * mmap'd shadow (those never reach the trap path). Each changed word is fed
+ * to the doorbell path, whose only effect is waking the pump earlier - the
+ * pump itself stays per-pass. The first call for a guest snapshots the window
+ * without firing, so pre-existing register values (the emulation's own init
+ * writes) cannot ring.
+ */
+static u32 svc_bar0_pcfg_log;
+
+static void svc_bar0_pcfg_scan(struct nv_vgpu_device *vgpu)
+{
+	struct svc_guest_sess *sess;
+	size_t bsz = 0;
+	u8 *base = svc_bar0_shadow(vgpu, &bsz);
+	u32 i;
+
+	if (!base || bsz < SVC_BAR0_PCFG_HI)
+		return;
+	sess = svc_guest_get(vgpu, true);
+	if (!sess)
+		return;
+	for (i = 0; i < SVC_BAR0_PCFG_WORDS; i++) {
+		u32 v;
+
+		memcpy(&v, base + SVC_BAR0_PCFG_LO + i * 4, 4);
+		if (!sess->pcfg_have) {
+			sess->pcfg_snap[i] = v;
+			continue;
+		}
+		if (v == sess->pcfg_snap[i])
+			continue;
+		sess->pcfg_snap[i] = v;
+		if (!v)
+			continue;	/* a clear, not a doorbell */
+		gsp_sema_service_doorbell(v);
+		if (svc_bar0_pcfg_log < 32) {
+			svc_bar0_pcfg_log++;
+			pr_info("vgsp-sema: PCFG doorbell (BAR0 shadow scan) off=0x%x val=0x%x\n",
+				SVC_BAR0_PCFG_LO + i * 4, v);
+		}
+	}
+	sess->pcfg_have = true;
+}
+
 static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
 {
 	u32 hdr, hi, lo, payload;
@@ -18173,6 +18305,102 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
 	return 0;
 }
 
+/*
+ * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's
+ * channel allocation declares its USERD in the FB aperture at a 0x18fc00/
+ * 0x18fe00-class offset, but the guest's GP_PUT/GP_GET writes for those
+ * channels land in the EMULATED BAR0 register shadow at that same offset
+ * (the vGPU guest driver programs its usermode/userd block through the
+ * register window; the FB shadow at the same offset holds nothing). Every
+ * existing source reads the FB/SYS shadows or scans guest RAM, so the block
+ * is invisible: svc_chan_userd returns -ENOENT, the pump never iterates the
+ * channel's GPFIFO ring and the channel retires with submissions=0.
+ *
+ * Watch policy (same spirit as the RAM-scan liveness latch): read the pair
+ * at bar0_shadow + userd_pa. A nonzero, coherent PUT is the guest's own
+ * write - the shadow is zeroed at device creation and no other writer
+ * targets that offset - so:
+ *   - PUT >= 2 (two publishes already): adopt immediately;
+ *   - PUT == 1 with a different value later (movement): adopt;
+ *   - PUT == 1 stable past SVC_BAR0_USERD_PATIENT_MS: adopt anyway, so a
+ *     client waiting on its single first submission cannot deadlock the
+ *     watch (the pump then consumes that very submission);
+ *   - unreadable/zero/incoherent: keep waiting (other sources unchanged).
+ * Adoption sets the BAR0 aperture so every later USERD read/write
+ * (svc_read_ap/svc_write_ap) and the pump's GP_GET write-back hit the same
+ * block the guest is updating. The walk seed mirrors the RAM-scan latch:
+ * the newest entries the client is waiting on, bounded to entries-1 slots.
+ */
+#define SVC_BAR0_USERD_PATIENT_MS 3000u
+
+static bool svc_bar0_userd_try(struct nv_vgpu_device *vgpu,
+			       struct svc_channel *ch, u32 *put, u32 *get)
+{
+	size_t bsz = 0;
+	u8 *base = svc_bar0_shadow(vgpu, &bsz);
+	u64 off = ch->userd_pa;
+	u32 p = 0, g = 0, span;
+
+	if (!base || !off)
+		return false;
+	if (off + SVC_USERD_GP_PUT + 4 > bsz)
+		return false;
+	/*
+	 * Only the FB-declared family is retried here: SYS-declared offsets
+	 * are real guest PAs the ordinary paths already read, and the CUDA/
+	 * kick channels resolve through their own sources before this point.
+	 */
+	if (ch->userd_as != SVC_APERTURE_FB)
+		return false;
+
+	memcpy(&p, base + off + SVC_USERD_GP_PUT, 4);
+	memcpy(&g, base + off + SVC_USERD_GP_GET, 4);
+
+	if (gsp_sema_dbg && (p || g) && ch->bar0_userd_log < 8) {
+		ch->bar0_userd_log++;
+		pr_info("sema-chan: BAR0 userd probe h=0x%x rank=%u off=0x%llx put=%u get=%u\n",
+			ch->h_object, ch->engine_type ? ch->ce_index
+						      : ch->gr_index,
+			off, p, g);
+	}
+	/* Idle, incoherent or an implausible register-looking value. */
+	if (!p || g > p || p > 0x200000u)
+		return false;
+
+	if (ch->bar0_userd_watch_pa != off) {
+		ch->bar0_userd_watch_pa = off;
+		ch->bar0_userd_watch_put = p;
+		ch->bar0_userd_watch_at = jiffies;
+		if (p >= 2)
+			goto adopt;
+		return false;	/* first sighting, p == 1: patience */
+	}
+	if (p != ch->bar0_userd_watch_put)
+		goto adopt;	/* movement: live writer */
+	if (time_after(jiffies, ch->bar0_userd_watch_at +
+		       msecs_to_jiffies(SVC_BAR0_USERD_PATIENT_MS)))
+		goto adopt;	/* stable nonzero past the patience window */
+	return false;
+
+adopt:
+	ch->userd_ok = true;
+	ch->userd_as = SVC_APERTURE_BAR0;
+	if (ch->gpfifo_entries > 1 && p) {
+		span = p < ch->gpfifo_entries ? p : ch->gpfifo_entries - 1;
+		ch->last_get = (p % ch->gpfifo_entries +
+				ch->gpfifo_entries - span) % ch->gpfifo_entries;
+	} else {
+		ch->last_get = 0;
+	}
+	ch->bar0_userd_watch_pa = 0;
+	*put = p;
+	*get = g;
+	pr_info("sema-chan: BAR0 USERD h=0x%x eng=%u ADOPT off=0x%llx put=%u get=%u ap=bar0 (walk seed last_get=%u; guest GP_PUT lives in the emulated BAR0 shadow)\n",
+		ch->h_object, ch->engine_type, off, p, g,
+		(u32)ch->last_get);
+	return true;
+}
+
 /* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */
 static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 				  struct svc_channel *ch, u64 *pa_out,
@@ -18349,6 +18577,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	 * at the layout offset.
 	 */
 	if (ch->engine_type != 0) {
+		/*
+		 * SESSION 153-CONT-34: before giving up on a CE channel, try
+		 * the registered offset read as a BAR0-shadow USERD - the GL
+		 * client's copy/2D channels declare the same FB offsets as its
+		 * GR channel and their GP_PUT writes land in the same shadow.
+		 */
+		if (svc_bar0_userd_try(vgpu, ch, put, get))
+			return 0;
 		svc_userd_sweep_sched(ch);
 		return -ENOENT;
 	}
@@ -18633,6 +18869,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			}
 		}
 	}
+	/*
+	 * SESSION 153-CONT-34: last resort for the GL client family - retry the
+	 * registered offset as a BAR0-shadow USERD. Every other source (layout,
+	 * ring-derived, RAM scan, FB sweep) has already missed the block whose
+	 * GP_PUT writes land in the emulated register window.
+	 */
+	if (svc_bar0_userd_try(vgpu, ch, put, get))
+		return 0;
 	return -ENOENT;
 }
 
@@ -36562,6 +36806,14 @@ static int svc_thread(void *arg)
 			svc_pb_discover(vgpu);
 
 			/*
+			 * SESSION 153-CONT-34: doorbell writes the guest made
+			 * directly into the mmap'd BAR0 shadow (PCFG window) -
+			 * feed them to the doorbell path so the pump is woken
+			 * without waiting for the next pass boundary.
+			 */
+			svc_bar0_pcfg_scan(vgpu);
+
+			/*
 			 * r580: the completion hint was hoisted above the
 			 * pumped fast path (it must run under load too); it is
 			 * not repeated here.
diff --git a/kernel/main.c b/kernel/main.c
index 2f35163..b5c7105 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -843,47 +843,42 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use
 		u8 tmp[8];
 		ssize_t ret;
 		static u32 bar0_log;
-		/*
-		 * SESSION 153-CONTINUED-23: the GL client's draw doorbell, if it
-		 * ever arrives through the trap path at all, must be visible -
-		 * widen the log from the single 0x88704 offset to every trapped
-		 * write in the PCFG window (0x88000..0x89000). Bounded: the BAR0
-		 * shadow is direct-mapped, so trap-path writes are rare; a cap
-		 * keeps a pathological guest from flooding dmesg.
-		 */
-		static u32 pcfg_log;
 
 		if (!vgpu->bar_emul_ctx)
 			return -ENODEV;
 		if (copy_from_user(tmp, buf, count))
 			return -EFAULT;
 
-		if ((u64)pos == 0x88704 ||
-		    ((u64)pos >= 0x88000 && (u64)pos < 0x89000 &&
-		     pcfg_log < 1024) ||
-		    (gsp_sema_dbg && bar0_log < 64)) {
+		if ((u64)pos >= 0x88000 && (u64)pos < 0x89000) {
+			/*
+			 * SESSION 153-CONTINUED-34: ANY write in the PCFG
+			 * window is a submission doorbell, not only the
+			 * virtual-function doorbell at 0x88704 the old code
+			 * dispatched. Feed every 4-byte write to the doorbell
+			 * path so the pump is woken for the client's channel
+			 * kicks no matter which register the driver poked
+			 * (the service thread also watches the mmap'd shadow
+			 * for writes that never trap). Logging stays bounded.
+			 */
 			static u32 db_log;
 			u64 v = 0;
 
 			memcpy(&v, tmp, count);
-
-			if ((u64)pos == 0x88704) {
-				if (gsp_sema_dbg && db_log < 512) {
-					pr_info("vgsp: DOORBELL write pos=0x%llx size=%zu val=0x%llx\n",
-						(u64)pos, count, v);
-					db_log++;
-				}
-				if (count == 4)
-					gsp_sema_service_doorbell((u32)v);
-			} else if ((u64)pos >= 0x88000 && (u64)pos < 0x89000) {
+			if (gsp_sema_dbg && db_log < 512) {
 				pr_info("vgsp: PCFG write pos=0x%llx size=%zu val=0x%llx\n",
 					(u64)pos, count, v);
-				pcfg_log++;
-			} else {
-				pr_info("vgsp: BAR0 write pos=0x%llx size=%zu val=0x%llx\n",
-					(u64)pos, count, v);
-				bar0_log++;
+				db_log++;
 			}
+			if (count == 4)
+				gsp_sema_service_doorbell((u32)v);
+		} else if ((u64)pos == 0x88704 ||
+			   (gsp_sema_dbg && bar0_log < 64)) {
+			u64 v = 0;
+
+			memcpy(&v, tmp, count);
+			pr_info("vgsp: BAR0 write pos=0x%llx size=%zu val=0x%llx\n",
+				(u64)pos, count, v);
+			bar0_log++;
 		}
 
 		ret = nv_vgpu_bar_emul_write(vgpu, 0, (u64)pos, tmp, count);