vgsp-dual: stage B+C minimal per-mdev separation (G1-G3, G5, G6, G7, G8a)

Goal 1.0: two mdevs + two QEMU guests running CUDA simultaneously.
Single-mdev behaviour is kept identical: the legacy guest-RAM record
still serves the first/only open mdev, the channel table degenerates to
the historical key, and the session flush fires at the same
last-channel-closed transition.

- G6: guest-RAM backing {file, path, ranges} per mdev (legacy record +
  up to 4 vgpu-bound records); debugfs gsp-phys-guest-ram accepts an
  mdev UUID as an extra first token and keeps the 4-token form; records
  released on mdev cleanup and physical release.
- G7: gsp_phys_gpa_to_hpa_vgpu(vgpu, gpa, *hpa); NULL keeps the
  historical first-open resolution. Forwarding operands pass the
  request's vgpu; the single-guest boot-ladder pin passes NULL.
- G1: the sema-service thread services every open mdev each pass.
- G2: channel table keyed by (vgpu, h_object); per-guest ranks;
  counter/scrub/opdef/pump snapshot+writeback matches per vgpu;
  scrubber records per guest.
- G3: per-guest session records drive the flush; the shared learned
  tables (staged G4) are only flushed when no guest has live channels.
- G5: hold/abort keyed by (vgpu, handle); dead-owner guards on the
  hold resolution paths; single-inflight serialization kept.
- G8a: guest-GPA pin refused while more than one mdev is open.

Build: make -j8 clean success (vgsp.ko). No module load. See
DUAL-REPORT.md for the per-item status and live-validation plan.

1 files changed, 492 insertions, 152 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index f8925df..c1eaf80 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -162,6 +162,10 @@ struct sema_map_entry {
 
 struct svc_channel {
 	bool	valid;
+	struct nv_vgpu_device *vgpu;	/* owning mdev (guest session); the
+					 * table is keyed by (vgpu, h_object)
+					 * because two guests' stock allocators
+					 * choose identical handles */
 	u32	h_object;
 	u32	engine_type;
 	u32	cid;		/* RM channel id (doorbell token vector) */
@@ -449,12 +453,65 @@ static u32 svc_ring_n;
  * client spins). One clean boot always works, so make the next session look
  * like one: when the channel set empties, request a flush and let the next
  * rebuild start from the same empty state as a fresh boot.
+ *
+ * Dual-guest staging (goal 1.0): the discovery tables are still shared
+ * between guests (per-guest learned tables are the staged G4 remainder), so
+ * the flush is requested only when the channel set is globally empty. The
+ * active decision is made per guest (svc_guests below): one guest closing
+ * its last channel never flushes while another guest still has live
+ * channels - otherwise the departing guest would wipe the live guest's
+ * pins/maps and stall it. Single-guest behaviour is unchanged: the one
+ * guest's last-channel close is also the global empty transition.
  */
 static u8 svc_sess_flush;
 static u8 svc_sess_active;	/* a channel of a session has registered (until flush) */
 static struct completion svc_sess_flush_done;	/* a pending flush has executed */
 static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session state) */
 
+/* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */
+#define SVC_MAX_GUESTS		8
+struct svc_guest_sess {
+	struct nv_vgpu_device	*vgpu;		/* NULL = free slot */
+	bool			active;		/* a channel of this guest is
+						 * registered until this guest's
+						 * last channel closes */
+	u64			last_build;	/* per-guest svc_rebuild freshness */
+};
+static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS];
+static DEFINE_SPINLOCK(svc_guest_lock);
+
+/*
+ * Find (or, with create, allocate) the session record for one mdev. The
+ * records are pointer-keyed; a removed mdev's record is freed by
+ * gsp_sema_service_vgpu_cleanup() so the pointer can never match a newly
+ * allocated device.
+ */
+static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu,
+					    bool create)
+{
+	struct svc_guest_sess *sess = NULL;
+	unsigned long flags;
+	int i;
+
+	if (!vgpu)
+		return NULL;
+	spin_lock_irqsave(&svc_guest_lock, flags);
+	for (i = 0; i < SVC_MAX_GUESTS; i++) {
+		if (svc_guests[i].vgpu == vgpu) {
+			sess = &svc_guests[i];
+			break;
+		}
+		if (create && !sess && !svc_guests[i].vgpu)
+			sess = &svc_guests[i];
+	}
+	if (sess && create && !sess->vgpu) {
+		memset(sess, 0, sizeof(*sess));
+		sess->vgpu = vgpu;
+	}
+	spin_unlock_irqrestore(&svc_guest_lock, flags);
+	return sess;
+}
+
 /*
  * CeUtils scrubber channel (the guest PMA's internal CE channel created at
  * driver init). When the PMA must hand out scrubbed pages it waits on the
@@ -476,6 +533,7 @@ static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session st
  * block count (so the scrub wait completes). Idempotent, forward-only.
  */
 struct svc_scrub {
+	struct nv_vgpu_device *vgpu;	/* owning mdev (NULL = free slot) */
 	bool	userd_ok;	/* a 4096-entry channel was registered */
 	u32	h_object;	/* handle of the registered scrubber channel */
 	u64	userd_pa;
@@ -492,7 +550,34 @@ struct svc_scrub {
 	u32	pa_gen;		/* scanner registration generation (logged) */
 	u32	log_budget;
 };
-static struct svc_scrub svc_scrub;
+/*
+ * One scrubber record per guest: every guest's PMA has its own 4096-entry
+ * scrubber channel, and the authored words must be written through the
+ * owning guest's address space (writing guest A's PA through guest B's VFIO
+ * device would corrupt B). Keyed by vgpu pointer, freed by
+ * gsp_sema_service_vgpu_cleanup().
+ */
+static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS];
+
+static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create)
+{
+	struct svc_scrub *s = NULL;
+	int i;
+
+	if (!vgpu)
+		return NULL;
+	for (i = 0; i < SVC_MAX_GUESTS; i++) {
+		if (svc_scrubs[i].vgpu == vgpu)
+			return &svc_scrubs[i];
+		if (create && !s && !svc_scrubs[i].vgpu)
+			s = &svc_scrubs[i];
+	}
+	if (s) {
+		memset(s, 0, sizeof(*s));
+		s->vgpu = vgpu;
+	}
+	return s;
+}
 static u8 svc_pb_passes;
 static u64 svc_pb_last;
 /*
@@ -1066,7 +1151,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 	}
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu &&
+		    svc.chan[i].h_object == desc->h_object) {
 			ch = &svc.chan[i];
 			break;
 		}
@@ -1079,7 +1165,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		u64 keep_release_pa = 0;
 		u8 keep_gr_index = 0, keep_ce_index = 0;
 		DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS);
-		bool same = ch->valid && ch->h_object == desc->h_object;
+		bool same = ch->valid && ch->vgpu == desc->vgpu &&
+			    ch->h_object == desc->h_object;
 
 		/*
 		 * A re-registration of the same channel must not lose the
@@ -1102,18 +1189,22 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 			bitmap_copy(ch->rel_done, keep_rel_done,
 				    SVC_GPFIFO_MAX_BITS);
 		/*
-		 * Rank among the live CE channels. The RM hands out object
-		 * handles in creation order and the guest's UVM allocates one
-		 * push buffer, ring and tracking semaphore per channel in that
-		 * same order, so the rank is what ties a channel to its ring
-		 * offset and its push-buffer base (see svc_ring_autopin /
-		 * svc_pb_pair).
+		 * Rank among the live CE channels OF THIS GUEST. The RM hands
+		 * out object handles in creation order and the guest's UVM
+		 * allocates one push buffer, ring and tracking semaphore per
+		 * channel in that same order, so the rank is what ties a
+		 * channel to its ring offset and its push-buffer base (see
+		 * svc_ring_autopin / svc_pb_pair). The rank must not count the
+		 * other guest's channels: their handles are independent and
+		 * their allocators interleave.
 		 */
 		if (desc->engine_type) {
 			u8 rank = 0;
 
 			for (k = 0; k < SVC_MAX_CHANNELS; k++)
-				if (svc.chan[k].valid && svc.chan[k].engine_type &&
+				if (svc.chan[k].valid &&
+				    svc.chan[k].vgpu == desc->vgpu &&
+				    svc.chan[k].engine_type &&
 				    svc.chan[k].h_object < desc->h_object)
 					rank++;
 			ch->ce_index = rank;
@@ -1121,13 +1212,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 			u8 rank = 0;
 
 			for (k = 0; k < SVC_MAX_CHANNELS; k++)
-				if (svc.chan[k].valid && !svc.chan[k].engine_type &&
+				if (svc.chan[k].valid &&
+				    svc.chan[k].vgpu == desc->vgpu &&
+				    !svc.chan[k].engine_type &&
 				    svc.chan[k].h_object < desc->h_object)
 					rank++;
 			ch->gr_index = rank;
 		}
 		ch->valid = true;
 		svc_sess_active = 1;
+		ch->vgpu = desc->vgpu;
 		ch->h_object = desc->h_object;
 		ch->engine_type = desc->engine_type;
 		ch->cid = desc->cid;
@@ -1145,6 +1239,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 			ch->gr_index = keep_gr_index;
 			ch->ce_index = keep_ce_index;
 		}
+		/* Per-guest session flag: this guest has a live channel. */
+		{
+			struct svc_guest_sess *sess =
+				svc_guest_get(desc->vgpu, true);
+
+			if (sess)
+				sess->active = true;
+		}
 	}
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
 
@@ -1169,25 +1271,30 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		 */
 		if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va &&
 		    desc->userd_pa) {
+			struct svc_scrub *s = svc_scrub_get(desc->vgpu, true);
+
+			if (!s)
+				goto scrub_done;	/* table full: max guests */
 			/* Re-seed the counting on a fresh scrubber object. */
-			if (!svc_scrub.userd_ok ||
-			    svc_scrub.h_object != desc->h_object ||
-			    svc_scrub.gpfifo_va != desc->gpfifo_va) {
-				svc_scrub.seeded = false;
-				svc_scrub.submitted = 0;
+			if (!s->userd_ok ||
+			    s->h_object != desc->h_object ||
+			    s->gpfifo_va != desc->gpfifo_va) {
+				s->seeded = false;
+				s->submitted = 0;
 			}
-			svc_scrub.userd_ok = true;
-			svc_scrub.h_object = desc->h_object;
-			svc_scrub.userd_pa = desc->userd_pa;
-			svc_scrub.userd_as = desc->userd_as ? desc->userd_as
-							    : SVC_APERTURE_SYS;
-			svc_scrub.gpfifo_va = desc->gpfifo_va;
-			svc_scrub.entries = desc->gpfifo_entries;
+			s->userd_ok = true;
+			s->h_object = desc->h_object;
+			s->userd_pa = desc->userd_pa;
+			s->userd_as = desc->userd_as ? desc->userd_as
+						     : SVC_APERTURE_SYS;
+			s->gpfifo_va = desc->gpfifo_va;
+			s->entries = desc->gpfifo_entries;
 			pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n",
 				desc->h_object, desc->gpfifo_va,
 				desc->gpfifo_entries, desc->userd_pa,
-				svc_scrub.userd_as);
+				s->userd_as);
 		}
+scrub_done:
 		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
 			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
 			desc->userd_pa, desc->engine_type, desc->cid,
@@ -1199,7 +1306,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 }
 
 /* P6b wave-1.5 hold state is defined with the forwarding section below. */
-static void svc_fwd_hold_abort(u32 h_object);
+static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object);
 /*
  * P6b wave-2 inline software fallback for a withheld (skipped) candidate;
  * defined with the forwarding section below. Service-thread only.
@@ -1218,7 +1325,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu,
 			   struct svc_channel *ch, u32 entry_idx);
 static void svc_opdef_reset_all(void);
 
-void gsp_sema_service_channel_del(u32 h_object)
+void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
 {
 	unsigned long flags;
 	int i;
@@ -1228,7 +1335,8 @@ void gsp_sema_service_channel_del(u32 h_object)
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == h_object) {
 			svc.chan[i].valid = false;
 			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
 				h_object, svc.chan[i].submissions);
@@ -1238,19 +1346,27 @@ void gsp_sema_service_channel_del(u32 h_object)
 	/*
 	 * ISSUE-0002: when the last channel closes, the next client is a new
 	 * session in a new UVM VA space; its discovery state must not inherit
-	 * this session's map/pins/page-directory latch. Request the flush
-	 * here; it executes at the next rebuild (the only writer of that
-	 * state) just like a fresh boot would start empty. Only the transition
-	 * from "a session existed" to "empty" may request it: redundant
-	 * deletes for handles that never registered would otherwise re-flush
-	 * (and wipe) a session that started after the real one ended.
+	 * this session's map/pins/page-directory latch. The active decision is
+	 * per guest (svc_guests) and the flush request is raised only when NO
+	 * guest has a live channel left: the discovery tables are still shared
+	 * between guests (staged G4), so flushing while another guest is live
+	 * would wipe that guest's pins and stall it. One guest: the last-channel
+	 * close IS the global empty transition, so behaviour is unchanged.
 	 */
 	if (svc_sess_active) {
+		struct svc_guest_sess *sess = svc_guest_get(vgpu, false);
+		bool any = false, own = false;
+
 		for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-			if (svc.chan[i].valid)
-				break;
+			if (!svc.chan[i].valid)
+				continue;
+			any = true;
+			if (svc.chan[i].vgpu == vgpu)
+				own = true;
 		}
-		if (i == SVC_MAX_CHANNELS) {
+		if (sess && !own)
+			sess->active = false;
+		if (!any) {
 			svc_sess_active = 0;
 			svc_sess_flush = 1;
 		}
@@ -1265,9 +1381,10 @@ void gsp_sema_service_channel_del(u32 h_object)
 	 * P6b wave 1.5: a deleted channel's held releases must not wait out
 	 * the timeout; mark the hold aborted so the pump applies them (or
 	 * drops them if the snapshot already lost the channel) on its next
-	 * pass.
+	 * pass. The hold is matched per (vgpu, handle): the other guest's
+	 * identical handle must not abort this guest's hold.
 	 */
-	svc_fwd_hold_abort(h_object);
+	svc_fwd_hold_abort(vgpu, h_object);
 }
 
 /*
@@ -5078,7 +5195,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == h) {
 			/*
 			 * The release targets the slot the push itself named
 			 * (release_pa learned from the release VA); the
@@ -5150,6 +5268,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 			spin_lock_irqsave(&svc.chan_lock, flags);
 			for (i = 0; i < SVC_MAX_CHANNELS; i++) {
 				if (svc.chan[i].valid &&
+				    svc.chan[i].vgpu == vgpu &&
 				    svc.chan[i].h_object == h) {
 					if (payload >
 					    svc.chan[i].pending_rel_pay) {
@@ -5229,7 +5348,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
  * harmless for an absolute release (its guard drops it), but an increment
  * would advance the slot twice.
  */
-static void svc_counter_pending_clear(u32 h)
+static void svc_counter_pending_clear(struct nv_vgpu_device *vgpu, u32 h)
 {
 	unsigned long flags;
 	int i;
@@ -5238,7 +5357,8 @@ static void svc_counter_pending_clear(u32 h)
 		return;
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == h) {
 			svc.chan[i].pending_rel_pay = 0;
 			svc.chan[i].pending_rel_inc = 0;
 			break;
@@ -5272,13 +5392,13 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
 		if (payload > 0x100000) {
 			ch->pending_rel_pay = 0;
 			ch->pending_rel_inc = 0;
-			svc_counter_pending_clear(ch->h_object);
+			svc_counter_pending_clear(ch->vgpu, ch->h_object);
 			return;
 		}
 	} else if (payload <= cur || payload - cur > 0x100000) {
 		ch->pending_rel_pay = 0;
 		ch->pending_rel_inc = 0;
-		svc_counter_pending_clear(ch->h_object);
+		svc_counter_pending_clear(ch->vgpu, ch->h_object);
 		if (cur > ch->counter_last)
 			ch->counter_last = cur;
 		return;
@@ -5289,7 +5409,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
 		return;
 	ch->pending_rel_pay = 0;
 	ch->pending_rel_inc = 0;
-	svc_counter_pending_clear(ch->h_object);
+	svc_counter_pending_clear(ch->vgpu, ch->h_object);
 	if (nv > ch->counter_last)
 		ch->counter_last = nv;
 	svc.releases_applied++;
@@ -5428,6 +5548,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 			ch->counter_pa = ipa & ~0x3ULL;
 			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
 				if (svc.chan[j].valid &&
+				    svc.chan[j].vgpu == vgpu &&
 				    svc.chan[j].h_object == ch->h_object) {
 					svc.chan[j].counter_pa =
 						ch->counter_pa;
@@ -5476,6 +5597,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 		spin_lock_irqsave(&svc.chan_lock, flags);
 		for (j = 0; j < SVC_MAX_CHANNELS; j++) {
 			if (svc.chan[j].valid &&
+			    svc.chan[j].vgpu == vgpu &&
 			    svc.chan[j].h_object != ch->h_object &&
 			    svc.chan[j].counter_pa == pa) {
 				collision = true;
@@ -5501,6 +5623,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 			ch->counter_pa = pa;
 			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
 				if (svc.chan[j].valid &&
+				    svc.chan[j].vgpu == vgpu &&
 				    svc.chan[j].h_object == ch->h_object) {
 					svc.chan[j].counter_pa = pa;
 					break;
@@ -5560,7 +5683,8 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == h) {
 			if (svc.chan[i].release_pa != (pa & ~0x3ULL)) {
 				svc.chan[i].release_pa = pa & ~0x3ULL;
 				svc.chan[i].relearn_n++;
@@ -7694,7 +7818,8 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu)
 		u32 slot, k;
 		bool have = false;
 
-		if (!svc.chan[i].valid || !svc.chan[i].engine_type)
+		if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu ||
+		    !svc.chan[i].engine_type)
 			continue;
 		ce = true;
 		if (svc.chan[i].ce_index >= 32)
@@ -8975,7 +9100,8 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
 		int pap = 0;
 		u32 r = c->gr_index;
 
-		if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS)
+		if (!c->valid || c->vgpu != vgpu || c->engine_type ||
+		    r >= SVC_GR_CONTENT_RANKS)
 			continue;
 		if (test_bit(r, &svc_gr_ring_done))
 			continue;
@@ -9316,7 +9442,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	 */
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (k = 0; k < SVC_MAX_CHANNELS; k++)
-		if (svc.chan[k].valid && svc.chan[k].engine_type &&
+		if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu &&
+		    svc.chan[k].engine_type &&
 		    svc.chan[k].ce_index + 1 > max_ce)
 			max_ce = svc.chan[k].ce_index + 1;
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
@@ -10792,7 +10919,7 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind,
 	}
 	if (kind != SVC_APERTURE_SYS)
 		return -EINVAL;
-	ret = gsp_phys_gpa_to_hpa(page, hpa);
+	ret = gsp_phys_gpa_to_hpa_vgpu(vgpu, page, hpa);
 	if (ret)
 		return ret;
 	*out_off = page_off;
@@ -10961,9 +11088,10 @@ struct svc_fwd_hold {
 
 static struct svc_fwd_hold svc_fwd_hold;
 
-static void svc_fwd_hold_abort(u32 h_object)
+static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object)
 {
 	if (READ_ONCE(svc_fwd_hold.active) &&
+	    svc_fwd_hold.vgpu == vgpu &&
 	    svc_fwd_hold.ch_handle == h_object)
 		WRITE_ONCE(svc_fwd_hold.aborted, true);
 }
@@ -11334,18 +11462,22 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu,
 	}
 }
 
-/* A slot whose channel is gone from this pass's snapshot: drop it. */
-static void svc_opdef_drop_gone(const struct svc_channel *local, int n)
+/* A slot of THIS pump's guest whose channel is gone from the pass snapshot:
+ * drop it. Slots of other guests are never touched (their channels are not in
+ * this snapshot by construction). */
+static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu,
+				const struct svc_channel *local, int n)
 {
 	int i, k;
 
 	for (i = 0; i < SVC_OPDEF_SLOTS; i++) {
 		struct svc_opdef_ent *e = &svc_opdef[i];
 
-		if (!e->active)
+		if (!e->active || e->vgpu != vgpu)
 			continue;
 		for (k = 0; k < n; k++)
-			if (local[k].h_object == e->h_object)
+			if (local[k].vgpu == e->vgpu &&
+			    local[k].h_object == e->h_object)
 				break;
 		if (k == n)
 			svc_opdef_clear(e);
@@ -11787,6 +11919,31 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * Is this mdev still on the driver list with an open VFIO device? Decides
+ * whether a hold that names it can still be resolved through it: a removed
+ * device must never be dereferenced (its struct can be freed before the next
+ * pump pass would see the hold). Pointer comparisons only - safe to call
+ * with a possibly-stale pointer.
+ */
+static bool svc_vgpu_open(const struct nv_vgpu_device *vgpu)
+{
+	struct nv_vgpu_device *d;
+	bool open = false;
+
+	if (!vgpu)
+		return false;
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+		if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) {
+			open = true;
+			break;
+		}
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+	return open;
+}
+
+/*
  * Channel vanished while a hold was active. Any resolved hold (completed,
  * aborted, failed or expired) applies the recorded releases through a shim
  * channel - holds exist only for CE channels and the vanished channel object
@@ -11806,6 +11963,18 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
 	bool done = smp_load_acquire(&svc_fwd_req.done);
 	bool aborted = READ_ONCE(svc_fwd_hold.aborted);
 
+	if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) {
+		/*
+		 * The owning mdev was removed mid-hold (the mdev cleanup path
+		 * already released its session state): its guest can never
+		 * observe the releases and dereferencing the device would be a
+		 * use-after-free. Drop the hold; the request slot frees for the
+		 * next guest.
+		 */
+		svc.fwd_hold_canceled++;
+		svc_fwd_hold_clear();
+		return;
+	}
 	if (!done && !aborted && !svc_fwd_hold_expired())
 		return;
 	/*
@@ -11858,6 +12027,15 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
 
 	if (!svc_fwd_hold.active)
 		return;
+	/*
+	 * A hold whose owning mdev is already gone must never be dereferenced:
+	 * drop it (the guest cannot observe the releases anyway).
+	 */
+	if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) {
+		svc.fwd_hold_canceled++;
+		svc_fwd_hold_clear();
+		return;
+	}
 	done = smp_load_acquire(&svc_fwd_req.done);
 	/*
 	 * P6b wave 2: the backend is going away; a skipped candidate whose
@@ -13471,7 +13649,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
-		if (svc.chan[i].valid)
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu)
 			local[n++] = svc.chan[i];
 	}
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
@@ -13507,7 +13685,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	 * pass's snapshot belongs to a vanished channel; its waiter is gone,
 	 * so drop the set instead of resolving it later.
 	 */
-	svc_opdef_drop_gone(local, n);
+	svc_opdef_drop_gone(vgpu, local, n);
 
 	for (i = 0; i < n; i++) {
 		struct svc_channel *ch = &local[i];
@@ -15020,7 +15198,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 		spin_lock_irqsave(&svc.chan_lock, flags);
 		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
-			if (svc.chan[k].valid &&
+			if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu &&
 			    svc.chan[k].h_object == ch->h_object) {
 				svc.chan[k].userd_pa = ch->userd_pa;
 				svc.chan[k].userd_as = ch->userd_as;
@@ -15334,7 +15512,7 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++)
-		if (svc.chan[i].valid)
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu)
 			local[n++] = svc.chan[i];
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
 
@@ -15725,9 +15903,18 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa)
  * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish
  * payload the guest's scrub wait polls; hsem_pa is the channel's host
  * semaphore (channelWaitForFreeEntry polls it). Plain state update.
+ *
+ * Dual-guest: the words are written through a specific guest's address space
+ * (see svc_scrub_pass), so the registration binds to the FIRST open mdev -
+ * the same single-guest convention the scanner tooling follows (it tracks
+ * one client). Refused with no open mdev rather than parked globally.
  */
 int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
 {
+	struct nv_vgpu_device *vgpu = NULL;
+	struct nv_vgpu_device *d;
+	struct svc_scrub *s;
+
 	if (!svc.task)
 		return -ENODEV;
 	/* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */
@@ -15736,12 +15923,28 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
 	if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX)
 		return -EINVAL;
 
-	svc_scrub.fin_pa = fin_pa;
-	svc_scrub.hsem_pa = hsem_pa;
-	svc_scrub.pa_ok = true;
-	svc_scrub.pa_gen++;
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+		if (atomic_read(&d->vfio_dev_open_count) > 0) {
+			vgpu = d;
+			break;
+		}
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+	if (!vgpu) {
+		pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n");
+		return -ENODEV;
+	}
+	s = svc_scrub_get(vgpu, true);
+	if (!s)
+		return -ENOSPC;
+
+	s->fin_pa = fin_pa;
+	s->hsem_pa = hsem_pa;
+	s->pa_ok = true;
+	s->pa_gen++;
 	pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n",
-		fin_pa, hsem_pa, svc_scrub.pa_gen);
+		fin_pa, hsem_pa, s->pa_gen);
 	return 0;
 }
 
@@ -15769,11 +15972,11 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
  */
 static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
 {
-	struct svc_scrub *s = &svc_scrub;
+	struct svc_scrub *s = svc_scrub_get(vgpu, false);
 	u32 put = 0;
 	u64 delta;
 
-	if (!s->userd_ok || !s->pa_ok || !s->entries)
+	if (!s || !s->userd_ok || !s->pa_ok || !s->entries)
 		return;
 	if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as,
 			&put, sizeof(put))) {
@@ -15830,7 +16033,9 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu)
 
 static int svc_thread(void *arg)
 {
+	struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS];
 	struct nv_vgpu_device *vgpu;
+	int n_vgpus, g;
 
 	pr_info("sema-svc: thread started\n");
 
@@ -15850,130 +16055,174 @@ static int svc_thread(void *arg)
 			complete_all(&svc_sess_flush_done);
 		}
 
-		vgpu = NULL;
+		/*
+		 * Service EVERY open mdev each pass, in list order. With one
+		 * mdev this is exactly the historical single-vgpu pass; with
+		 * two, both guests get a pump turn per iteration (hardware
+		 * forwarding stays serialized by the single-inflight slot; see
+		 * the forwarding section). The list walk is the historical
+		 * bounded scan under nv_vgpu_driver.lock.
+		 */
+		n_vgpus = 0;
 		mutex_lock(&nv_vgpu_driver.lock);
 		{
 			struct nv_vgpu_device *d;
 
 			list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
-				if (atomic_read(&d->vfio_dev_open_count) > 0) {
-					vgpu = d;
-					break;
-				}
+				if (atomic_read(&d->vfio_dev_open_count) > 0 &&
+				    n_vgpus < SVC_MAX_GUESTS)
+					vgpus[n_vgpus++] = d;
 			}
 		}
 		mutex_unlock(&nv_vgpu_driver.lock);
 
-		if (!vgpu) {
+		if (!n_vgpus) {
 			msleep(500);
 			continue;
 		}
 
-		svc_handle_probe(vgpu);
-		svc_handle_walk_probe(vgpu);
+		/* Host-injection queue drain: once per pass, not per guest. */
 		svc_handle_map_add();
-		svc_handle_chan_dump(vgpu);
-		svc_scrub_pass(vgpu);
 
-		if (svc_sess_flush || !svc.direct_base ||
-		    time_after(jiffies,
-			       (unsigned long)(svc.last_build + 10 * HZ)))
-			svc_rebuild(vgpu);
+		for (g = 0; g < n_vgpus; g++) {
+			struct svc_guest_sess *sess;
+			u64 done_before;
 
-		/*
-		 * Channel-driven execution: notice submissions as they are made
-		 * and run the pushed work immediately, so the guest's UVM
-		 * progress waits complete in milliseconds rather than in the
-		 * time it takes the fallback scan to find the push. A doorbell
-		 * token means a channel was just scheduled; run everything now.
-		 */
-		{
-			u64 done_before = svc.gpfifo_entries_done;
+			vgpu = vgpus[g];
+			svc_handle_probe(vgpu);
+			svc_handle_walk_probe(vgpu);
+			svc_handle_chan_dump(vgpu);
+			svc_scrub_pass(vgpu);
+
+			/*
+			 * Per-guest rebuild freshness: the discovery map is
+			 * still shared between guests (staged G4), but each
+			 * guest's RAM must be scanned on its own clock, or the
+			 * first guest's 10 s tick would starve the second
+			 * guest of rebuilds. The freshness stamp is the one
+			 * svc_rebuild() publishes (svc.last_build).
+			 */
+			sess = svc_guest_get(vgpu, true);
+			if (svc_sess_flush || !svc.direct_base ||
+			    !sess ||
+			    time_after(jiffies,
+				       (unsigned long)(sess->last_build +
+						       10 * HZ))) {
+				svc_rebuild(vgpu);
+				if (sess)
+					sess->last_build = svc.last_build;
+			}
 
+			/*
+			 * Channel-driven execution: notice submissions as they
+			 * are made and run the pushed work immediately, so the
+			 * guest's UVM progress waits complete in milliseconds
+			 * rather than in the time it takes the fallback scan
+			 * to find the push. A doorbell token means a channel
+			 * was just scheduled; run everything now.
+			 */
+			done_before = svc.gpfifo_entries_done;
 			svc.doorbell_pending = false;
 			svc_pump_channels(vgpu);
-			pumped = (svc.gpfifo_entries_done != done_before);
-		}
+			if (svc.gpfifo_entries_done != done_before)
+				pumped = true;
 
-		/*
-		 * Host CUDA bridge: marshal one queued launch into a daemon
-		 * job and resolve completions. Inert until a launch is seen.
-		 */
-		gsp_cuda_bridge_service(vgpu);
+			/*
+			 * Host CUDA bridge: marshal one queued launch into a
+			 * daemon job and resolve completions. Inert until a
+			 * launch is seen.
+			 */
+			gsp_cuda_bridge_service(vgpu);
 
-		/*
-		 * Releases whose target VA was not translatable when the scan
-		 * saw them: retry every pass, so a pin or a learned pair applies
-		 * them immediately instead of on the next full sweep.
-		 */
-		svc_apply_pending(vgpu);
+			/*
+			 * Releases whose target VA was not translatable when
+			 * the scan saw them: retry every pass, so a pin or a
+			 * learned pair applies them immediately instead of on
+			 * the next full sweep.
+			 */
+			svc_apply_pending(vgpu);
 
-		/*
-		 * A fresh VA->PA injection changes which releases are
-		 * applicable: run the fast sliced sweep while the forced window
-		 * is open (the pump's stall path also uses it).
-		 */
-		if (time_before(jiffies, (unsigned long)svc.fast_force_until))
-			svc_apply_sweep_slice(vgpu);
+			/*
+			 * A fresh VA->PA injection changes which releases are
+			 * applicable: run the fast sliced sweep while the
+			 * forced window is open (the pump's stall path also
+			 * uses it).
+			 */
+			if (time_before(jiffies,
+					(unsigned long)svc.fast_force_until))
+				svc_apply_sweep_slice(vgpu);
+		}
 
 		/*
 		 * While the pump is making progress the client is streaming
 		 * pushes and waiting on their releases: loop tightly and skip
-		 * the discovery/sweep passes, which otherwise dominate the pass
-		 * period (the pump cadence is what the client's context init
-		 * deadline sees). Scans resume the first pass without progress.
+		 * the discovery/sweep passes, which otherwise dominate the
+		 * pass period (the pump cadence is what the client's context
+		 * init deadline sees). Scans resume the first pass without
+		 * progress. (With two guests, any guest's progress keeps the
+		 * loop tight; the next iteration pumps both.)
 		 */
 		if (pumped) {
 			msleep(1);
 			continue;
 		}
 
-		/*
-		 * Push-buffer base discovery: a chunked walk of guest RAM
-		 * finds the pages whose first push names a channel, so a
-		 * channel that appears with work already waiting is pinned and
-		 * pumped without waiting for the rolling sweep.
-		 */
-		svc_pb_discover(vgpu);
+		for (g = 0; g < n_vgpus; g++) {
+			vgpu = vgpus[g];
 
-		/*
-		 * Completion hint: re-post the events for a few seconds after a
-		 * completion, so a client that starts waiting after the
-		 * edge-triggered wake-up still observes it.
-		 */
-		if (time_before(jiffies, (unsigned long)svc.post_hint_until) &&
-		    time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) {
-			int rc;
+			/*
+			 * Push-buffer base discovery: a chunked walk of guest
+			 * RAM finds the pages whose first push names a
+			 * channel, so a channel that appears with work
+			 * already waiting is pinned and pumped without
+			 * waiting for the rolling sweep.
+			 */
+			svc_pb_discover(vgpu);
 
-			svc.last_hint = jiffies;
-			rc = gsp_rm_post_all_events(vgpu);
-			if (gsp_sema_dbg)
-				pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
-						    rc);
-		}
+			/*
+			 * Completion hint: re-post the events for a few
+			 * seconds after a completion, so a client that starts
+			 * waiting after the edge-triggered wake-up still
+			 * observes it.
+			 */
+			if (time_before(jiffies,
+					(unsigned long)svc.post_hint_until) &&
+			    time_after(jiffies,
+				       (unsigned long)svc.last_hint + HZ / 2)) {
+				int rc;
+
+				svc.last_hint = jiffies;
+				rc = gsp_rm_post_all_events(vgpu);
+				if (gsp_sema_dbg)
+					pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
+							    rc);
+			}
 
-		/*
-		 * Fast path for the pushes themselves: every mapped page is a
-		 * candidate pushbuffer page, and there are far fewer of those
-		 * than there are pages of guest RAM.
-		 */
-		if (svc.direct_base && svc.map_n)
-			svc_apply_mapped(vgpu);
+			/*
+			 * Fast path for the pushes themselves: every mapped
+			 * page is a candidate pushbuffer page, and there are
+			 * far fewer of those than there are pages of guest
+			 * RAM.
+			 */
+			if (svc.direct_base && svc.map_n)
+				svc_apply_mapped(vgpu);
 
-		/*
-		 * Rolling RAM scan: the pushbuffers of some clients are not
-		 * covered by the VA map, so sweep guest RAM in bounded windows
-		 * in parallel with the targeted pass. Work per pass is fixed,
-		 * which keeps the emulation responsive while a full sweep
-		 * completes every few seconds.
-		 */
-		svc_apply_pushes(vgpu);
+			/*
+			 * Rolling RAM scan: the pushbuffers of some clients
+			 * are not covered by the VA map, so sweep guest RAM in
+			 * bounded windows in parallel with the targeted pass.
+			 * Work per pass is fixed, which keeps the emulation
+			 * responsive while a full sweep completes every few
+			 * seconds.
+			 */
+			svc_apply_pushes(vgpu);
 
-		/*
-		 * Deferred readability probes for faked CE pushes (observation
-		 * only; see svc_retry_poll).
-		 */
-		svc_retry_poll(vgpu);
+			/*
+			 * Deferred readability probes for faked CE pushes
+			 * (observation only; see svc_retry_poll).
+			 */
+			svc_retry_poll(vgpu);
+		}
 
 		msleep(10);
 	}
@@ -16096,6 +16345,97 @@ void gsp_sema_service_stop(void)
 	svc_vas_free_all();
 }
 
+/*
+ * Release every piece of service state owned by one mdev (dual-guest goal
+ * 1.0). Called from the mdev cleanup path after that mdev's emulation thread
+ * stopped and before its struct can be freed:
+ *  - quiesce forwarding so no in-flight work item can still be reading the
+ *    request's vgpu, and abort a release hold that names it;
+ *  - drop any hold that survives (a request that never queued leaves no work
+ *    item to clear it) BEFORE the device can go away - resolution would
+ *    dereference it;
+ *  - drop the guest's channels; request the global session flush only when no
+ *    guest has live channels left (the discovery tables are still shared;
+ *    staged G4);
+ *  - drop the guest's held-operand release slots (its waiters are gone);
+ *  - free the per-guest session and scrubber records so the vgpu pointer can
+ *    never match a newly allocated device.
+ */
+void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu)
+{
+	unsigned long flags;
+	int i;
+
+	if (!vgpu)
+		return;
+
+	/*
+	 * The forward work item runs on its own kthread and may be using the
+	 * request's vgpu; quiesce waits for it (bounded by the submit's
+	 * 200 ms poll cap) and aborts an active hold. Resume afterwards so the
+	 * other guests keep forwarding.
+	 */
+	gsp_sema_service_work_quiesce();
+	if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu)
+		WRITE_ONCE(svc_fwd_hold.aborted, true);
+	gsp_sema_service_work_resume();
+
+	if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) {
+		/*
+		 * Pure state clear (never a resolution through the vanishing
+		 * device): the releases belong to a guest that is gone.
+		 */
+		svc.fwd_hold_canceled++;
+		svc_fwd_hold_clear();
+	}
+
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		struct svc_channel *ch = &svc.chan[i];
+
+		if (!ch->valid || ch->vgpu != vgpu)
+			continue;
+		pr_info("sema-svc: channel 0x%x dropped (mdev cleanup; submissions=%llu)\n",
+			ch->h_object, ch->submissions);
+		memset(ch, 0, sizeof(*ch));
+	}
+	if (svc_sess_active) {
+		struct svc_guest_sess *sess = svc_guest_get(vgpu, false);
+		bool any = false;
+
+		for (i = 0; i < SVC_MAX_CHANNELS; i++)
+			if (svc.chan[i].valid) {
+				any = true;
+				break;
+			}
+		if (sess)
+			sess->active = false;
+		if (!any) {
+			svc_sess_active = 0;
+			svc_sess_flush = 1;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+	/* Held-operand release slots of this guest cannot be delivered. */
+	for (i = 0; i < SVC_OPDEF_SLOTS; i++)
+		if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu)
+			svc_opdef_clear(&svc_opdef[i]);
+
+	/* Free the per-guest session and scrubber records. */
+	spin_lock_irqsave(&svc_guest_lock, flags);
+	for (i = 0; i < SVC_MAX_GUESTS; i++)
+		if (svc_guests[i].vgpu == vgpu)
+			memset(&svc_guests[i], 0, sizeof(svc_guests[i]));
+	spin_unlock_irqrestore(&svc_guest_lock, flags);
+	for (i = 0; i < SVC_MAX_GUESTS; i++)
+		if (svc_scrubs[i].vgpu == vgpu)
+			memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i]));
+
+	if (svc_sess_flush && svc.task)
+		wake_up_process(svc.task);
+}
+
 /* ------------------------------------------------------------------ */
 /* host CUDA bridge exports (gsp_cuda_bridge.c)                        */
 /* ------------------------------------------------------------------ */