vgsp: PRIV VA-only booking bit + verify the exact slot before consuming unreadable CE entries

- fwclient mask bit 15 (0x8000): book the PRIV_ACCESS_MAP VA (the FWCLIENT
  entry form is init-only; the golden9 form carries the VA at VA_GOLDEN9_PRIV).
  Accepted on V6 (status 0), completing the maximal promoted context.
- gsp_sema_service: svc_entry_want_reached() verifies the exact tracking slot
  against entry_want before the three unreadable-CE consume shortcuts treat the
  entry as delivered; when the slot lags, re-seed the expectation and let
  svc_fake_release author or queue the exact slot (UVM registration stall pair
  with the pool-VA verified-resolver queue).

2 files changed, 104 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 3383b35..c049c44 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -701,6 +701,12 @@ bool gsp_phys_boot_requested(void)
  */
 #define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY	(1u << 13)
 #define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY	(1u << 14)
+/*
+ * Mode-25 bisection bit 15: book the PRIV_ACCESS_MAP VA (the FWCLIENT entry
+ * form is init-only with no VA, while the golden9 form is combined PA+VA at
+ * VA_GOLDEN9_PRIV). One VA-only booking entry, like the accepted globals.
+ */
+#define GSP_PHYS_RMCALL_FWCLIENT_MASK_PRIV_VA_ONLY	(1u << 15)
 #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
 	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC)
 
@@ -11079,6 +11085,27 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
 				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, status,
 				rpc_result, boot_stage.rmcall_channel);
 		}
+
+		/*
+		 * Bit 15: the PRIV_ACCESS_MAP VA booking the FWCLIENT entry
+		 * form omits (it is init-only with no VA). One VA-only entry.
+		 */
+		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_PRIV_VA_ONLY) {
+			gsp_phys_rmcall_promote_entry_fill(p, 0, 10, 0,
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, 0, 0, 0, 0);
+			p->entry_count = cpu_to_le32(1);
+			rpc_result = 0;
+			status = 0;
+			ret = gsp_phys_rmcall_control(bar0,
+					boot_stage.rmcall_subdevice,
+					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+					p, sizeof(*p), 24, NULL, 0,
+					&rpc_result, &status,
+					"GPU_PROMOTE_CTX (PRIV VA-only)");
+			pr_info("vgsp-boot: rm-call: PROMOTE_CTX PRIV VA-only: bufferId=10 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
+				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, status,
+				rpc_result, boot_stage.rmcall_channel);
+		}
 	} else {
 		pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
 		       mode);
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2c6836f..83ca6bd 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -7957,6 +7957,38 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch
 	return true;
 }
 
+/*
+ * True only when the exact tracking-semaphore slot this entry wants really
+ * reached entry_want. expected_sem_payload can move past entry_want through
+ * other entries' releases/fakes while this slot still lags; the consume
+ * shortcuts treat that as "already delivered" and strand the client's wait.
+ * A missing translation is "not proven reached": the caller re-seeds and
+ * lets svc_fake_release author or queue the exact slot.
+ */
+static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu,
+				   const struct svc_channel *ch)
+{
+	u64 spa = 0;
+	int sap = SVC_APERTURE_SYS;
+	u32 cur = 0;
+
+	if (!ch->entry_want)
+		return true;
+	if (!ch->last_rel_va)
+		return false;
+	if (svc_pool_sem_va(ch->last_rel_va)) {
+		if (svc_pool_pa_resolve(vgpu, ch->h_object, ch->last_rel_va,
+					&spa, &sap, NULL) &&
+		    svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap))
+			return false;
+	} else if (svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap)) {
+		return false;
+	}
+	if (svc_read_ap(vgpu, spa, svc_ap_of(sap), &cur, sizeof(cur)))
+		return false;
+	return cur >= ch->entry_want;
+}
+
 /* Mark a consumed GPFIFO slot for the current put epoch. */
 static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
 {
@@ -10617,6 +10649,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 									if (ch->entry_want &&
 									    ch->expected_sem_payload >
 									    ch->entry_want) {
+										/*
+										 * The expectation can move past this entry without this
+										 * slot ever being written (another entry's decoded
+										 * release or fake advanced it). Verify the exact slot;
+										 * re-seed and author it when the slot still lags, so
+										 * consuming the entry never strands the client's wait.
+										 */
+										if (!svc_entry_want_reached(vgpu, ch)) {
+											pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
+												ch->h_object, ch->ce_index, ch->entry_want,
+												ch->expected_sem_payload);
+											ch->expected_sem_payload = ch->entry_want;
+											if (!svc_fake_release(vgpu, ch))
+												svc.gpfifo_starved++;
+										}
 										ch->defer_since = 0;
 										ch->rescue_fail = 0;
 										ch->unread_va = 0;
@@ -10764,6 +10811,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							if (ch->entry_want &&
 							    ch->expected_sem_payload >
 							    ch->entry_want) {
+								/*
+								 * The expectation can move past this entry without this
+								 * slot ever being written (another entry's decoded
+								 * release or fake advanced it). Verify the exact slot;
+								 * re-seed and author it when the slot still lags, so
+								 * consuming the entry never strands the client's wait.
+								 */
+								if (!svc_entry_want_reached(vgpu, ch)) {
+									pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
+										ch->h_object, ch->ce_index, ch->entry_want,
+										ch->expected_sem_payload);
+									ch->expected_sem_payload = ch->entry_want;
+									if (!svc_fake_release(vgpu, ch))
+										svc.gpfifo_starved++;
+								}
 								ch->defer_since = 0;
 								ch->rescue_fail = 0;
 								ch->unread_va = 0;
@@ -10864,6 +10926,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							    ch->expected_sem_payload >
 							    ch->entry_want) {
 								/*
+								 * The expectation can move past this entry without this
+								 * slot ever being written (another entry's decoded
+								 * release or fake advanced it). Verify the exact slot;
+								 * re-seed and author it when the slot still lags, so
+								 * consuming the entry never strands the client's wait.
+								 */
+								if (!svc_entry_want_reached(vgpu, ch)) {
+									pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
+										ch->h_object, ch->ce_index, ch->entry_want,
+										ch->expected_sem_payload);
+									ch->expected_sem_payload = ch->entry_want;
+									if (!svc_fake_release(vgpu, ch))
+										svc.gpfifo_starved++;
+								}
+								/*
 								 * Already delivered
 								 * (the content sweep
 								 * applied this entry's