vgsp: hold every unreadable CE entry for the grace; delete the silent pre-rescue fake

An unreadable CE entry whose channel already had a decoded release VA was faked immediately and consumed with no log line, before any rescue attempt. A later pin-batch rewind then executed the skipped copy against a staging buffer the client had since refilled - the deterministic vadd input shift seen in runs 94/122/151/173b/215b, each with a silent pre-rescue fake.

Now every unreadable CE entry with an expected payload is held for the bounded grace and the rescue is retried on each pass, so a translation that is still coming wins and the work runs before the release. Only an entry whose release was already delivered (the channel moved past entry_want) is consumed at once; the post-rescue fake remains the bounded, visible escape. The CE grace default moves 15000 -> 25000 ms (a translation arrived ~17 s after first examination in 215b).

Under live campaign 229-236 (module 97eba5b7; green so far).

2 files changed, 42 insertions, 36 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index be1d88a..f897ce0 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -6670,7 +6670,7 @@ u64 gsp_sema_copy_dst2;
  * write the next chunk's data over a finished one), so the hold is the only
  * chance the work has.
  */
-unsigned gsp_sema_ce_grace_ms = 15000;
+unsigned gsp_sema_ce_grace_ms = 25000;
 
 /*
  * Bounded work-before-release hold (ms) for an unreadable CE entry. The
@@ -9860,22 +9860,49 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 						 * lets the client read unstaged
 						 * memory (observed: cuMemcpyDtoH
 						 * returning zeros after its HtoD push
-						 * was skipped unreadable). With a
-						 * decoded release VA (last_rel_va)
-						 * the exact slot is already known:
-						 * deliver or queue it and consume the
-						 * entry at once - never hold the
-						 * channel for a translation that may
-						 * never arrive. Only a channel with no
-						 * exact slot gets one bounded grace so
-						 * a late translation can teach it the
-						 * slot; after that it is consumed
-						 * either way (the release content
-						 * sweep is the safety net).
+						 * was skipped unreadable). A known
+						 * release VA (last_rel_va) is NOT a
+						 * license to fake immediately: the
+						 * entry was unreadable only because
+						 * its page translation had not landed
+						 * yet, and a later pin-batch read of
+						 * the same queued entry would then
+						 * execute its copy against a staging
+						 * buffer the client has since
+						 * refilled (observed: the deterministic
+						 * vadd input shift, runs 94/122/151/
+						 * 173b/215b, each with a silent
+						 * pre-rescue fake). Hold every
+						 * unreadable CE entry for the bounded
+						 * grace; the rescue below is retried
+						 * on each pass, so a translation that
+						 * is still coming wins and the work
+						 * runs before the release. Only an
+						 * entry whose release was already
+						 * delivered (the channel moved past
+						 * entry_want) is consumed at once -
+						 * its wait no longer exists - and the
+						 * fake stays the bounded escape after
+						 * the grace plus a rescue attempt.
 						 */
 						if (ch->engine_type &&
-						    !ch->last_rel_va &&
+						    ch->expected_sem_payload &&
 						    !gsp_sema_fake_rel) {
+							if (ch->entry_want &&
+							    ch->expected_sem_payload >
+							    ch->entry_want) {
+								ch->defer_since = 0;
+								ch->rescue_fail = 0;
+								ch->unread_va = 0;
+								ch->unread_until = 0;
+								svc_replay_note(ch, idx, put);
+								ch->last_get =
+									(idx + 1) %
+									ch->gpfifo_entries;
+								svc.gpfifo_skipped++;
+								kfree(pb);
+								continue;
+							}
 							if (!ch->defer_since)
 								ch->defer_since =
 									jiffies;
@@ -9888,27 +9915,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								break;
 							}
 						}
-						if (svc_fake_release(vgpu, ch)) {
-							/*
-							 * Watch (observe only):
-							 * did this push's page
-							 * ever become readable?
-							 * See svc_retry_poll().
-							 */
-							svc_retry_enqueue(va, len,
-									  ch->expected_sem_payload);
-							ch->push_hold_until = 0;
-							ch->defer_since = 0;
-							ch->unread_va = 0;
-							ch->unread_until = 0;
-							svc_replay_note(ch, idx, put);
-							ch->last_get = (idx + 1) %
-								       ch->gpfifo_entries;
-							svc.gpfifo_skipped++;
-							stalled = true;
-							kfree(pb);
-							continue;
-						}
 						if (svc.false_trans && svc.false_trans < 8)
 							pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n",
 								va, ch->h_object, len);
diff --git a/kernel/main.c b/kernel/main.c
index 1732ad8..7028bdd 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2659,7 +2659,7 @@ MODULE_PARM_DESC(sema_copy_dst2, "Second destination window for sema_copy_dst (d
  * a service pass.
  */
 module_param_named(sema_ce_grace_ms, gsp_sema_ce_grace_ms, uint, 0644);
-MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 15000)");
+MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 25000)");
 
 /*
  * Bounded work-before-release hold (ms) for an unreadable CE entry; the