From 83ded3466f0f6367387be6c74dbeea9fe0e661f8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Wed, 30 Sep 2026 20:47:06 -0400 Subject: sema-svc: never re-process a GPFIFO slot already consumed in the put epoch A pin-batch rewind replays recently consumed entries. An old entry whose rescue cannot validate any candidate then sits in the CE grace path - and because the client's put has advanced since, the replayed entry no longer matches any "new" expectation: it waits the full 8 s grace, blocking every later entry behind it on that channel. Observed live: the first push of rank 0 re-processed and skipped twice, 38 s apart, while eight later pushes queued behind it. The pump now marks each consumed slot in a per-channel bitmap valid for the current put epoch (cleared when put wraps, so a recycled slot in the next epoch is processed normally) and consumes a replayed slot immediately. Rewinds still refresh readable entries; they simply no longer resurrect entries that were already handled. Evidence: real-release watcher cycle, replay-after-put-advance no longer produces 'skipping stale entry ... after 8000 ms grace' for old slots. --- kernel/gsp_sema_service.c | 60 +++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 26 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 7e15f44..dc8e085 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -157,13 +157,18 @@ struct svc_channel { * immediately instead of waiting out the grace * window (observed: one 8 s grace per push, * serializing the runtime init) */ - u32 replay[32]; /* packed {put:16,slot:16} of recently consumed - * entries. A pin-batch rewind replays entries; - * re-processing one costs a full grace window - * (observed: the same VA skipped twice, 38 s - * apart), so a replayed slot whose producing - * put has not advanced is consumed at once. */ - u32 replay_next; + DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); + /* GPFIFO slots already consumed in the current + * put epoch (the client's put value since it + * last wrapped). A pin-batch rewind replays + * consumed entries; re-processing an old one + * runs a futile rescue and holds the pump for + * a full grace window per replay (observed: + * the first push re-skipped twice, 38 s + * apart, gating every later entry). Cleared + * when put wraps, so a recycled slot in the + * next epoch is processed normally. */ + u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ @@ -4982,28 +4987,20 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch return true; } -#define SVC_REPLAY_SLOTS 32 - -/* Remember a consumed GPFIFO slot and the put it was consumed at. */ +/* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { - ch->replay[ch->replay_next % SVC_REPLAY_SLOTS] = - ((put & 0xffff) << 16) | (idx & 0xffff); - ch->replay_next++; + (void)put; + if (idx < SVC_GPFIFO_MAX_BITS) + set_bit(idx, ch->slot_done_epoch); } -/* Was this slot consumed at this put? Then the entry is a rewind replay. */ +/* Was this slot already consumed in the current put epoch? */ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) { - u32 rec = ((put & 0xffff) << 16) | (idx & 0xffff); - u32 n = ch->replay_next < SVC_REPLAY_SLOTS ? - ch->replay_next : SVC_REPLAY_SLOTS; - u32 i; - - for (i = 0; i < n; i++) - if (ch->replay[i] == rec) - return true; - return false; + (void)put; + return idx < SVC_GPFIFO_MAX_BITS && + test_bit(idx, ch->slot_done_epoch); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -5095,6 +5092,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; + /* + * A wrapped put means every slot may hold a new entry: clear + * the per-epoch consumed marks so recycled slots are + * processed (a rewind replay of the previous epoch must not + * suppress them). + */ + if (put < ch->put_prev) + bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + ch->put_prev = put; + get_before = ch->last_get; applied_before = svc.releases_applied; @@ -5676,9 +5683,10 @@ if (gsp_sema_fake_rel && svc.chan[k].rescue_last = ch->rescue_last; svc.chan[k].rescue_fail = ch->rescue_fail; svc.chan[k].defer_since = ch->defer_since; - svc.chan[k].replay_next = ch->replay_next; - memcpy(svc.chan[k].replay, ch->replay, - sizeof(ch->replay)); + svc.chan[k].put_prev = ch->put_prev; + bitmap_copy(svc.chan[k].slot_done_epoch, + ch->slot_done_epoch, + SVC_GPFIFO_MAX_BITS); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives -- cgit v1.2.3