diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 60 |
1 files changed, 34 insertions, 26 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 7e15f44..dc8e085 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -157,13 +157,18 @@ struct svc_channel { * immediately instead of waiting out the grace * window (observed: one 8 s grace per push, * serializing the runtime init) */ - u32 replay[32]; /* packed {put:16,slot:16} of recently consumed - * entries. A pin-batch rewind replays entries; - * re-processing one costs a full grace window - * (observed: the same VA skipped twice, 38 s - * apart), so a replayed slot whose producing - * put has not advanced is consumed at once. */ - u32 replay_next; + DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); + /* GPFIFO slots already consumed in the current + * put epoch (the client's put value since it + * last wrapped). A pin-batch rewind replays + * consumed entries; re-processing an old one + * runs a futile rescue and holds the pump for + * a full grace window per replay (observed: + * the first push re-skipped twice, 38 s + * apart, gating every later entry). Cleared + * when put wraps, so a recycled slot in the + * next epoch is processed normally. */ + u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ @@ -4982,28 +4987,20 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch return true; } -#define SVC_REPLAY_SLOTS 32 - -/* Remember a consumed GPFIFO slot and the put it was consumed at. */ +/* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { - ch->replay[ch->replay_next % SVC_REPLAY_SLOTS] = - ((put & 0xffff) << 16) | (idx & 0xffff); - ch->replay_next++; + (void)put; + if (idx < SVC_GPFIFO_MAX_BITS) + set_bit(idx, ch->slot_done_epoch); } -/* Was this slot consumed at this put? Then the entry is a rewind replay. */ +/* Was this slot already consumed in the current put epoch? */ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) { - u32 rec = ((put & 0xffff) << 16) | (idx & 0xffff); - u32 n = ch->replay_next < SVC_REPLAY_SLOTS ? - ch->replay_next : SVC_REPLAY_SLOTS; - u32 i; - - for (i = 0; i < n; i++) - if (ch->replay[i] == rec) - return true; - return false; + (void)put; + return idx < SVC_GPFIFO_MAX_BITS && + test_bit(idx, ch->slot_done_epoch); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -5095,6 +5092,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; + /* + * A wrapped put means every slot may hold a new entry: clear + * the per-epoch consumed marks so recycled slots are + * processed (a rewind replay of the previous epoch must not + * suppress them). + */ + if (put < ch->put_prev) + bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + ch->put_prev = put; + get_before = ch->last_get; applied_before = svc.releases_applied; @@ -5676,9 +5683,10 @@ if (gsp_sema_fake_rel && svc.chan[k].rescue_last = ch->rescue_last; svc.chan[k].rescue_fail = ch->rescue_fail; svc.chan[k].defer_since = ch->defer_since; - svc.chan[k].replay_next = ch->replay_next; - memcpy(svc.chan[k].replay, ch->replay, - sizeof(ch->replay)); + svc.chan[k].put_prev = ch->put_prev; + bitmap_copy(svc.chan[k].slot_done_epoch, + ch->slot_done_epoch, + SVC_GPFIFO_MAX_BITS); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives |