diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 98 |
1 files changed, 98 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3da8e54..52819da 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1616,6 +1616,25 @@ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ +/* + * r551: releases DELIVERED by the opdef hold-expiry paths (flush/sweep) are + * remembered by (channel, entry, va, payload) for a short window, so a + * re-decode of the parked entry does not withhold the (monotonic, + * forward-only) release again - see the helpers with the opdef machinery. + * Declared here because svc_session_flush() resets them per session. + */ +#define SVC_OPDEF_DONE_SLOTS 256 +#define SVC_OPDEF_DONE_TTL_MS 60000u +struct svc_opdef_done_rec { + u32 h_object; + u32 entry_idx; + u64 va; + u32 payload; + unsigned long j; +}; +static struct svc_opdef_done_rec svc_opdef_done[SVC_OPDEF_DONE_SLOTS]; +static u32 svc_opdef_done_next; + void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { unsigned long flags; @@ -12209,6 +12228,9 @@ static void svc_session_flush(void) memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; + /* r551: delivered-release marks are per-session (same handles reused). */ + memset(svc_opdef_done, 0, sizeof(svc_opdef_done)); + svc_opdef_done_next = 0; /* * Re-arm the diagnostic log budgets per session. The first client of * a boot would otherwise spend every budget and the next session's @@ -17244,6 +17266,59 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) memset(e, 0, sizeof(*e)); } +/* + * r551: remember releases DELIVERED by the hold-expiry paths (flush and + * sweep) by (channel, entry, va, payload) for a short window. The pump + * re-decodes a parked entry on every pass; without this mark each re-decode + * recorded a fresh opdef slot and held the entry again - r550 + * victory_strided: SWEEP-EXPIRED every ~3 s for the SAME entry six times + * (entries 41..43, ~18-21 s per entry, OPDEF-EXPIRED only after six + * cycles), so a ~40-entry init sequence walked for 12+ minutes with the + * client at 100 % CPU. The release is a monotonic completion signal and + * every delivery path is forward-only, so re-applying it is a no-op: a + * re-decode of an already-delivered entry must neither withhold the + * release again nor re-arm the hold. + */ +static void svc_opdef_done_note(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i = svc_opdef_done_next++ % SVC_OPDEF_DONE_SLOTS; + + svc_opdef_done[i].h_object = h; + svc_opdef_done[i].entry_idx = entry_idx; + svc_opdef_done[i].va = r->va; + svc_opdef_done[i].payload = r->payload; + svc_opdef_done[i].j = jiffies; +} + +static bool svc_opdef_done_seen(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + svc_opdef_done[i].va == r->va && + svc_opdef_done[i].payload == r->payload && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + +/* Any release of this entry delivered recently? (hold gate) */ +static bool svc_opdef_done_entry(u32 h, u32 entry_idx) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + static bool svc_opdef_contains(const struct svc_opdef_ent *e, const struct svc_fwd_rel *r) { @@ -17372,6 +17447,16 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (svc.op_tr_pending) { if (rescue) return e != NULL; + /* + * r551: this release was already delivered forward-only by the + * hold-expiry flush/sweep; the pump's re-decode of the parked + * entry must not withhold it (or re-arm the hold) again. + */ + if (svc_opdef_done_seen(ch->h_object, entry_idx, r)) { + svc_rel_va_seen = r->va; + svc_rel_count++; + return true; + } if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -17426,6 +17511,9 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + /* r551: a delivered entry is consumed, never held again. */ + if (svc_opdef_done_entry(ch->h_object, entry_idx)) + return false; if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -17483,7 +17571,11 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return; if (e->n) { + u8 i; + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(ch->h_object, entry_idx, &e->rel[i]); pr_info("sema-opdef: EXPIRED h=0x%x entry=%u n=%u applied (operand never translated; op stays skipped)\n", ch->h_object, entry_idx, e->n); } @@ -17667,7 +17759,13 @@ static void svc_opdef_expire_sweep(void) } spin_unlock_irqrestore(&svc.chan_lock, flags); if (local) { + u8 i; + svc_fwd_rel_apply_all(e->vgpu, local, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(e->h_object, + (u32)e->entry_idx, + &e->rel[i]); svc.opdef_expired++; pr_info_ratelimited("sema-opdef: SWEEP-EXPIRED h=0x%x entry=%u n=%u applied forward-only (entry parked past the hold window)\n", e->h_object, e->entry_idx, e->n); |