diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 522 |
1 files changed, 474 insertions, 48 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 96c9064..d9a0fd6 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -257,8 +257,6 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ - u32 op_retry; /* passes an entry has been held while an operand - * targeted scan was in progress (0 = none) */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -610,6 +608,17 @@ struct sema_svc { u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ + /* + * ISSUE-0002 (r62): unresolved-operand entry holds and their withheld + * release sets. held counts passes an entry waited on a translation; + * applied counts release sets delivered when the retried decode + * completed every op; expired counts holds flushed by the bounded + * deadline (the op stays skipped, exactly as the pre-r62 one-pass + * skip did). + */ + u64 opdef_held; + u64 opdef_applied; + u64 opdef_expired; u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ @@ -1024,6 +1033,16 @@ static void svc_fwd_hold_abort(u32 h_object); */ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu); +struct svc_fwd_rel; +/* + * ISSUE-0002 (r62) unresolved-operand entry hold and release deferral: + * defined with the forwarding section below. Service-thread only. + */ +static bool svc_opdef_release(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 entry_idx, + bool rescue, const struct svc_fwd_rel *r); +static void svc_opdef_reset_all(void); + void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; @@ -3261,10 +3280,11 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, - svc.ops_once_skipped, svc.ops_reident); + svc.ops_once_skipped, svc.ops_reident, + svc.opdef_held, svc.opdef_applied, svc.opdef_expired); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, @@ -3542,9 +3562,21 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * calling for this same page). Tell the pump so it can hold the entry * and retry it while the scan is in progress: a copy whose operand is * only resolvable by the scan needs several passes. - */ - if (svc_resolve_in_progress(va & ~0xfffULL)) - svc.op_tr_pending = 1; + * + * ISSUE-0002 (r58/r62): hold on ANY unresolvable operand, not only + * while a targeted scan is already running. The guest's pma scrub + * submits memsets whose operand translations (client buffer VAs + * pinned by the scanner) land ~1 s after the push; the one-pass skip + * lost that work forever (r58 c3: three bigop memsets rc=-22, their + * pins one pass later, the channel frozen with the client in + * scrubCheckAndWaitForSize). The pump holds the entry AND withholds + * its releases (svc_opdef_release) until the retried decode runs + * every op, bounded by SVC_OPDEF_HOLD_MS. Retrying the op without + * withholding the release corrupted the next session (r59: the late + * memset overwrote refilled data after the client had passed the + * release), so the two must move together. + */ + svc.op_tr_pending = 1; return -ENOENT; } @@ -5948,38 +5980,64 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * The pre-apply bookkeeping above is unchanged; * only the apply moves. Overflow applies inline. */ - if (svc_fwd_defer_arm()) { - if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { - struct svc_fwd_rel *r = - &svc_fwd_rel[svc_fwd_rel_n++]; - - r->kind = SVC_FWD_REL_906F; - r->op = (u8)op; - r->inc_claimed = 0; - r->va = va; - r->payload = payload; - r->entry_idx = entry_idx; - r->dma = 0; - svc_fwd_relay_note(vgpu, ch, r, - 0, false); - svc_fwd_defer_armed = true; - svc.fwd_rel_deferred++; - } else { + /* + * ISSUE-0002 (r62): an unresolved operand of + * this pass owns the entry and its releases; + * svc_opdef_release records this one for the + * hold (or applies a completed set) so the + * client cannot pass the release before the + * retried op runs. + */ + { + struct svc_fwd_rel or; + + memset(&or, 0, sizeof(or)); + or.kind = SVC_FWD_REL_906F; + or.op = (u8)op; + or.va = va; + or.payload = payload; + or.entry_idx = entry_idx; + if (svc_opdef_release(vgpu, ch, entry_idx, + replay, &or)) { /* - * P6b wave 2: an overflowed - * release still applies - * inline; author a withheld - * candidate first. + * Held with the entry (or an + * earlier set was applied + * here): no inline apply. */ - svc.fwd_rel_overflow++; - svc_fwd_soft_copy_inline(vgpu); + } else if (svc_fwd_defer_arm()) { + if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { + struct svc_fwd_rel *r = + &svc_fwd_rel[svc_fwd_rel_n++]; + + r->kind = SVC_FWD_REL_906F; + r->op = (u8)op; + r->inc_claimed = 0; + r->va = va; + r->payload = payload; + r->entry_idx = entry_idx; + r->dma = 0; + svc_fwd_relay_note(vgpu, ch, r, + 0, false); + svc_fwd_defer_armed = true; + svc.fwd_rel_deferred++; + } else { + /* + * P6b wave 2: an + * overflowed release + * still applies inline; + * author a withheld + * candidate first. + */ + svc.fwd_rel_overflow++; + svc_fwd_soft_copy_inline(vgpu); + svc_rel_apply_906f(vgpu, ch, + va, payload, + op); + } + } else { svc_rel_apply_906f(vgpu, ch, - va, payload, - op); + va, payload, op); } - } else { - svc_rel_apply_906f(vgpu, ch, va, payload, - op); } } off += 4 + count * 4; @@ -6056,6 +6114,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } n_ops = 0; /* + * ISSUE-0002 (r62): an unresolved operand + * of this pass owns the entry and its + * releases; svc_opdef_release records + * this one for the hold (or applies a + * completed set) so the client cannot + * pass the release before the retried + * op runs. + */ + { + struct svc_fwd_rel or; + + memset(&or, 0, sizeof(or)); + or.kind = SVC_FWD_REL_REPORTSEM; + or.va = va; + or.payload = st.rs_payload; + or.entry_idx = entry_idx; + if (svc_opdef_release(vgpu, ch, + entry_idx, + replay, &or)) + break; + } + /* * P6b wave 1.5: with a forwarding * candidate captured and the deferral * knob on, record the release instead @@ -6516,6 +6596,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * the record path so the pump still sees the * release as decoded. Overflow applies inline. */ + /* + * ISSUE-0002 (r62): an unresolved operand of + * this pass owns the entry and its releases; + * svc_opdef_release records this one (with its + * one-shot increment claim and raw launch word) + * for the hold, or applies the completed set, + * so the client cannot pass the release before + * the retried op runs. + */ + { + struct svc_fwd_rel or; + + memset(&or, 0, sizeof(or)); + or.kind = SVC_FWD_REL_LAUNCH; + or.op = pending ? 1 : 0; + or.inc_claimed = inc_apply ? 1 : 0; + or.va = sem_va; + or.payload = st.sem_payload; + or.entry_idx = entry_idx; + or.dma = dma; + if (svc_opdef_release(vgpu, ch, + entry_idx, replay, + &or)) { + /* release ends the push */ + n_ops = 0; + break; + } + } if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = @@ -6692,6 +6800,12 @@ static void svc_session_flush(void) svc.direct_base = 0; svc_pending_n = 0; svc_pending_next = 0; + /* + * ISSUE-0002 (r62): the previous client's held releases belong to + * waiters that no longer exist; drop the slots instead of resolving + * them against the next session. + */ + svc_opdef_reset_all(); svc_pool_pa_n = 0; svc_pool_pa_next = 0; svc_retry_next = 0; @@ -10546,6 +10660,284 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); } +/* ------------------------------------------------------------------ */ +/* ISSUE-0002 (r62): unresolved-operand entry hold, releases withheld */ +/* ------------------------------------------------------------------ */ + +/* + * The guest's pma scrub submits vidmem memsets whose operand translations + * (client buffer VAs pinned by the scanner) land about a second after the + * push. Decoding such a push fails that operand for the pass; the op is + * retried while a bounded entry hold keeps the push pending. r59 proved + * that retrying the op is only safe when the entry's releases are + * withheld with it: applying the release first lets the client advance + * (and refill the staging buffer), and the late re-run then overwrites + * finished data (r59: c2 VADD bad, payload zeros). These slots are that + * withheld release set: recorded while the entry is held on an unresolved + * operand, applied in order when the retried decode completes every op - + * or when the bounded hold expires and the entry is consumed with the op + * left skipped, exactly as the pre-r62 one-pass skip did. + * + * Keyed by (device, channel, entry); a slot is alive only while that + * entry is held, so a handful covers every concurrently held channel. + * Cleared on apply, on expiry, when the entry is consumed (a slot for a + * different entry of the same channel), and when the channel/session + * disappears. Service thread only (decode and pump run on it). + */ +#define SVC_OPDEF_SLOTS 8 +/* + * Bounded hold: comfortably above the observed ~1 s pin latency and well + * inside the 25 s CE grace, so a genuinely dead operand delays its + * channel by this much and then falls back to the old skip-and-release. + */ +#define SVC_OPDEF_HOLD_MS 10000u + +struct svc_opdef_ent { + bool active; + struct nv_vgpu_device *vgpu; + u32 h_object; + u32 entry_idx; + unsigned long since; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +}; + +static struct svc_opdef_ent svc_opdef[SVC_OPDEF_SLOTS]; +static u32 svc_opdef_log; /* bounded record-append diagnostic */ + +static struct svc_opdef_ent *svc_opdef_find(struct nv_vgpu_device *vgpu, + u32 h, u32 entry_idx) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (e->active && e->vgpu == vgpu && e->h_object == h && + e->entry_idx == entry_idx) + return e; + } + return NULL; +} + +static struct svc_opdef_ent *svc_opdef_alloc(struct nv_vgpu_device *vgpu, + u32 h, u32 entry_idx) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) { + memset(e, 0, sizeof(*e)); + e->active = true; + e->vgpu = vgpu; + e->h_object = h; + e->entry_idx = entry_idx; + e->since = jiffies; + return e; + } + } + return NULL; +} + +static void svc_opdef_clear(struct svc_opdef_ent *e) +{ + memset(e, 0, sizeof(*e)); +} + +static bool svc_opdef_contains(const struct svc_opdef_ent *e, + const struct svc_fwd_rel *r) +{ + u8 i; + + for (i = 0; i < e->n; i++) + if (e->rel[i].kind == r->kind && e->rel[i].va == r->va && + e->rel[i].payload == r->payload) + return true; + return false; +} + +/* + * Apply the slot's recorded releases and clear it. Forward-only, so a + * record whose word already carries the payload is a no-op; a record + * whose page is still unreadable re-queues through the existing pending + * path inside the specific apply helper. + */ +static void svc_opdef_apply(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 entry_idx) +{ + struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + + if (!e) + return; + if (e->n) { + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + svc.opdef_applied++; + if (gsp_sema_dbg && svc_opdef_log < 32) { + svc_opdef_log++; + pr_info("sema-opdef: apply h=0x%x entry=%u n=%u (ops complete)\n", + ch->h_object, entry_idx, e->n); + } + } + svc_opdef_clear(e); +} + +/* + * Release-site gate, called from svc_decode_stream just before a decoded + * release would be applied inline. Returns true when the caller must NOT + * apply its own release now: + * - an op of this pass still lacks a translation (svc.op_tr_pending): + * record the release for the held entry (never from a rescue decode, + * whose bytes may be a false pairing); + * - an earlier record set exists for this entry: when no op is pending + * any more the set is applied here and the caller's release is part + * of it (matched) or applied by the caller (not matched); while an op + * is still pending the set stays held. + * Returns false when no op is pending and no set exists: the caller + * applies inline, byte-identical to the pre-r62 behavior. + */ +static bool svc_opdef_release(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 entry_idx, + bool rescue, const struct svc_fwd_rel *r) +{ + struct svc_opdef_ent *e; + + if (!ch || entry_idx == SVC_ENTRY_NONE || !r) + return false; + e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + if (svc.op_tr_pending) { + if (rescue) + return e != NULL; + if (!e) { + e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); + if (!e) + return false; /* table full: inline apply */ + } + if (!svc_opdef_contains(e, r)) { + if (r->payload && e->n < SVC_FWD_REL_MAX) { + e->rel[e->n++] = *r; + if (gsp_sema_dbg && svc_opdef_log < 32) { + svc_opdef_log++; + pr_info("sema-opdef: hold h=0x%x entry=%u kind=%u va=0x%llx pay=%u (operand pending)\n", + ch->h_object, entry_idx, + r->kind, r->va, r->payload); + } + } + } + /* + * The release is decoded and owned by the slot; the pump's + * rel_seen bookkeeping must see it on every retry pass. + */ + svc_rel_va_seen = r->va; + svc_rel_count++; + return true; + } + if (!e) + return false; + /* + * A release is decoded and delivered on this pass: the pump's + * rel_seen bookkeeping must see it, otherwise a CE channel treats + * the entry as release-less and holds it for the grace window. + */ + svc_rel_va_seen = r->va; + svc_rel_count++; + if (svc_opdef_contains(e, r)) { + svc_opdef_apply(vgpu, ch, entry_idx); + return true; + } + /* Apply the set, then let the caller apply its own (unrecorded) release. */ + svc_opdef_apply(vgpu, ch, entry_idx); + return false; +} + +/* + * Pump hold decision for an entry whose pass set svc.op_tr_pending. Uses + * the slot's first-pending timestamp as the deadline (a slot may not + * exist yet when the push carried no release: allocate one to time the + * hold). Returns false when the bounded hold is exhausted, so the pump + * flushes the slot and consumes the entry. + */ +static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 entry_idx) +{ + struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + + if (!e) { + e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); + if (!e) + return false; + } + if (time_before(jiffies, + e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) { + svc.opdef_held++; + return true; + } + return false; +} + +/* Deadline hit: deliver the withheld releases; the op stays skipped. */ +static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 entry_idx) +{ + struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + + if (!e) + return; + if (e->n) { + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + pr_info("sema-opdef: EXPIRED h=0x%x entry=%u n=%u applied (operand never translated; op stays skipped)\n", + ch->h_object, entry_idx, e->n); + } + svc.opdef_expired++; + svc_opdef_clear(e); +} + +/* + * The ring advanced past a slot's entry (a slot for another entry of the + * same channel is being processed): its set belongs to a consumed entry. + * Deliver late (forward-only, so it cannot double-apply) and drop it. + */ +static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 keep_idx) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active || e->vgpu != vgpu || + e->h_object != ch->h_object || e->entry_idx == keep_idx) + continue; + if (e->n) + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + svc_opdef_clear(e); + } +} + +/* A slot whose channel is gone from this pass's snapshot: drop it. */ +static void svc_opdef_drop_gone(const struct svc_channel *local, int n) +{ + int i, k; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) + continue; + for (k = 0; k < n; k++) + if (local[k].h_object == e->h_object) + break; + if (k == n) + svc_opdef_clear(e); + } +} + +static void svc_opdef_reset_all(void) +{ + memset(svc_opdef, 0, sizeof(svc_opdef)); +} + /* * Proof line for a held record set resolved without a successful forwarded * copy (failure/timeout/abort): the releases below are applied directly, so @@ -12394,6 +12786,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_hold_channel_gone(local, n))) svc_fwd_hold_gone_finish(vgpu); + /* + * ISSUE-0002 (r62): an op-hold slot whose channel is not in this + * pass's snapshot belongs to a vanished channel; its waiter is gone, + * so drop the set instead of resolving it later. + */ + svc_opdef_drop_gone(local, n); + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -12657,6 +13056,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * ISSUE-0002 (r62): a slot recorded for a different + * entry of this channel belongs to an entry the ring + * already moved past: deliver its releases late + * (forward-only, so it cannot double-apply) and drop + * it before this entry is examined. + */ + svc_opdef_drop_stale(vgpu, ch, (u32)idx); + + /* * A pin-batch rewind replays recently consumed * entries. Re-processing one (rescue, grace) costs * seconds and delivers nothing, and a slot consumed @@ -13020,20 +13428,27 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) len); } /* - * An operand of this push - * failed to translate but a - * targeted RAM scan for it is - * in progress: hold the entry - * and retry it next pass (the - * scan is incremental), like - * the unreadable-push hold. - * Bounded so a scan that finds - * nothing cannot wedge the - * channel. + * ISSUE-0002 (r62): an + * unresolved operand holds + * the entry AND withholds + * this pass's releases + * (recorded by + * svc_opdef_release above) + * until the retried decode + * completes every op or the + * bounded hold expires. + * Retrying the op while its + * release was already + * visible corrupted the next + * client (r59: the client + * advanced and the late + * memset overwrote refilled + * data), so the hold and the + * deferral move together. */ if (svc.op_tr_pending && - ch->op_retry < 24) { - ch->op_retry++; + svc_opdef_hold(vgpu, ch, + (u32)idx)) { svc.gpfifo_nomap++; /* * P6b wave 1.5: @@ -13058,7 +13473,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - ch->op_retry = 0; + if (svc.op_tr_pending) + /* + * Deadline hit: the + * operand never + * translated; deliver the + * withheld releases and + * consume the entry (the + * op stays skipped, the + * pre-r62 fallback). + */ + svc_opdef_flush(vgpu, ch, + (u32)idx); rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; |