diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 257 |
1 files changed, 145 insertions, 112 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index d9a0fd6..77a1fae 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1041,6 +1041,8 @@ struct svc_fwd_rel; static bool svc_opdef_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 entry_idx, bool rescue, const struct svc_fwd_rel *r); +static void svc_opdef_drop(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 entry_idx); static void svc_opdef_reset_all(void); void gsp_sema_service_channel_del(u32 h_object) @@ -5981,12 +5983,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * only the apply moves. Overflow applies inline. */ /* - * ISSUE-0002 (r62): an unresolved operand of - * this pass owns the entry and its releases; - * svc_opdef_release records this one for the - * hold (or applies a completed set) so the - * client cannot pass the release before the - * retried op runs. + * ISSUE-0002 (r62/r65): an unresolved operand + * of this pass holds the entry. Order matters: + * a captured forwarding candidate owns the + * release (the forwarded hardware copy's + * completion is the delivery edge - r65: a + * software apply here let the client pass the + * launch-completion wait while the physical + * launch was still in flight and later timed + * out, so the client read zeros); only when no + * candidate is captured does the operand hold + * own the release, and then it is recorded + * instead of applied so the client cannot pass + * it before the retried op runs. */ { struct svc_fwd_rel or; @@ -5997,14 +6006,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; - if (svc_opdef_release(vgpu, ch, entry_idx, - replay, &or)) { - /* - * Held with the entry (or an - * earlier set was applied - * here): no inline apply. - */ - } else if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm()) { + svc_opdef_drop(vgpu, ch, + entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -6034,6 +6038,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, va, payload, op); } + } else if (svc_opdef_release(vgpu, ch, + entry_idx, + replay, &or)) { + /* + * Held with the entry (or an + * earlier set was applied + * here): no inline apply. + */ } else { svc_rel_apply_906f(vgpu, ch, va, payload, op); @@ -6114,36 +6126,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } n_ops = 0; /* - * ISSUE-0002 (r62): an unresolved operand - * of this pass owns the entry and its - * releases; svc_opdef_release records - * this one for the hold (or applies a - * completed set) so the client cannot - * pass the release before the retried - * op runs. - */ - { - struct svc_fwd_rel or; - - memset(&or, 0, sizeof(or)); - or.kind = SVC_FWD_REL_REPORTSEM; - or.va = va; - or.payload = st.rs_payload; - or.entry_idx = entry_idx; - if (svc_opdef_release(vgpu, ch, - entry_idx, - replay, &or)) - break; - } - /* - * P6b wave 1.5: with a forwarding - * candidate captured and the deferral - * knob on, record the release instead - * of applying it; the pump applies it - * once the forwarded copy completes. - * Overflow applies inline. + * ISSUE-0002 (r62/r65): a captured + * forwarding candidate owns the + * release (the hardware completion + * is the delivery edge); only + * without one does the operand hold + * own it, and then it is recorded + * instead of applied. */ if (svc_fwd_defer_arm()) { + svc_opdef_drop(vgpu, ch, + entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -6170,6 +6163,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); + } else { + struct svc_fwd_rel or; + + memset(&or, 0, sizeof(or)); + or.kind = SVC_FWD_REL_REPORTSEM; + or.va = va; + or.payload = st.rs_payload; + or.entry_idx = entry_idx; + if (svc_opdef_release(vgpu, ch, + entry_idx, + replay, &or)) + break; } svc_rel_apply_reportsem(vgpu, ch, va, st.rs_payload); @@ -6597,13 +6602,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * release as decoded. Overflow applies inline. */ /* - * ISSUE-0002 (r62): an unresolved operand of - * this pass owns the entry and its releases; - * svc_opdef_release records this one (with its + * ISSUE-0002 (r62/r65): a captured forwarding + * candidate owns the release - the forwarded + * hardware copy's completion is the delivery + * edge. r65: applying it here (operand-hold + * path) let the client pass the launch- + * completion wait while the physical launch + * was still in flight and later timed out, so + * the client read zeros. Only without a + * candidate does the operand hold own the + * release, and then it is recorded (with its * one-shot increment claim and raw launch word) - * for the hold, or applies the completed set, - * so the client cannot pass the release before - * the retried op runs. + * instead of applied. */ { struct svc_fwd_rel or; @@ -6616,81 +6626,89 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.payload = st.sem_payload; or.entry_idx = entry_idx; or.dma = dma; - if (svc_opdef_release(vgpu, ch, - entry_idx, replay, - &or)) { + if (svc_fwd_defer_arm()) { + svc_opdef_drop(vgpu, ch, + entry_idx); + if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { + struct svc_fwd_rel *r = + &svc_fwd_rel[svc_fwd_rel_n++]; + + r->kind = SVC_FWD_REL_LAUNCH; + r->op = pending ? 1 : 0; + r->inc_claimed = inc_apply ? 1 : 0; + r->va = sem_va; + r->payload = st.sem_payload; + r->entry_idx = entry_idx; + r->dma = dma; + svc_fwd_relay_note(vgpu, ch, r, + dma, true); + /* + * P6b wave 1.5: do + * NOT bump + * svc.pushes_pending + * here - + * svc_rel_apply_launch() + * counts it once when + * the deferred record + * is applied (or drops + * it unapplied). + */ + svc_rel_va_seen = sem_va; + svc_rel_count++; + svc_fwd_defer_armed = true; + svc.fwd_rel_deferred++; + } else { + /* + * P6b wave 2: an + * overflowed release + * still applies inline; + * author a withheld + * candidate first. + */ + svc.fwd_rel_overflow++; + svc_fwd_soft_copy_inline(vgpu); + if (svc_rel_apply_launch(vgpu, ch, + sem_va, + st.sem_payload, + cur, pending, + inc, inc_apply)) { + svc_rel_va_seen = sem_va; + svc_rel_count++; + } + } + } else if (svc_opdef_release(vgpu, ch, + entry_idx, + replay, &or)) { /* release ends the push */ n_ops = 0; break; - } - } - if (svc_fwd_defer_arm()) { - if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { - struct svc_fwd_rel *r = - &svc_fwd_rel[svc_fwd_rel_n++]; - - r->kind = SVC_FWD_REL_LAUNCH; - r->op = pending ? 1 : 0; - r->inc_claimed = inc_apply ? 1 : 0; - r->va = sem_va; - r->payload = st.sem_payload; - r->entry_idx = entry_idx; - r->dma = dma; - svc_fwd_relay_note(vgpu, ch, r, - dma, true); - /* - * P6b wave 1.5: do NOT bump - * svc.pushes_pending here - - * svc_rel_apply_launch() - * counts it once when the - * deferred record is applied - * (or drops it unapplied). - */ - svc_rel_va_seen = sem_va; - svc_rel_count++; - svc_fwd_defer_armed = true; - svc.fwd_rel_deferred++; } else { /* - * P6b wave 2: an overflowed - * release still applies - * inline; author a withheld - * candidate first. + * Deferral is enabled but no + * forwarded hardware copy can + * run for this push + * (guest-memory copy path + * off, or no candidate): + * apply the decoded release + * directly, exactly once. + * The note is the proof line + * that a decoded release is + * never left to a copy that + * will not engage. */ - svc.fwd_rel_overflow++; - svc_fwd_soft_copy_inline(vgpu); - if (svc_rel_apply_launch(vgpu, ch, - sem_va, + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) + svc_fwd_rel_direct_note(ch->h_object, + sem_va, + st.sem_payload); + if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, - cur, pending, - inc, inc_apply)) { + cur, pending, inc, + inc_apply)) { svc_rel_va_seen = sem_va; svc_rel_count++; } } - } else { - /* - * Deferral is enabled but no forwarded - * hardware copy can run for this push - * (guest-memory copy path off, or no - * candidate): apply the decoded release - * directly, exactly once. The note is - * the proof line that a decoded release - * is never left to a copy that will - * not engage. - */ - if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) - svc_fwd_rel_direct_note(ch->h_object, - sem_va, - st.sem_payload); - if (svc_rel_apply_launch(vgpu, ch, sem_va, - st.sem_payload, - cur, pending, inc, - inc_apply)) { - svc_rel_va_seen = sem_va; - svc_rel_count++; - } } /* release ends the push */ n_ops = 0; @@ -10894,6 +10912,21 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } /* + * The forwarding machinery owns this release now (a candidate was captured + * for the pass and its record set carries the same release): drop any + * record a held-operand pass left for this entry without applying it, so + * the forwarded hardware copy's completion is the single delivery edge. + */ +static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 entry_idx) +{ + struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + + if (e) + svc_opdef_clear(e); +} + +/* * The ring advanced past a slot's entry (a slot for another entry of the * same channel is being processed): its set belongs to a consumed entry. * Deliver late (forward-only, so it cannot double-apply) and drop it. |