diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 29 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 47 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 28 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1261 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 30 |
5 files changed, 1270 insertions, 125 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 90a0a27..69a749a 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -7596,6 +7596,35 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } { + /* + * P6b wave-1.5/wave-2 deferred guest releases and copy + * authorship: the sema service's held-release lifecycle plus + * the deferral/hardware-primary parameter state. pending is the + * current hold gauge (releases waiting on the forwarded copy); + * copy_skipped is the withheld software copies, copy_soft/ + * copy_fail the service-thread fallback copies; the rest are + * monotonic counters. + */ + u64 g_copy_skipped = 0, g_copy_soft = 0, g_copy_fail = 0; + u64 g_deferred = 0, g_applied = 0; + u64 g_fallback = 0, g_pending = 0, g_overflow = 0; + u64 g_engaged = 0, g_timeouts = 0, g_canceled = 0; + + gsp_sema_service_fwd_hold_stats(&g_copy_skipped, &g_copy_soft, + &g_copy_fail, &g_deferred, + &g_applied, &g_fallback, + &g_pending, &g_overflow, + &g_engaged, &g_timeouts, + &g_canceled); + seq_printf(m, + "work_guard: copy_skipped=%llu deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu defer_param=%u copy_soft=%llu copy_fail=%llu hw_primary_param=%u\n", + g_copy_skipped, g_deferred, g_applied, g_fallback, + g_pending, g_overflow, g_engaged, g_timeouts, + g_canceled, gsp_phys_work_defer_release_enabled(), + g_copy_soft, g_copy_fail, + gsp_phys_work_hw_primary_enabled()); + } + { char guest_ram[512]; gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index da6d7fc..512d30b 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -201,6 +201,53 @@ bool gsp_phys_guest_mem_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); /* + * P6b wave-1.5 deferred guest release (default 0). When set together with + * gsp_phys_work_forward=1 and gsp_phys_guest_mem=1, a decoded CE push whose + * guest copy is forwarded to the real engine defers its guest-visible + * semaphore release(s) until the hardware copy completes (or its bounded + * window expires), then applies them from the service thread. The software + * copy still runs inline, exactly as in wave 1. With 0, every release is + * applied inline as before: byte-identical wave-1 behavior. One-line + * rollback: gsp_phys_work_defer_release=0. + */ +static bool gsp_phys_work_defer_release_param; +module_param_named(gsp_phys_work_defer_release, gsp_phys_work_defer_release_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_work_defer_release, + "Defer a forwarded CE push's guest-visible release until its hardware copy completes (default 0: releases apply inline, byte-identical wave-1 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1)"); + +bool gsp_phys_work_defer_release_enabled(void) +{ + return gsp_phys_work_defer_release_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_work_defer_release_enabled); + +/* + * P6b wave-2 hardware-primary guest copies (default 0). When set together + * with gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and + * gsp_phys_work_defer_release=1, a decoded CE push whose copy is forwarded + * to the real engine skips the software copy (the hardware authors the + * data) while the wave-1.5 hold still withholds the guest-visible release + * until the hardware copy completes. If the hardware path fails, times out, + * is aborted, or the request cannot be queued as a guest copy, the service + * thread performs the software copy exactly once before applying the + * releases, so the guest always reads correct data. With 0, every behavior + * is byte-identical to wave 1.5 (the software copy always runs inline and + * copy_skipped stays 0). One-line rollback: gsp_phys_work_hw_primary=0. + */ +static bool gsp_phys_work_hw_primary_param; +module_param_named(gsp_phys_work_hw_primary, gsp_phys_work_hw_primary_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_work_hw_primary, + "Skip the software copy for a forwarded guest CE push (hardware authors the data; the release still waits for hardware completion; one software fallback copy on any hardware/queue failure; default 0: wave-1.5 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); + +bool gsp_phys_work_hw_primary_enabled(void) +{ + return gsp_phys_work_hw_primary_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 1d1db64..70b98e4 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -84,7 +84,9 @@ bool gsp_phys_ctrl_forward_enabled(void); * guest's CE pushes stay software-emulated exactly as before). When set, the * sema service shadows each decoded CE push as one copy on the debugfs ladder's * real channel using only module-owned ladder buffers; the software release - * remains authoritative and is applied unchanged either way. + * remains authoritative and is applied unchanged either way, unless + * gsp_phys_work_defer_release moves a forwarded push's release after its + * hardware copy (see below). */ bool gsp_phys_work_forward_enabled(void); @@ -98,6 +100,30 @@ bool gsp_phys_work_forward_enabled(void); bool gsp_phys_guest_mem_enabled(void); /* + * True when the gsp_phys_work_defer_release module parameter is set (default + * 0: a forwarded guest CE push's release is applied inline with the software + * copy, byte-identical to wave 1). When set with gsp_phys_work_forward and + * gsp_phys_guest_mem, the release(s) a forwarded push decoded are held on the + * service thread until the hardware copy completes, its bounded timeout + * expires, or the forwarding is aborted; they are then applied from the + * service thread. One-line rollback: gsp_phys_work_defer_release=0. + */ +bool gsp_phys_work_defer_release_enabled(void); + +/* + * True when the gsp_phys_work_hw_primary module parameter is set (default + * 0: byte-identical to wave 1.5, the software copy always runs inline). When + * set with gsp_phys_work_forward, gsp_phys_guest_mem and + * gsp_phys_work_defer_release, a forwarded guest CE copy skips the software + * copy (the hardware authors the data) and the release still lands only + * after hardware completion through the wave-1.5 hold. Any hardware failure, + * timeout, abort or failed guest queue falls back to exactly one software + * copy in the service thread before the releases are applied, so the guest + * always reads correct data. One-line rollback: gsp_phys_work_hw_primary=0. + */ +bool gsp_phys_work_hw_primary_enabled(void); + +/* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): * its VFIO type-1 container keeps the guest DMA mappings privately, so diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 51c51b2..1f599fc 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -445,6 +445,31 @@ struct sema_svc { u64 fwd_cand_vid_vid; u64 fwd_guest_resolve_fail; u64 fwd_guest_resolve_log; /* first-16 diagnostic budget */ + /* + * P6b wave-1.5 deferred guest releases: releases recorded during a + * decode pass whose application waits for the forwarded hardware copy + * (fwd_rel_deferred/fwd_rel_pending), the ones applied once it + * completed (fwd_rel_applied) or failed/timed out/was aborted + * (fwd_rel_fallback), and the record-bound drops (fwd_rel_overflow). + * P6b wave 2 adds the hardware-primary copy accounting: + * fwd_copy_skipped (the software copy was skipped; hardware authors + * the data), fwd_copy_soft (fallback software copies the service + * thread performed before applying releases) and fwd_copy_fail + * (fallback copy attempts that failed). All zero while the feature is + * off. The hold counters describe the lifecycle of the single held + * entry. + */ + u64 fwd_copy_skipped; + u64 fwd_copy_soft; + u64 fwd_copy_fail; + u64 fwd_rel_deferred; + u64 fwd_rel_applied; + u64 fwd_rel_fallback; + u64 fwd_rel_pending; + u64 fwd_rel_overflow; + u64 fwd_hold_engaged; + u64 fwd_hold_timeouts; + u64 fwd_hold_canceled; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -641,6 +666,15 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) desc->h_object, SVC_MAX_CHANNELS); } +/* P6b wave-1.5 hold state is defined with the forwarding section below. */ +static void svc_fwd_hold_abort(u32 h_object); + +/* + * P6b wave-2 inline software fallback for a withheld (skipped) candidate; + * defined with the forwarding section below. Service-thread only. + */ +static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu); + void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; @@ -659,6 +693,14 @@ void gsp_sema_service_channel_del(u32 h_object) } } spin_unlock_irqrestore(&svc.chan_lock, flags); + + /* + * P6b wave 1.5: a deleted channel's held releases must not wait out + * the timeout; mark the hold aborted so the pump applies them (or + * drops them if the snapshot already lost the channel) on its next + * pass. + */ + svc_fwd_hold_abort(h_object); } /* @@ -1916,6 +1958,23 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.fwd_guest_fallback, svc.fwd_cand_none, svc.fwd_cand_sys_sys, svc.fwd_cand_mixed, svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail); + /* + * P6b wave-1.5: deferred-release/hold summary. Printed whenever any + * counter moved so an armed run that never engaged a hold is still + * diagnosable from the log. P6b wave 2 adds the copy accounting. + */ + if (svc.fwd_copy_skipped || svc.fwd_copy_soft || svc.fwd_copy_fail || + svc.fwd_rel_deferred || svc.fwd_rel_applied || + svc.fwd_rel_fallback || svc.fwd_rel_pending || + svc.fwd_rel_overflow || svc.fwd_hold_engaged || + svc.fwd_hold_timeouts || svc.fwd_hold_canceled) + pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu\n", + svc.fwd_rel_deferred, svc.fwd_rel_applied, + svc.fwd_rel_fallback, svc.fwd_rel_pending, + svc.fwd_rel_overflow, svc.fwd_hold_engaged, + svc.fwd_hold_timeouts, svc.fwd_hold_canceled, + svc.fwd_copy_skipped, svc.fwd_copy_soft, + svc.fwd_copy_fail); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -2209,9 +2268,17 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, * an HPA through the guest-RAM backing file and a VID operand into an HPA * through the vmalloc backing of the shadow; no address is ever guessed. * Service-thread only, like the rest of the pump state. + * + * P6b wave 2: `skipped` records that the software copy was not run for this + * capture because the hardware-primary path is armed to make the real engine + * author the data. It is only ever set on a decode pass whose releases are + * deferred through the wave-1.5 hold (or fall back to one service-thread + * software copy); a skipped candidate is copied exactly once later by + * svc_fwd_soft_copy() when the hardware does not deliver. */ struct svc_fwd_cand { bool valid; + bool skipped; /* P6b wave 2: software copy withheld */ u8 src_kind; /* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */ u8 dst_kind; u64 src_page, dst_page; /* resolver page: GPA / shadow byte offset */ @@ -2221,6 +2288,46 @@ struct svc_fwd_cand { static struct svc_fwd_cand svc_fwd_cand; +/* + * P6b wave-1.5 deferred guest releases: every release group the current + * decode pass decoded while a forwarding candidate was captured is recorded + * here instead of being applied inline. The record carries exactly what the + * three release sites need to re-run their apply body from the service thread + * when the hardware copy finishes: the encoding kind, the 906f semaphore op / + * LAUNCH pending flag, the LAUNCH one-shot increment claim, the release VA and + * payload, the GPFIFO entry index, and the raw LAUNCH_DMA word. Reset with + * svc_fwd_cand at the start of every decode pass; consumed by the pump on the + * same service thread. + */ +#define SVC_FWD_REL_MAX 8 +#define SVC_FWD_REL_906F 1 +#define SVC_FWD_REL_REPORTSEM 2 +#define SVC_FWD_REL_LAUNCH 3 + +struct svc_fwd_rel { + u8 kind; /* SVC_FWD_REL_* */ + u8 op; /* 906f: host sem op / launch: pending flag */ + u8 inc_claimed; /* launch: one-shot inc release claimed */ + u64 va; + u32 payload; + u32 entry_idx; + u32 dma; /* launch: raw LAUNCH_DMA word */ +}; + +static struct svc_fwd_rel svc_fwd_rel[SVC_FWD_REL_MAX]; +static u8 svc_fwd_rel_n; +/* True while the current decode pass actually deferred a release. */ +static bool svc_fwd_defer_armed; + +/* + * P6b wave 2: positive sentinel returned by svc_copy() when the captured + * forwarding candidate's software copy is deliberately withheld (the + * hardware-primary path will have the real engine author the data). + * svc_run_ops() treats it as "no software work": it is neither a success nor + * a failure and touches no memsets counters. + */ +#define SVC_COPY_FORWARDED 1 + /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) @@ -2297,6 +2404,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (!overlap && src_off + total <= SEMA_PAGE_SIZE && dst_off + total <= SEMA_PAGE_SIZE) { svc_fwd_cand.valid = true; + svc_fwd_cand.skipped = false; svc_fwd_cand.src_kind = src_kind; svc_fwd_cand.dst_kind = dst_kind; svc_fwd_cand.src_page = src_page; @@ -2304,6 +2412,32 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, svc_fwd_cand.src_off = src_off; svc_fwd_cand.dst_off = dst_off; svc_fwd_cand.bytes = (u32)total; + + /* + * P6b wave 2: with the hardware-primary path armed + * (parameter on and all three prerequisites on, so the + * request can queue as a guest copy and its releases + * can be held), arm the defer machinery for this + * decode and withhold the software copy: the forwarded + * request's real copy authors the data and the + * wave-1.5 hold keeps the guest release behind it. + * Every fallback path (request refused/not queued as + * guest, hardware failure/timeout/abort, stop flush) + * runs the software copy exactly once through + * svc_fwd_soft_copy() before applying the releases. + * Returning here skips the copy loop and the watch + * logic exactly like an executed copy; svc_run_ops() + * ignores the sentinel. + */ + if (gsp_phys_work_hw_primary_enabled() && + gsp_phys_work_defer_release_enabled() && + gsp_phys_work_forward_enabled() && + gsp_phys_guest_mem_enabled()) { + svc_fwd_defer_armed = true; + svc_fwd_cand.skipped = true; + svc.fwd_copy_skipped++; + return SVC_COPY_FORWARDED; + } } } @@ -2534,6 +2668,13 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); + /* + * P6b wave 2: a withheld software copy (hardware-primary) is + * not software work: it is neither a success nor a failure, so + * no memsets counter moves and no op log fires for it. + */ + if (rc == SVC_COPY_FORWARDED) + continue; if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) @@ -2903,6 +3044,95 @@ static void svc_counter_learn(u32 h, u64 sem_va) spin_unlock_irqrestore(&svc.chan_lock, flags); } +/* + * P6b wave-1.5 release-apply helpers: the exact bodies the three release + * sites used inline, extracted so a deferred record can re-run the same apply + * from the service thread once the forwarded hardware copy completed (or its + * window expired). Semantics are identical to the inline code; each returns + * true only when the generic VA write ran (the LAUNCH site keys its + * svc_rel_va_seen/svc_rel_count bookkeeping on that, exactly as before). + * Service-thread only, like every other release path. + */ +static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 va, + u32 payload, u32 op) +{ + u32 cur = 0; + + if (ch->h_object && svc_counter_va(va)) { + svc_counter_learn(ch->h_object, va); + svc_counter_release_apply(vgpu, ch->h_object, va, payload); + return false; + } + if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { + svc.pushes_pending++; + cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; + if (!svc_sem_write(vgpu, va, cur)) { + svc.releases_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + return true; + } + } + return false; +} + +static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 va, + u32 payload) +{ + u32 cur = 0; + + if (ch->h_object && svc_counter_va(va)) { + svc_counter_learn(ch->h_object, va); + svc_counter_release_apply(vgpu, ch->h_object, va, payload); + return false; + } + if (svc_sem_read(vgpu, va, &cur)) { + svc_pending_add(va, payload, 0); + return false; + } + if (payload <= cur) + return false; + svc.pushes_pending++; + if (!svc_sem_write(vgpu, va, payload)) { + svc.releases_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + return true; + } + return false; +} + +static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 sem_va, + u32 payload, u32 cur, bool pending, + bool inc, bool inc_apply) +{ + if (pending || inc_apply) { + svc.pushes_pending++; + if (inc) + cur += payload; + else + cur = payload; + /* + * GR completion counters live in a client semaphore page whose + * generic translation can be a stale content-scanned page; + * those releases go through the injected-pin-backed exact slot + * (svc_counter_learn / svc_counter_release_apply) below. + */ + if (ch->engine_type == 0 && svc_counter_va(sem_va)) { + /* handled by the exact-slot path */ + } else if (!svc_sem_write(vgpu, sem_va, cur)) { + svc.releases_applied++; + svc.post_hint_until = jiffies + 10 * HZ; + return true; + } + } + if (ch->engine_type == 0) + svc_counter_learn(ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload); + return false; +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -2937,9 +3167,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* * P6b wave 1: clear the guest-forwarding candidate at the start of * every decode pass; the first eligible copy captured below is the one - * svc_fwd_request() sees when this pass ends. + * svc_fwd_request() sees when this pass ends. P6b wave 1.5: the + * deferred-release records are per-pass state too. P6b wave 2: the + * withheld-copy flag is per-capture state and is cleared with the + * candidate (the defer arming below only persists while releases are + * actually recorded). */ svc_fwd_cand.valid = false; + svc_fwd_cand.skipped = false; + svc_fwd_rel_n = 0; + svc_fwd_defer_armed = false; svc.pages_decoded++; if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; @@ -3113,7 +3350,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, (u64)(v0 & 0xfffffffc); u32 payload = v2; u32 op = v4 & 0x7; - u32 cur = 0; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && @@ -3142,20 +3378,45 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * is a client UVM allocation outside every * scanned window), and a dropped release leaves * the client's tracker polling forever. + * + * P6b wave 1.5: with a forwarding candidate + * captured and the deferral knob on, record the + * release instead of applying it; the pump + * applies it once the forwarded copy completes. + * The pre-apply bookkeeping above is unchanged; + * only the apply moves. Overflow applies inline. */ - if (ch->h_object && svc_counter_va(va)) { - svc_counter_learn(ch->h_object, va); - svc_counter_release_apply(vgpu, ch->h_object, - va, payload); - } else if (!svc_sem_read(vgpu, va, &cur) && - payload > cur) { - svc.pushes_pending++; - cur = (op == HOST_SEM_OP_REDUCTION) ? - cur + payload : payload; - if (!svc_sem_write(vgpu, va, cur)) { - svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) { + if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { + struct svc_fwd_rel *r = + &svc_fwd_rel[svc_fwd_rel_n++]; + + r->kind = SVC_FWD_REL_906F; + r->op = (u8)op; + r->inc_claimed = 0; + r->va = va; + r->payload = payload; + r->entry_idx = entry_idx; + r->dma = 0; + svc_fwd_defer_armed = true; + svc.fwd_rel_deferred++; + } else { + /* + * P6b wave 2: an overflowed + * release still applies + * inline; author a withheld + * candidate first. + */ + svc.fwd_rel_overflow++; + svc_fwd_soft_copy_inline(vgpu); + svc_rel_apply_906f(vgpu, ch, + va, payload, + op); } + } else { + svc_rel_apply_906f(vgpu, ch, va, payload, + op); } } off += 4 + count * 4; @@ -3190,7 +3451,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case HOST_M_REPORT_SEM_D: { u64 va; - u32 cur = 0; if ((vv[k] & HOST_RS_OP_MASK) != HOST_RS_OP_RELEASE || @@ -3231,35 +3491,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ops_runs++; } n_ops = 0; - if (ch->h_object && svc_counter_va(va)) { - /* - * Exact-slot delivery for GR - * counters (see the 906f SEM - * block above). - */ - svc_counter_learn(ch->h_object, va); - svc_counter_release_apply(vgpu, - ch->h_object, va, - st.rs_payload); - break; - } - if (svc_sem_read(vgpu, va, &cur)) { + /* + * P6b wave 1.5: with a forwarding + * candidate captured and the deferral + * knob on, record the release instead + * of applying it; the pump applies it + * once the forwarded copy completes. + * Overflow applies inline. + */ + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) { + if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { + struct svc_fwd_rel *r = + &svc_fwd_rel[svc_fwd_rel_n++]; + + r->kind = SVC_FWD_REL_REPORTSEM; + r->op = 0; + r->inc_claimed = 0; + r->va = va; + r->payload = st.rs_payload; + r->entry_idx = entry_idx; + r->dma = 0; + svc_fwd_defer_armed = true; + svc.fwd_rel_deferred++; + break; + } /* - * No translation yet: queue - * the release so it lands when - * the page is discovered. + * P6b wave 2: an overflowed + * release still applies + * inline; author a withheld + * candidate first. */ - svc_pending_add(va, st.rs_payload, 0); - break; - } - if (st.rs_payload <= cur) - break; - svc.pushes_pending++; - if (!svc_sem_write(vgpu, va, - st.rs_payload)) { - svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.fwd_rel_overflow++; + svc_fwd_soft_copy_inline(vgpu); } + svc_rel_apply_reportsem(vgpu, ch, va, + st.rs_payload); break; } default: @@ -3539,35 +3806,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ops_runs++; } - if (pending || inc_apply) { - svc.pushes_pending++; - if (inc) - cur += st.sem_payload; - else - cur = st.sem_payload; - /* - * GR completion counters live in a - * client semaphore page whose generic - * translation can be a stale - * content-scanned page; writing there - * corrupts an unrelated page and never - * reaches the client. Those releases go - * through the injected-pin-backed slot - * (svc_counter_learn / - * svc_counter_release_apply) below. - */ - if (ch->engine_type == 0 && - sem_va >= 0x204000000ULL && - sem_va < 0x204100000ULL) { - /* handled by the exact-slot path */ - } else if (!svc_sem_write(vgpu, sem_va, cur)) { - svc.releases_applied++; - svc_rel_va_seen = sem_va; - svc_rel_count++; - svc.post_hint_until = jiffies + 10 * HZ; - } - } - /* * Exact-slot fallback: the releasing channel's * completion counter was enumerated from the @@ -3582,11 +3820,67 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * counter; learn its guest PA from the (pinned) * release VA so the publish path can advance the * exact slot the client polls. + * + * P6b wave 1.5: with a forwarding candidate + * captured and the deferral knob on, record the + * release - including the one-shot increment + * claim made above and the raw launch word - + * instead of applying it; the pump applies it + * once the forwarded copy completes. The + * svc_rel_va_seen/svc_rel_count pair moves to + * the record path so the pump still sees the + * release as decoded. Overflow applies inline. */ - if (ch->engine_type == 0) - svc_counter_learn(ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, - st.sem_payload); + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) { + if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { + struct svc_fwd_rel *r = + &svc_fwd_rel[svc_fwd_rel_n++]; + + r->kind = SVC_FWD_REL_LAUNCH; + r->op = pending ? 1 : 0; + r->inc_claimed = inc_apply ? 1 : 0; + r->va = sem_va; + r->payload = st.sem_payload; + r->entry_idx = entry_idx; + r->dma = dma; + /* + * P6b wave 1.5: do NOT bump + * svc.pushes_pending here - + * svc_rel_apply_launch() + * counts it once when the + * deferred record is applied + * (or drops it unapplied). + */ + svc_rel_va_seen = sem_va; + svc_rel_count++; + svc_fwd_defer_armed = true; + svc.fwd_rel_deferred++; + } else { + /* + * P6b wave 2: an overflowed + * release still applies + * inline; author a withheld + * candidate first. + */ + svc.fwd_rel_overflow++; + svc_fwd_soft_copy_inline(vgpu); + if (svc_rel_apply_launch(vgpu, ch, + sem_va, + st.sem_payload, + cur, pending, + inc, inc_apply)) { + svc_rel_va_seen = sem_va; + svc_rel_count++; + } + } + } else if (svc_rel_apply_launch(vgpu, ch, sem_va, + st.sem_payload, + cur, pending, inc, + inc_apply)) { + svc_rel_va_seen = sem_va; + svc_rel_count++; + } /* release ends the push */ n_ops = 0; } @@ -5175,6 +5469,18 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) */ struct svc_fwd_req { bool guest; + /* + * P6b wave 1.5: the request's captured releases are held for the + * hardware copy. defer is snapshotted by the pump before queue_work + * (the release records are service-thread state that the next decode + * pass resets); done/hw_rc are written by the work item and read by + * the pump's hold check. A request slot is never reused while a hold + * is active (svc_fwd_request refuses to queue), so the held request + * keeps its slot until the hold is resolved. + */ + bool defer; + bool done; + int hw_rc; struct nv_vgpu_device *vgpu; u8 src_kind, dst_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ @@ -5256,6 +5562,453 @@ static void svc_fwd_cand_note(void) svc.fwd_cand_mixed++; } +/* + * P6b wave 2: software fallback for a withheld (skipped) forwarded copy. + * Replays the capture's linear copy between its two resolved operands: SYS + * through the guest-DMA GPA (svc_read/svc_write -> vfio_dma_rw), VID through + * the emulated BAR1 vidmem shadow, in 256-byte bounce chunks like svc_copy()'s + * loop. The capture already guarantees one page per side and that the + * in-page ranges fit, so the only remaining bounds are the shadow size (in + * the AP helpers) and the guest-DMA mapping's own range checks. Returns 0 or + * -errno. Service-thread only: the guest-DMA helpers require the kthread + * context, exactly like svc_copy(). Exactly-once is enforced by the caller + * (hold.copy_done, or the per-pass skipped flag for the inline fallback). + */ +static int svc_fwd_soft_copy(struct nv_vgpu_device *vgpu, + const struct svc_fwd_cand *cand) +{ + u8 bounce[256]; + size_t o; + + if (!vgpu || !cand || !cand->valid || !cand->bytes || + cand->bytes > SEMA_PAGE_SIZE) + return -EINVAL; + if (cand->src_kind != SVC_APERTURE_SYS && + cand->src_kind != SVC_APERTURE_FB) + return -EINVAL; + if (cand->dst_kind != SVC_APERTURE_SYS && + cand->dst_kind != SVC_APERTURE_FB) + return -EINVAL; + + for (o = 0; o < cand->bytes; ) { + size_t chunk = cand->bytes - o; + int rc; + + if (chunk > sizeof(bounce)) + chunk = sizeof(bounce); + rc = svc_read_ap(vgpu, cand->src_page + cand->src_off + o, + cand->src_kind, bounce, chunk); + if (rc) + return rc; + rc = svc_write_ap(vgpu, cand->dst_page + cand->dst_off + o, + cand->dst_kind, bounce, chunk); + if (rc) + return rc; + o += chunk; + } + return 0; +} + +/* + * Inline fallback: the pass captured a skipped candidate whose release + * records are being applied immediately (the request was not queued as a + * guest copy). Author the data once before the releases; clearing `skipped` + * makes the copy exactly-once for the whole decode pass. Service-thread only. + */ +static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) +{ + int rc; + + if (!svc_fwd_cand.valid || !svc_fwd_cand.skipped) + return; + svc_fwd_cand.skipped = false; + rc = svc_fwd_soft_copy(vgpu, &svc_fwd_cand); + if (rc) + svc.fwd_copy_fail++; + else + svc.fwd_copy_soft++; +} + +/* ------------------------------------------------------------------ */ +/* P6b wave-1.5 held release state */ +/* ------------------------------------------------------------------ */ + +/* + * At most one decode pass's releases are held: the forwarded hardware copy + * the pump queued for that push is the only thing standing between the + * decoded release(s) and the guest. The record array and the hold are + * service-thread state; the work item communicates only through svc_fwd_req's + * done/hw_rc. While a hold is active, svc_fwd_request() refuses new requests + * so the request slot the hold reads cannot be reused. The hold is resolved + * by the pump on a later pass: apply on hw success, apply anyway on hw + * failure / 500 ms timeout / abort, drop on a vanished channel. + */ +#define SVC_FWD_HOLD_TIMEOUT_MS 500u + +struct svc_fwd_hold { + bool active; + bool aborted; + /* + * P6b wave 2: snapshot of the skipped candidate this hold's hardware + * copy was forwarded for, plus the exactly-once latch for its software + * fallback. If the hardware copy does not deliver (failure, timeout, + * abort, canceled vanish, stop flush), svc_fwd_hold_soft_copy() runs + * the copy once before the recorded releases are applied; a success + * (done && hw_rc == 0) never copies. + */ + bool copy_done; + struct svc_fwd_cand cand; + /* + * P6b wave 1.5 cross-device safety: the device whose channel engaged + * the hold. Every apply/consume path writes through THIS device's + * address space, never through whichever device happens to be first + * open when the hold is resolved. + */ + struct nv_vgpu_device *vgpu; + u32 ch_handle; + u64 gpfifo_va; + u32 entry_idx; + u8 nrel; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; + u64 last_rel_va; + unsigned long since; +}; + +static struct svc_fwd_hold svc_fwd_hold; + +static void svc_fwd_hold_abort(u32 h_object) +{ + if (READ_ONCE(svc_fwd_hold.active) && + svc_fwd_hold.ch_handle == h_object) + WRITE_ONCE(svc_fwd_hold.aborted, true); +} + +/* + * P6b wave 2: exactly-once software fallback for a hold whose candidate + * skipped the software copy. No-op for a non-skipped candidate, a copy that + * already ran, or a hold whose hardware copy completed successfully (the + * caller decides that last part; this helper only latches once). Runs on the + * service thread. The counter moves on the first attempt only: fwd_copy_soft + * on success, fwd_copy_fail on failure. + */ +static void svc_fwd_hold_soft_copy(struct nv_vgpu_device *vgpu) +{ + int rc; + + if (!vgpu || !svc_fwd_hold.cand.valid || !svc_fwd_hold.cand.skipped || + svc_fwd_hold.copy_done) + return; + svc_fwd_hold.copy_done = true; + rc = svc_fwd_soft_copy(vgpu, &svc_fwd_hold.cand); + if (rc) + svc.fwd_copy_fail++; + else + svc.fwd_copy_soft++; +} + +/* + * Re-apply a deferred LAUNCH record: the decode read the semaphore to compute + * its pending/inc decision, so the apply re-reads the value only for the + * increment arithmetic and uses the RECORDED decision for the write gate - a + * recycled/rewound pool slot must not turn a write the wave-1 path would have + * skipped (op==0: no pending at decode) into a stale absolute write. An + * unreadable page queues the release the same way the decode did. + */ +static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *r) +{ + u32 cur = 0; + bool pending = r->op != 0; + bool inc = DMA_SEM_REDUCTION_EN(r->dma) && + DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC; + + if (svc_sem_read(vgpu, r->va, &cur)) { + svc_pending_add(r->va, r->payload, inc ? 1 : 0); + svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending, + inc, r->inc_claimed); + return; + } + svc_rel_apply_launch(vgpu, ch, r->va, r->payload, cur, pending, + inc, r->inc_claimed); +} + +/* Apply the recorded releases of one deferred push (service thread). */ +static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, + const struct svc_fwd_rel *rel, u8 n) +{ + u8 i; + + for (i = 0; i < n; i++) { + switch (rel[i].kind) { + case SVC_FWD_REL_906F: + svc_rel_apply_906f(vgpu, ch, rel[i].va, + rel[i].payload, rel[i].op); + break; + case SVC_FWD_REL_REPORTSEM: + svc_rel_apply_reportsem(vgpu, ch, rel[i].va, + rel[i].payload); + break; + case SVC_FWD_REL_LAUNCH: + svc_fwd_rel_apply_launch_deferred(vgpu, ch, &rel[i]); + break; + default: + break; + } + } +} + +static bool svc_fwd_hold_expired(void) +{ + return time_after(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); +} + +/* Engage the hold for the current pass's records. */ +static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u32 entry_idx) +{ + u8 n = svc_fwd_rel_n; + + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_hold.rel, svc_fwd_rel, sizeof(svc_fwd_hold.rel)); + svc_fwd_hold.active = true; + svc_fwd_hold.aborted = false; + /* + * P6b wave 2: snapshot the skipped candidate so its software fallback + * can run later from this hold, independently of the per-pass + * svc_fwd_cand (which the next decode pass resets). + */ + svc_fwd_hold.copy_done = false; + svc_fwd_hold.cand = svc_fwd_cand; + svc_fwd_hold.vgpu = vgpu; + svc_fwd_hold.ch_handle = ch->h_object; + svc_fwd_hold.gpfifo_va = ch->gpfifo_va; + svc_fwd_hold.entry_idx = entry_idx; + svc_fwd_hold.nrel = n; + svc_fwd_hold.last_rel_va = svc_rel_va_seen; + svc_fwd_hold.since = jiffies; + svc.fwd_hold_engaged++; + svc.fwd_rel_pending = n; +} + +/* Still present in this pass's channel snapshot? */ +static bool svc_fwd_hold_channel_gone(const struct svc_channel *local, int n) +{ + int i; + + for (i = 0; i < n; i++) + if (local[i].h_object == svc_fwd_hold.ch_handle) + return false; + return true; +} + +static void svc_fwd_hold_clear(void) +{ + svc_fwd_hold.active = false; + svc_fwd_hold.aborted = false; + svc_fwd_hold.copy_done = false; + memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); + svc_fwd_hold.vgpu = NULL; + svc_fwd_hold.nrel = 0; + svc.fwd_rel_pending = 0; +} + +/* + * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held + * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure) + * can skip the entry loop. The recorded releases are applied through the shim + * (holds are CE-only), the hold is cleared and the timeout counted; unlike + * svc_fwd_hold_poll() this writes no GP_GET/tail state, so it is safe on an + * unpumpable channel. No-op when the hold is not this channel's or has not + * expired; clearing guarantees it cannot apply twice. + */ +static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + struct nv_vgpu_device *hv; + struct svc_channel shim; + bool done; + + if (!READ_ONCE(svc_fwd_hold.active) || + svc_fwd_hold.ch_handle != ch->h_object) + return; + if (!svc_fwd_hold_expired()) + return; + done = smp_load_acquire(&svc_fwd_req.done); + hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; + /* + * P6b wave 2: a skipped candidate whose hardware copy did not complete + * successfully is authored by one software copy here, BEFORE the + * recorded releases below. A successful hardware copy needs none. + */ + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_soft_copy(hv); + if (hv) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_hold.ch_handle; + shim.engine_type = 1; /* holds are CE-only */ + svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + } + if (done && svc_fwd_req.hw_rc == 0) { + svc.fwd_rel_applied++; + } else { + svc.fwd_rel_fallback++; + if (!done) + svc.fwd_hold_timeouts++; + } + svc_fwd_hold_clear(); +} + +/* + * Poll the held entry (top of the pump's entry loop). Applies the recorded + * releases once the work item has an outcome or the timeout/abort fires, then + * mirrors the normal consume tail; returns true when the entry was consumed. + * When the copy is still in flight it returns false and the entry stays. + */ +static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? + svc_fwd_hold.vgpu : vgpu; + bool done = smp_load_acquire(&svc_fwd_req.done); + bool expired = svc_fwd_hold_expired(); + bool aborted = READ_ONCE(svc_fwd_hold.aborted); + u32 idx = svc_fwd_hold.entry_idx; + + if (!done && !expired && !aborted) + return false; + if (done && svc_fwd_req.hw_rc == 0) { + svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_applied++; + } else { + /* + * P6b wave 2: failure/timeout/abort fallback - author the + * skipped data once, then apply the releases exactly as + * wave 1.5 does. + */ + svc_fwd_hold_soft_copy(hv); + svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + svc.fwd_rel_fallback++; + if (!done) + svc.fwd_hold_timeouts++; + } + /* Mirror the normal consume tail for the held entry. */ + svc_replay_note(ch, idx, put); + ch->last_get = (idx + 1) % ch->gpfifo_entries; + ch->rescue_fail = 0; + ch->defer_since = 0; + ch->submissions++; + svc.gpfifo_entries_done++; + if (ch->engine_type && ch->expected_sem_payload) + ch->expected_sem_payload++; + ch->last_rel_va = svc_fwd_hold.last_rel_va; + { + u32 g = ch->last_get; + + svc_write_ap(hv, ch->userd_pa + SVC_USERD_GP_GET, + ch->userd_as, &g, sizeof(g)); + } + svc_fwd_hold_clear(); + return true; +} + +/* + * Channel vanished while a hold was active. A successfully completed work + * item or an abort (quiesce / channel delete seen before the snapshot lost + * the channel) applies the recorded releases through a shim channel - holds + * exist only for CE channels and the vanished channel object is not required; + * only a copy that did not complete successfully (timeout/failure/cancel) + * drops without applying. The stored engage-time device is preferred over the + * caller's so the releases never land in a different device's address space. + */ +static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? + svc_fwd_hold.vgpu : vgpu; + bool done = smp_load_acquire(&svc_fwd_req.done); + bool aborted = READ_ONCE(svc_fwd_hold.aborted); + + if (!done && !aborted && !svc_fwd_hold_expired()) + return; + if ((done && svc_fwd_req.hw_rc == 0) || aborted) { + struct svc_channel shim; + + /* + * P6b wave 2: an abort (or a vanished channel after a + * failure/timeout) applies its releases through the shim; + * author the skipped data once first unless the hardware copy + * itself completed successfully. + */ + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_soft_copy(hv); + if (hv) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_hold.ch_handle; + shim.engine_type = 1; /* holds are CE-only */ + svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + } + if (done && svc_fwd_req.hw_rc == 0) + svc.fwd_rel_applied++; + else { + svc.fwd_rel_fallback++; + if (!done) + svc.fwd_hold_timeouts++; + } + } else { + /* + * Canceled (expired without a successful copy): wave 1.5 + * drops the releases, but the skipped data is still authored + * once before the hold is released so the destination is + * correct if anything else observes it. + */ + svc_fwd_hold_soft_copy(hv); + svc.fwd_hold_canceled++; + } + svc_fwd_hold_clear(); +} + +/* Service-thread stop path: apply whatever is still held, best-effort. */ +static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? + svc_fwd_hold.vgpu : vgpu; + struct svc_channel shim; + bool done; + + if (!svc_fwd_hold.active) + return; + done = smp_load_acquire(&svc_fwd_req.done); + /* + * P6b wave 2: the backend is going away; a skipped candidate whose + * hardware copy did not complete successfully is authored by one + * software copy before the best-effort release apply. + */ + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_soft_copy(hv); + if (hv) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_hold.ch_handle; + shim.engine_type = 1; /* holds are CE-only */ + svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + } + if (done && svc_fwd_req.hw_rc == 0) + svc.fwd_rel_applied++; + else + svc.fwd_rel_fallback++; + svc_fwd_hold_clear(); +} + static void svc_fwd_work_fn(struct work_struct *work) { bool guest = svc_fwd_req.guest; @@ -5263,12 +6016,17 @@ static void svc_fwd_work_fn(struct work_struct *work) u32 bytes = 0; unsigned long start = jiffies; int ret; + int hw_rc = 0; /* * P6b wave 1: try the guest hardware copy first when the pump marked - * this request guest. Every failure falls through to the unchanged - * ladder submission below. The request slot is stable here: the - * INFLIGHT bit is held, so only this reader touches it. + * this request guest. P6b wave 1.5: when the request's releases are + * held (defer), a guest failure never falls back to the ladder copy - + * hw_rc/done tell the pump, which applies the releases itself (on + * success or on the bounded fallback). Without defer every wave-1 + * behavior is unchanged. The request slot is stable here: the + * INFLIGHT bit is held and svc_fwd_request refuses a new request + * while a hold is active. */ if (guest) { u64 src_hpa = 0, dst_hpa = 0; @@ -5289,16 +6047,21 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; - if (resolved && - !gsp_phys_backend_guest_ce_copy(src_hpa, src_off, - dst_hpa, dst_off, - svc_fwd_req.bytes, - GSP_PHYS_BACKEND_WORK_POLL_MS)) { - svc.fwd_guest_ok++; - clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); - return; - } - if (!resolved) { + if (resolved) { + hw_rc = gsp_phys_backend_guest_ce_copy(src_hpa, src_off, + dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + if (!hw_rc) { + svc.fwd_guest_ok++; + if (!svc_fwd_req.defer) { + clear_bit_unlock(SVC_FWD_INFLIGHT, + &svc_fwd_state); + return; + } + } + } else { + hw_rc = src_rc ? src_rc : dst_rc; svc.fwd_guest_resolve_fail++; svc.fwd_guest_resolve_log++; if (svc.fwd_guest_resolve_log <= 16) @@ -5312,6 +6075,38 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.vgpu ? 1 : 0, svc_fwd_req.bytes); } + if (svc_fwd_req.defer) { + /* + * Withheld releases: publish the outcome to the pump + * and wake it; the service thread applies the recorded + * releases (success or bounded fallback) and mirrors + * the normal consume tail. + */ + if (hw_rc) { + svc.fwd_guest_fallback++; + if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: guest copy failed rc=%d, held releases will be applied by the service (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", + hw_rc, svc.fwd_guest_req, + svc.fwd_guest_ok, + svc.fwd_guest_fallback, + svc.fwd_guest_resolve_fail); + } + svc_fwd_req.hw_rc = hw_rc; + /* + * done before INFLIGHT: a reuser that acquires the bit + * can never overwrite the slot before this outcome is + * visible. + */ + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + { + struct task_struct *task = READ_ONCE(svc.task); + + if (task) + wake_up_process(task); + } + return; + } svc.fwd_guest_fallback++; if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", @@ -5349,14 +6144,18 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); -/* Called only from the service thread (the pump). */ -static void svc_fwd_request(struct nv_vgpu_device *vgpu, +/* + * Called only from the service thread (the pump). Returns true when a guest + * (hardware-copy) request was queued; the caller then decides whether to hold + * the entry for its deferred releases. All wave-1 gates are unchanged. + */ +static bool svc_fwd_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { if (!ch->engine_type) - return; + return false; if (!gsp_phys_work_forward_enabled()) - return; + return false; /* Classify this decode pass's candidate for the run summary. */ svc_fwd_cand_note(); if (!gsp_phys_backend_work_ready()) { @@ -5364,7 +6163,19 @@ static void svc_fwd_request(struct nv_vgpu_device *vgpu, if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (backend not ready) dropped=%llu\n", svc_fwd_dropped); - return; + return false; + } + /* + * P6b wave 1.5: never reuse the request slot while a hold is active - + * the held request's done/hw_rc are still being read. The caller + * applies this pass's recorded releases inline instead. + */ + if (svc_fwd_hold.active) { + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (release hold active) dropped=%llu\n", + svc_fwd_dropped); + return false; } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused) { @@ -5373,7 +6184,7 @@ static void svc_fwd_request(struct nv_vgpu_device *vgpu, if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (forwarding paused for release) dropped=%llu\n", svc_fwd_dropped); - return; + return false; } if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); @@ -5381,16 +6192,23 @@ static void svc_fwd_request(struct nv_vgpu_device *vgpu, if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (one in flight) dropped=%llu\n", svc_fwd_dropped); - return; + return false; } /* * Fill the request slot before queueing (INFLIGHT is held, so no * other writer can race it). A captured candidate runs on the guest * path only when gsp_phys_guest_mem=1; otherwise, and with no - * candidate, the request stays the unchanged ladder shadow. + * candidate, the request stays the unchanged ladder shadow. defer is + * snapshotted here (the release records are per-decode-pass state + * that the next pass resets): only a guest request whose push + * deferred releases is held. */ + svc_fwd_req.done = false; + svc_fwd_req.hw_rc = 0; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); + svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && + svc_fwd_rel_n > 0; if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; @@ -5422,14 +6240,17 @@ static void svc_fwd_request(struct nv_vgpu_device *vgpu, clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); - return; + return false; } } if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + return false; } mutex_unlock(&svc_fwd_lock); + return svc_fwd_req.guest; } /* @@ -5440,9 +6261,24 @@ static void svc_fwd_request(struct nv_vgpu_device *vgpu, */ void gsp_sema_service_work_quiesce(void) { + bool canceled; + mutex_lock(&svc_fwd_lock); svc_fwd_paused = true; - cancel_work_sync(&svc_fwd_work); + canceled = cancel_work_sync(&svc_fwd_work); + /* + * A work item canceled before it ran never clears INFLIGHT nor + * publishes its outcome: do both here so the bit is never stranded + * and a held entry resolves through the abort path instead of + * waiting out its timeout. + */ + if (canceled) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_req.hw_rc = -ECANCELED; + smp_store_release(&svc_fwd_req.done, true); + } + if (READ_ONCE(svc_fwd_hold.active)) + WRITE_ONCE(svc_fwd_hold.aborted, true); mutex_unlock(&svc_fwd_lock); } @@ -5488,6 +6324,42 @@ void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, *resolve_fail = svc.fwd_guest_resolve_fail; } +/* + * P6b wave-1.5/wave-2 deferred-release and copy counters for the + * gsp-phys-state work_guard line (see gsp_sema_service.h). Same plain-read + * discipline as the wave-1 stats above. + */ +void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, + u64 *copy_fail, u64 *deferred, + u64 *applied, u64 *fallback, + u64 *pending, u64 *overflow, + u64 *engaged, u64 *timeouts, + u64 *canceled) +{ + if (copy_skipped) + *copy_skipped = svc.fwd_copy_skipped; + if (copy_soft) + *copy_soft = svc.fwd_copy_soft; + if (copy_fail) + *copy_fail = svc.fwd_copy_fail; + if (deferred) + *deferred = svc.fwd_rel_deferred; + if (applied) + *applied = svc.fwd_rel_applied; + if (fallback) + *fallback = svc.fwd_rel_fallback; + if (pending) + *pending = svc.fwd_rel_pending; + if (overflow) + *overflow = svc.fwd_rel_overflow; + if (engaged) + *engaged = svc.fwd_hold_engaged; + if (timeouts) + *timeouts = svc.fwd_hold_timeouts; + if (canceled) + *canceled = svc.fwd_hold_canceled; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -5519,6 +6391,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_max = 0; n_gr = svc_gr_max; + /* + * P6b wave 1.5: a hold whose channel is absent from this pass's + * snapshot - or belongs to another device than the one being pumped - + * cannot be resolved through the entry loop. Once the work item has an + * outcome (or the timeout fires) resolve it through the same finish + * path, which applies the recorded releases against the hold's own + * device, then release the request slot for new requests. + */ + if (svc_fwd_hold.active && + (svc_fwd_hold.vgpu != vgpu || + svc_fwd_hold_channel_gone(local, n))) + svc_fwd_hold_gone_finish(vgpu); + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -5527,9 +6412,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u32 put = 0, get = 0, guard = 0; u32 get_before; u64 applied_before; + bool held = false; svc_notify_complete(vgpu, ch); + /* + * P6b wave 1.5: enforce the hold cap on EVERY pass for the held + * channel, before the no-GPFIFO / USERD / GP_PUT early + * `continue`s below can skip the entry loop. The check is a + * no-op unless this channel owns an expired hold. + */ + if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) + svc_fwd_hold_expire_check(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; @@ -5596,6 +6491,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u32 len, e0, e1, g; /* + * P6b wave 1.5: the held entry is resolved before + * anything else touches it - it was never consumed + * (last_get still points at it). Applied when the + * hardware copy completed, its window expired or the + * forwarding was aborted; otherwise left in place. + * svc_fwd_hold_poll mirrors the normal consume tail. + */ + if (svc_fwd_hold.active && + svc_fwd_hold.vgpu == vgpu && + ch->h_object == svc_fwd_hold.ch_handle) { + if (!svc_fwd_hold_poll(vgpu, ch, put)) + break; + /* Consumed: process the next entry. */ + continue; + } + + /* * Remember the payload this entry should deliver the * first time it is examined. If the pool advances past * it later, the release was already delivered (the @@ -5728,30 +6640,97 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->op_retry < 24) { ch->op_retry++; svc.gpfifo_nomap++; + /* + * P6b wave 1.5: + * releases recorded + * for this pass are + * applied inline: the + * entry is retried and + * is never held. P6b + * wave 2: if this pass + * withheld the software + * copy for a captured + * candidate, author it + * once first. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) { + svc_fwd_soft_copy_inline(vgpu); + svc_fwd_rel_apply_all(vgpu, ch, + svc_fwd_rel, + svc_fwd_rel_n); + } kfree(pb); break; } ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); - /* - * P5 wave 1 / P6b: - * shadow this CE push - * on the ladder - * channel, or run it - * against guest memory - * when the decode - * captured a trusted - * sysmem copy and - * gsp_phys_guest_mem=1. - * The software release - * above is already - * applied and stays - * authoritative; every - * hardware failure - * falls back to the - * ladder copy. - */ - svc_fwd_request(vgpu, ch); + { + bool fwd_guest_queued; + + /* + * P5 wave 1 / P6b: + * shadow this CE push + * on the ladder + * channel, or run it + * against guest memory + * when the decode + * captured a trusted + * sysmem copy and + * gsp_phys_guest_mem=1. + * The software copy has + * already run; the + * guest-visible release + * was applied inline + * unless wave 1.5 + * recorded it for the + * held entry below. + */ + fwd_guest_queued = + svc_fwd_request(vgpu, ch); + /* + * P6b wave 1.5: when + * this pass recorded + * releases and the + * request queued as a + * guest hardware copy, + * hold the entry: do + * not consume it, do + * not write GP_GET; + * the pump resolves it + * on a later pass. If + * any gate kept the + * request from queueing + * (or a hold is already + * active), apply the + * records now and + * continue normally. + * P6b wave 2: the + * inline apply authors + * a withheld candidate + * with one software copy + * first; a queued guest + * request leaves the + * authorship to the + * hardware (or its hold + * fallback). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) { + if (fwd_guest_queued && + !svc_fwd_hold.active) { + svc_fwd_hold_engage(vgpu, + ch, + (u32)idx); + kfree(pb); + break; + } + svc_fwd_soft_copy_inline(vgpu); + svc_fwd_rel_apply_all(vgpu, ch, + svc_fwd_rel, + svc_fwd_rel_n); + } + } /* * A CE push that should carry this * channel's next tracking release but @@ -6160,9 +7139,21 @@ if (gsp_sema_fake_rel && ch->userd_as, &g, sizeof(g)); } + /* + * P6b wave 1.5: a held channel's entry was deliberately not + * consumed. Suppress the progress publications and the stalled + * verdict for it (the work item will report the outcome); the + * state write-back below still persists the decode-time + * rel_done claim and the unchanged last_get. + */ + held = svc_fwd_hold.active && + svc_fwd_hold.vgpu == vgpu && + svc_fwd_hold.ch_handle == ch->h_object; /* Advance the client-visible completion counter. */ - svc_counter_autoreg(vgpu, ch, n_gr); - svc_counter_publish(vgpu, ch, put); + if (!held) { + svc_counter_autoreg(vgpu, ch, n_gr); + svc_counter_publish(vgpu, ch, put); + } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { @@ -6211,9 +7202,9 @@ if (gsp_sema_fake_rel && } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (ch->last_get != put) + if (!held && ch->last_get != put) stalled = true; - else if (ch->last_get != get_before && + else if (!held && ch->last_get != get_before && ch->expected_sem_payload && svc.releases_applied == applied_before) /* @@ -6844,6 +7835,26 @@ static int svc_thread(void *arg) msleep(10); } + /* + * P6b wave 1.5: never strand a held entry on the stop path - apply + * whatever is still held (best-effort, service thread) before exit. + * The backend is going away, so the hold's hardware outcome can no + * longer gate the guest release. + */ + if (svc_fwd_hold.active) { + struct nv_vgpu_device *d, *found = NULL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + svc_fwd_hold_stop_flush(found); + } + pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); complete(&svc.stopped); @@ -6920,7 +7931,15 @@ void gsp_sema_service_stop(void) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; } - cancel_work_sync(&svc_fwd_work); + /* + * A work item canceled before it ran never cleared INFLIGHT; clear + * it here so the next session is not refused by a stranded bit. + */ + if (cancel_work_sync(&svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_req.hw_rc = -ECANCELED; + smp_store_release(&svc_fwd_req.done, true); + } if (svc_fwd_wq) { destroy_workqueue(svc_fwd_wq); svc_fwd_wq = NULL; diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 9638dde..35d1c0f 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -78,9 +78,11 @@ void gsp_sema_service_work_resume(void); /* * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest - * line: guest-eligible requests seen by the pump, requests that fell back to - * the ladder copy, and verified hardware copies. Plain reads of the service - * counters; safe from the debugfs show path. + * line: guest-eligible requests seen by the pump, guest requests whose + * hardware copy failed (wave 1: fell back to the ladder copy; with + * gsp_phys_work_defer_release the service applies the held releases instead), + * and verified hardware copies. Plain reads of the service counters; safe + * from the debugfs show path. */ void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok); @@ -94,6 +96,28 @@ void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, u64 *vid_vid, u64 *resolve_fail); /* + * P6b wave-1.5/wave-2 deferred-release and copy counters for the + * gsp-phys-state work_guard line: copy_skipped (the software copy was + * withheld because hardware-primary authorship is armed; always 0 when + * gsp_phys_work_hw_primary=0), copy_soft (software fallback copies the + * service thread performed before applying releases), copy_fail (fallback + * copy attempts that failed), releases deferred until the hardware copy + * completed, holds applied on completion (including a hold whose channel + * vanished after a successful copy), holds applied because the copy failed / + * timed out / forwarding was aborted, releases currently held (gauge), + * deferred records dropped by the record bound, holds engaged, holds that hit + * the timeout, and holds dropped unresolved after their channel vanished (no + * successful copy had completed). All zero while the feature is off. Plain + * reads, safe from the debugfs show path. + */ +void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, + u64 *copy_fail, u64 *deferred, + u64 *applied, u64 *fallback, + u64 *pending, u64 *overflow, + u64 *engaged, u64 *timeouts, + u64 *canceled); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). |