diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 362 |
1 files changed, 350 insertions, 12 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 25a3cdb..51c51b2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -42,6 +42,8 @@ #include <linux/sort.h> #include <linux/bitmap.h> #include <linux/workqueue.h> +#include <linux/mm.h> +#include <linux/vmalloc.h> #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" @@ -420,6 +422,30 @@ struct sema_svc { u32 doorbell_log; /* token log budget */ bool doorbell_pending; /* pump on next service pass */ + /* + * P6b wave-1 guest forwarding (sema-service side): eligible requests + * marked guest, requests that fell back to the ladder copy, verified + * hardware copies. The booter keeps its own submit/verify/pool + * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet + * for the gsp-phys-state work_guest line. + */ + u64 fwd_guest_req; + u64 fwd_guest_fallback; + u64 fwd_guest_ok; + /* + * P6b wave-1 candidate accounting, one increment per forwarding + * request attempt once the pump is enabled: no eligible candidate was + * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail + * counts guest-work resolution failures that fell back to the ladder + * copy. All zero while the feature is off, so the summary stays silent. + */ + u64 fwd_cand_none; + u64 fwd_cand_sys_sys; + u64 fwd_cand_mixed; + u64 fwd_cand_vid_vid; + u64 fwd_guest_resolve_fail; + u64 fwd_guest_resolve_log; /* first-16 diagnostic budget */ + /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; @@ -1877,6 +1903,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop); + /* + * P6b wave 1: guest-forwarding summary. Printed whenever any counter + * moved (not only on requests) so a run that captured no candidate is + * still diagnosable from the log. + */ + if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys || + svc.fwd_cand_mixed || svc.fwd_cand_vid_vid || + svc.fwd_guest_resolve_fail) + pr_info("sema-svc: work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n", + svc.fwd_guest_req, svc.fwd_guest_ok, + svc.fwd_guest_fallback, svc.fwd_cand_none, + svc.fwd_cand_sys_sys, svc.fwd_cand_mixed, + svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -2158,6 +2197,30 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, return 0; } +/* + * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the + * decoded push whose operands the software path resolved successfully. + * Captured by svc_copy() while the push's work runs, so the candidate is + * exactly the copy the software path executed; one capture per decode pass, + * consumed by svc_fwd_request() on the same service thread. Each operand + * records its aperture kind and the resolver's page/offset: SYS holds a + * page-aligned guest GPA, VID holds a page-aligned byte offset into the + * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into + * an HPA through the guest-RAM backing file and a VID operand into an HPA + * through the vmalloc backing of the shadow; no address is ever guessed. + * Service-thread only, like the rest of the pump state. + */ +struct svc_fwd_cand { + bool valid; + u8 src_kind; /* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */ + u8 dst_kind; + u64 src_page, dst_page; /* resolver page: GPA / shadow byte offset */ + u32 src_off, dst_off; /* in-page offsets */ + u32 bytes; +}; + +static struct svc_fwd_cand svc_fwd_cand; + /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) @@ -2200,6 +2263,51 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } /* + * P6b wave 1: the software path has now resolved both operands, so + * this successful CE copy is the guest-forwarding candidate. Physical + * operands are the norm here (the guest pushes raw vidmem offsets as + * physical destinations, with virtual sources); after the block above, + * a virtual side already holds its resolved pa + aperture, while a + * physical side carries its raw address + the push's aperture, so the + * same page/offset/kind handling covers both. Eligibility stays + * conservative: one page per side, 4-aligned, 1..4096 bytes, and no + * same-page overlap within one aperture; anything doubtful is left to + * the ladder submission. Only the first eligible copy of the push is + * kept (svc_fwd_cand is reset per decode pass). + */ + if (!svc_fwd_cand.valid && + total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) { + u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1)); + u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1)); + u8 src_kind = (src_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* + * Same-domain operands inside the same page must not overlap: + * the engine's copy direction would be undefined. Different + * domains (SYS vs VID) are disjoint by construction. + */ + bool overlap = src_kind == dst_kind && src_page == dst_page && + src_off < dst_off + total && + dst_off < src_off + total; + + if (!overlap && src_off + total <= SEMA_PAGE_SIZE && + dst_off + total <= SEMA_PAGE_SIZE) { + svc_fwd_cand.valid = true; + svc_fwd_cand.src_kind = src_kind; + svc_fwd_cand.dst_kind = dst_kind; + svc_fwd_cand.src_page = src_page; + svc_fwd_cand.dst_page = dst_page; + svc_fwd_cand.src_off = src_off; + svc_fwd_cand.dst_off = dst_off; + svc_fwd_cand.bytes = (u32)total; + } + } + + /* * Data-path watch: for copies of exactly sema_copy_watch bytes, log * the first 8 source and destination bytes so a broken link in the * chain (wrong source page, wrong destination, silent wipe) is @@ -2826,6 +2934,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } memset(&st, 0, sizeof(st)); + /* + * P6b wave 1: clear the guest-forwarding candidate at the start of + * every decode pass; the first eligible copy captured below is the one + * svc_fwd_request() sees when this pass ends. + */ + svc_fwd_cand.valid = false; svc.pages_decoded++; if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; @@ -5020,7 +5134,8 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) } /* ------------------------------------------------------------------ */ -/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1) */ +/* P5/P6b wave-1 shadow work forwarding */ +/* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data) */ /* ------------------------------------------------------------------ */ /* @@ -5029,12 +5144,46 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) * request that arrives while one is in flight (or before the backend/ladder * is ready) is dropped and counted. The work item runs the ladder's * module-owned copy and observes the real semaphore; the software release - * path is unchanged and stays authoritative. No guest VA or guest PA ever - * reaches the engine in this wave. + * path is unchanged and stays authoritative. + * + * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual + * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1, + * the work item instead resolves each operand to a host PA and runs one real + * CE copy against guest memory or the emulated vidmem shadow on a fresh + * scratch-PT line: a SYS operand resolves through the guest-RAM backing file, + * a VID operand through the vmalloc backing of BAR1. Any failure (translation, + * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout, + * verify) falls back to the unchanged ladder copy, so the software pump is + * never gated by the hardware path. Shadow-mode limit: the software copy + * already applied the same bytes, so hardware execution is proven by + * acceptance of the translated-HPA submission plus the semaphore release and + * the direct-map consistency check; a late hardware write can theoretically + * race a guest reuse of the page. The window is one ordered workqueue hop plus + * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary. */ #define SVC_FWD_INFLIGHT 0 #define SVC_FWD_LOG_MASK 63 +/* + * One pending request, written by the service thread under svc_fwd_lock + * before queue_work and read by the work item without that lock (quiesce + * holds that lock across cancel_work_sync, so the work item must never take + * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the + * slot write before the work item runs. vgpu is carried for the VID operand + * math only; it is stable because the request is single-inflight and quiesce + * quiesces the work item before the backend/device state is released. + */ +struct svc_fwd_req { + bool guest; + struct nv_vgpu_device *vgpu; + u8 src_kind, dst_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ + u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ + u32 src_off, dst_off; + u32 bytes; +}; + +static struct svc_fwd_req svc_fwd_req; + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -5049,14 +5198,129 @@ static u64 svc_fwd_dropped; static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; +/* + * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve + * the page-aligned GPA through the guest-RAM backing file; VID operands + * resolve the shadow byte offset through the vmalloc backing of BAR1 (the + * same buffer the AP helpers use). The resolver's in-page part, if any, is + * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind, + * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT + * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno. + * The caller falls back to the ladder copy on any failure. + */ +static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, + u64 pa, u32 off, u32 bytes, u64 *hpa, + u32 *out_off) +{ + u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off; + int ret; + + if (kind == SVC_APERTURE_FB) { + u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL; + struct page *pg; + + if (!base || page > vgpu->bars[1].size || + (u64)page_off + bytes > vgpu->bars[1].size - page) + return -ERANGE; + pg = vmalloc_to_page(base + page); + if (!pg) + return -EFAULT; + *hpa = (u64)page_to_phys(pg); + *out_off = page_off; + return 0; + } + if (kind != SVC_APERTURE_SYS) + return -EINVAL; + ret = gsp_phys_gpa_to_hpa(page, hpa); + if (ret) + return ret; + *out_off = page_off; + return 0; +} + +/* Classify the pass's captured candidate into the forwarding counters. */ +static void svc_fwd_cand_note(void) +{ + if (!svc_fwd_cand.valid) { + svc.fwd_cand_none++; + return; + } + if (svc_fwd_cand.src_kind == SVC_APERTURE_FB && + svc_fwd_cand.dst_kind == SVC_APERTURE_FB) + svc.fwd_cand_vid_vid++; + else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS && + svc_fwd_cand.dst_kind == SVC_APERTURE_SYS) + svc.fwd_cand_sys_sys++; + else + svc.fwd_cand_mixed++; +} + static void svc_fwd_work_fn(struct work_struct *work) { + bool guest = svc_fwd_req.guest; u64 src_va = 0, dst_va = 0, sem_va = 0; u32 bytes = 0; unsigned long start = jiffies; int ret; /* + * P6b wave 1: try the guest hardware copy first when the pump marked + * this request guest. Every failure falls through to the unchanged + * ladder submission below. The request slot is stable here: the + * INFLIGHT bit is held, so only this reader touches it. + */ + if (guest) { + u64 src_hpa = 0, dst_hpa = 0; + u32 src_off = 0, dst_off = 0; + int src_rc, dst_rc; + bool resolved; + + src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.src_kind, + svc_fwd_req.src_page, + svc_fwd_req.src_off, + svc_fwd_req.bytes, &src_hpa, + &src_off); + dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, + svc_fwd_req.dst_kind, + svc_fwd_req.dst_page, + svc_fwd_req.dst_off, + svc_fwd_req.bytes, &dst_hpa, + &dst_off); + resolved = !src_rc && !dst_rc; + if (resolved && + !gsp_phys_backend_guest_ce_copy(src_hpa, src_off, + dst_hpa, dst_off, + svc_fwd_req.bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS)) { + svc.fwd_guest_ok++; + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + return; + } + if (!resolved) { + svc.fwd_guest_resolve_fail++; + svc.fwd_guest_resolve_log++; + if (svc.fwd_guest_resolve_log <= 16) + pr_info("gsp-phys-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n", + svc_fwd_req.src_kind, + svc_fwd_req.src_page, + svc_fwd_req.src_off, src_rc, + svc_fwd_req.dst_kind, + svc_fwd_req.dst_page, + svc_fwd_req.dst_off, dst_rc, + svc_fwd_req.vgpu ? 1 : 0, + svc_fwd_req.bytes); + } + svc.fwd_guest_fallback++; + if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", + svc.fwd_guest_req, svc.fwd_guest_ok, + svc.fwd_guest_fallback, + svc.fwd_guest_resolve_fail); + } + + /* * Re-resolve the ladder operands on the work item: the backend may * have gone away between the pump's gate and this run. A false return * means no module buffer is usable; drop without touching anything. @@ -5086,12 +5350,15 @@ static void svc_fwd_work_fn(struct work_struct *work) static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* Called only from the service thread (the pump). */ -static void svc_fwd_request(const struct svc_channel *ch) +static void svc_fwd_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) { if (!ch->engine_type) return; if (!gsp_phys_work_forward_enabled()) return; + /* Classify this decode pass's candidate for the run summary. */ + svc_fwd_cand_note(); if (!gsp_phys_backend_work_ready()) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) @@ -5116,6 +5383,34 @@ static void svc_fwd_request(const struct svc_channel *ch) svc_fwd_dropped); return; } + /* + * Fill the request slot before queueing (INFLIGHT is held, so no + * other writer can race it). A captured candidate runs on the guest + * path only when gsp_phys_guest_mem=1; otherwise, and with no + * candidate, the request stays the unchanged ladder shadow. + */ + svc_fwd_req.guest = svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled(); + if (svc_fwd_req.guest) { + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.src_kind = svc_fwd_cand.src_kind; + svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind; + svc_fwd_req.src_page = svc_fwd_cand.src_page; + svc_fwd_req.dst_page = svc_fwd_cand.dst_page; + svc_fwd_req.src_off = svc_fwd_cand.src_off; + svc_fwd_req.dst_off = svc_fwd_cand.dst_off; + svc_fwd_req.bytes = svc_fwd_cand.bytes; + svc.fwd_guest_req++; + } else { + svc_fwd_req.vgpu = NULL; + svc_fwd_req.src_kind = 0; + svc_fwd_req.dst_kind = 0; + svc_fwd_req.src_page = 0; + svc_fwd_req.dst_page = 0; + svc_fwd_req.src_off = 0; + svc_fwd_req.dst_off = 0; + svc_fwd_req.bytes = 0; + } svc_fwd_requests++; if (!svc_fwd_wq) { /* @@ -5158,6 +5453,41 @@ void gsp_sema_service_work_resume(void) mutex_unlock(&svc_fwd_lock); } +/* + * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest + * line (see gsp_sema_service.h). Plain reads: the service thread and the + * work item are the only writers and the values are monotonic. + */ +void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok) +{ + if (req) + *req = svc.fwd_guest_req; + if (fallback) + *fallback = svc.fwd_guest_fallback; + if (ok) + *ok = svc.fwd_guest_ok; +} + +/* + * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request + * classification of the captured operand mix plus guest-work resolution + * failures. Same plain-read discipline as the stats above. + */ +void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, + u64 *vid_vid, u64 *resolve_fail) +{ + if (none) + *none = svc.fwd_cand_none; + if (sys_sys) + *sys_sys = svc.fwd_cand_sys_sys; + if (mixed) + *mixed = svc.fwd_cand_mixed; + if (vid_vid) + *vid_vid = svc.fwd_cand_vid_vid; + if (resolve_fail) + *resolve_fail = svc.fwd_guest_resolve_fail; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -5404,16 +5734,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); /* - * P5 wave 1: shadow this - * CE push on the ladder - * channel. The software - * release above is already + * P5 wave 1 / P6b: + * shadow this CE push + * on the ladder + * channel, or run it + * against guest memory + * when the decode + * captured a trusted + * sysmem copy and + * gsp_phys_guest_mem=1. + * The software release + * above is already * applied and stays - * authoritative; the work - * item only observes the - * real semaphore. + * authoritative; every + * hardware failure + * falls back to the + * ladder copy. */ - svc_fwd_request(ch); + svc_fwd_request(vgpu, ch); /* * A CE push that should carry this * channel's next tracking release but |