diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 3924 |
1 files changed, 3805 insertions, 119 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 993e507..c93ecc9 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -205,6 +205,18 @@ struct svc_channel { u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ + u32 h_vaspace; /* r835: the client VAS this channel was + * allocated in (NV_CHANNEL_ALLOC_PARAMS. + * hVASpace). Channel-scoped VA resolution + * consults this VAS's page tables FIRST: + * every client's RM window sits at the same + * VAs (0x1200xxxxx) and the global VA-keyed + * maps conflate them - the X server's ring + * VA resolved through the GL client's page + * tables (walk-probe vas h=0xcaf00005 + * answered 0x120090000) and its push stream + * decoded as zero pages, wedging X the same + * way the RC watchdog's ring was starved. */ u8 ce_index; /* rank among the live CE channels (creation order) */ u8 gr_index; /* rank among the live GR channels (creation order) */ u64 gpfifo_va; @@ -506,6 +518,20 @@ struct svc_channel { */ u8 userd_scans; /* + * r828: the RC watchdog's notifier VA, captured from the class2d + * SET_NOTIFY_A/B methods on subchannel 3 and zeroed at the NOTIFY + * (see svc_notify2d_write). Survives the pass snapshot via the + * write-back list. + */ + u64 notify2d_va; + /* r829: bounded count of zero-entry (bad-translation) holds; see the + * zero-entry hold in svc_pump_channels. Persists via the write-back. */ + u32 zero_held; + /* r831: the registered userd value is a real guest PA (the driver's + * own declaration; e.g. the RC watchdog channel). The ring auto-pin's + * ring+0x2000 convention must not overwrite it. */ + bool userd_reg; + /* * r471: the completion-counter publish must never advance the * guest's word past a push whose content was still unwritten when * the entry was consumed (the streams_events staging-reuse @@ -652,6 +678,21 @@ static u32 svc_ring_n; */ static u8 svc_sess_flush; static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */ +static u8 svc_sess_flush_forced; /* execute the next flush past the live-channel skip + * (ISSUE-0002 client-session boundary aid; see + * gsp_sema_service_sess_flush_force) */ +/* + * SESSION 148 perf tracing: sema_trace_h selects one channel handle; the + * pump then logs a compact state line for it once a second, and + * svc_counter_publish logs why it skipped (last_get gate / arming gate) or + * the value it wrote. This is the instrument for locating the 22.4 s/frame + * glxgears pacer without guessing at windows. + */ +unsigned long long gsp_sema_trace_h; +module_param_named(sema_trace_h, gsp_sema_trace_h, ullong, 0644); +MODULE_PARM_DESC(sema_trace_h, + "Trace one channel handle (0x...; 0=off): 1/s state + publish gates"); +static unsigned long svc_perf_last; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static u32 svc_sess_gen; /* m176: bumped by every executed session flush; * stamps channels/scrubbers so per-session learned @@ -786,6 +827,12 @@ static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ +/* SESSION 133: ranks that already consumed their one shadow ring scan. */ +static unsigned long svc_gr_ring_scan_done; +static unsigned long svc_gr_ring_scan_last; /* rate limit for the scan */ +/* SESSION 138: bounded witness for the GR request builder's page list and + * the fill's candidate set (one launch's wiring printed per budget slice). */ +static u32 svc_gr_pages_log; static bool svc_pb_first_push(const u8 *buf) { @@ -958,6 +1005,8 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 rescue_map_ok; /* SESSION 148: content-verified + * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pool_alt_applied; /* pool releases fanned to a * displaced pin (ISSUE-0002) */ @@ -1136,6 +1185,17 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u32 fwd_gr_nocand_log; /* SESSION 127: first-N + * request-without-candidate + * witnesses */ + u64 fwd_gr_nordy; /* SESSION 127: backend-not- + * ready request refusals */ + u64 fwd_gr_holdbusy;/* SESSION 127: hold-active + * request busy verdicts */ + u64 fwd_gr_inflight;/* SESSION 127: inflight-bit + * request busy verdicts */ + u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill + * ENOENT page witnesses */ u64 fwd_gr_defer; /* m150: held for data-arg retry */ u64 fwd_gr_dstpend; /* r541: held for an unexecuted * device write on a data page */ @@ -1541,10 +1601,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; + ch->h_vaspace = desc->h_vaspace; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; + /* + * r831: a registered userd value that looks like a real guest + * PA (the RC watchdog's channel carries its own userd block) + * is the driver's own declaration and outranks the ring+0x2000 + * convention guess the ring auto-pin applies (that guess is + * right for the CUDA GR layout, where the registered value is + * pool-relative - observed 0x10000). Tiny registered values + * stay guesses. + */ + ch->userd_reg = desc->userd_pa >= 0x10000000ULL; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as @@ -1586,12 +1657,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * channel; the stashed geometry survives, so the scrub path * does not depend on the channel-table identity at all. */ - if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + /* + * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry + * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also + * matched every client channel (this driver version: 32768 + * entries, not the 1024 the comment below assumed), so the + * single scrub slot bound to whichever client channel + * registered last (observed r701: the slot held h=0xbeee0102 + * userd=0x8f400/entries=32768 and the pass authored put=0 for + * the wrong channel while the real scrubber's payload stayed + * zero - the vkAllocateMemory -2 gate). + */ + if (desc->gpfifo_entries == 4096 && desc->gpfifo_va && desc->userd_pa) { struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); if (!s) goto scrub_done; /* table full: max guests */ + pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n", + (void *)desc->vgpu, (int)(s - svc_scrubs), + desc->h_object, desc->gpfifo_entries); /* Re-seed the counting on a fresh scrubber object or a * new client session (m176: the counter/submitted state * is per-session; a recycled scrubber handle after a @@ -1616,6 +1701,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, s->userd_as); + } else if (desc->gpfifo_entries >= 4096 && + desc->gpfifo_entries != 32768) { + /* m112b: a non-32768 channel with >=4096 entries that + * fails the stash condition (missing VA or USERD PA) + * is a scrubber candidate the service cannot author + * for - log it so a silent no-authoring boot is + * diagnosable. */ + pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n", + desc->h_object, desc->gpfifo_entries, + desc->gpfifo_va, desc->userd_pa); } scrub_done: pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", @@ -1662,6 +1757,8 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump + * (sema_dbg only, 4 per boot) */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ @@ -1814,6 +1911,64 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, * the lesser evil; it is logged loudly for the record. */ #define SVC_VAS_CAND_AGE_MS 15000u +/* + * r705h: the candidate age-out deadline is runtime-tunable. The parked + * COPY_SERVER_PDES candidate for the GL/Vulkan client ages out (UNVERIFIED) + * at exactly the moment the client's channels first register, so the + * content-verified walk can never anchor it; raising the deadline keeps the + * candidate parked until a host tool supplies a verified ring pair via + * sema_pdb_discover. 0 disables the age-out entirely. + */ +static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS; +module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644); + +/* + * SESSION 127 C5C0 translator gate. When 1, the sweep additionally + * translates continuation entries into launch work (tail-aware method + * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5 + * copy-out capture, candidate-owned release withholding). + * + * SESSION 130 re-test of the r727 "translator regressions" (translator on, + * utility sweep OFF, r746): victory_saxpy and victory_math now PASS with + * checksums identical to the gate-off baseline (saxpy + * csum=981440890.500000), so the saxpy/math half of the r727 regression + * is FIXED (the SESSION 129/130 sweep changes); victory_mail PASSES. The + * remaining xlate-on regression is victory_multikernel: its launch is + * captured and submitted to the physical channel but the forward times + * out repeatedly (`GR LAUNCH FAILED: timeout after 2000 ms + * (qmd_rel=0xc0ffee01 want=0x600df00d ...) h=0x5c000019 + * push=0x20043c394`, `hold active`), so the client hangs - the default + * stays 0 until that forward path is fixed. The UTILITY SWEEP + * (`sema_gr_util_scan`) is the OTHER regression source: at 24 it broke + * saxpy (`bad=1047527`) and hung math on r745, so it stays opt-in + * (default 0) until its channel reads prefer the direct walk over pins. + */ +static bool gsp_sema_gr_xlate; +module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gr_xlate, + "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)"); +/* + * SESSION 128 utility-channel inline-download materialization + * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload + * channels (the MEM2MEM/A140 + 2D utilities, registered but with an + * unresolved USERD so the pump's put-bounded entry loop never decodes + * them) carry the engine-side code/data downloads: the + * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180.. + * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose + * payload the engine fetches from the next GPFIFO entry (the + * source-view call). Without this pass those destinations stay zero in + * the vidmem shadow - observed: the Vulkan client's code-heap/SASS + * upload (12 KB) and command-stream staging (256 B) never land, so the + * forwarded launch has no program content. Bounded: scan up to N + * entries per channel per pass, materialize once-only keyed by + * (push_va, content hash, dst, bytes) via the same + * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate. + */ +static u32 gsp_sema_gr_util_scan; +module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644); +MODULE_PARM_DESC(sema_gr_util_scan, + "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)"); +MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)"); static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) { @@ -1824,8 +1979,9 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) if (!svc_vas_cand[i].valid) continue; - if (time_before(jiffies, svc_vas_cand[i].stamp + - msecs_to_jiffies(SVC_VAS_CAND_AGE_MS))) + if (!svc_vas_cand_age_ms || + time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(svc_vas_cand_age_ms))) continue; memset(&d, 0, sizeof(d)); d.h_vaspace = svc_vas_cand[i].h_vaspace; @@ -1833,7 +1989,7 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) d.num_entries = svc_vas_cand[i].entries; d.b_vidmem = svc_vas_cand[i].vidmem; pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", - d.h_vaspace, SVC_VAS_CAND_AGE_MS, d.pdb, + d.h_vaspace, svc_vas_cand_age_ms, d.pdb, d.num_entries); gsp_sema_service_vaspace_add(&d); svc_vas_cand[i].valid = false; @@ -2651,6 +2807,139 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } /* + * r820: driver-declared kernel-buffer translations (PROMOTE_CTX). + * + * The guest RM announces the physical backing of a channel's context + * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries + * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are + * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker + * segment, the FECS/patch/access-map blocks - so the userspace pin + * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs) + * and the page-table walk can answer with a stale/wrong page. Observed + * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA + * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the + * driver's own promote entry said pa=0x11e120000. The NOP sanity push of + * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and + * the X server died with "Failed to initialize DMA". + * + * The registry is a small table keyed by the entry VA, newest wins: a + * client that re-promotes the same buffer - observed between two Xorg + * starts, the same channel object announced a NEW pa for the same VA - + * replaces the old record, so a stale backing is never preferred over + * the live statement. VA-only bookings (bInitialize=0 re-promotes of + * already-initialized buffers, zero PA/size) and bNonmapped entries + * (va=0) carry no backing and must not erase the last statement. + * + * Lookups run before the learned cache in svc_lookup_trusted: the wrong + * r817 walk answer had been cached in the learned table (svc_lat_store + * after every walk success), so a promote hit must both pre-empt and + * overwrite that entry; the hit re-stores the corrected page into the + * learned table so the direct svc_lat_lookup callers heal too. + */ +#define SVC_PROMO_MAX 256 +struct svc_promo_rec { + u64 va; + u64 pa; + u64 size; + unsigned long seq; + u32 phys_attr; + u16 buffer_id; + bool valid; +}; +static struct svc_promo_rec svc_promo[SVC_PROMO_MAX]; +static unsigned long svc_promo_seq; +static DEFINE_SPINLOCK(svc_promo_lock); +static u32 svc_promo_hit_log; /* bounded "answered from PROMOTE_CTX" witness */ + +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id) +{ + struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + if (!va || !pa || !size) + return; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + if (svc_promo[i].valid && svc_promo[i].va == va) { + rec = &svc_promo[i]; + break; + } + if (!svc_promo[i].valid && !free_rec) + free_rec = &svc_promo[i]; + if (svc_promo[i].valid && + (!oldest || svc_promo[i].seq < oldest->seq)) + oldest = &svc_promo[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (!rec) { + spin_unlock_irqrestore(&svc_promo_lock, flags); + return; + } + rec->va = va; + rec->pa = pa; + rec->size = size; + rec->phys_attr = phys_attr; + rec->buffer_id = (u16)buffer_id; + rec->seq = ++svc_promo_seq; + rec->valid = true; + spin_unlock_irqrestore(&svc_promo_lock, flags); +} + +static int svc_promo_lookup(u64 va, u64 *addr, int *ap) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + const struct svc_promo_rec *rec = &svc_promo[i]; + + /* + * r823: answer ONLY the base page. The promoted (va, size) + * pair is a VA-space booking, not a contiguous mapping: the + * ctx MAIN buffer's span [0x120020000, 0x120153000) contains + * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and + * the range arithmetic answered such an interior VA with + * MAIN.pa + offset - a zero page (observed r823: the GPFIFO + * VA resolved to 0x11ce70000, the ring-entry validation read + * va=0x0 len=0 and the latch never fired). Only the page at + * the declared VA has a proven claim to the declared PA; the + * pushbuffer's NOP sits exactly there (nvPushHeader writes at + * the segment base). + */ + if (!rec->valid || va < rec->va || + va - rec->va >= SEMA_PAGE_SIZE) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + *addr = rec->pa + (va - rec->va); + /* + * physAttr 0x4 was observed only on vidmem entries (pa in + * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on + * system memory. Unknown values fall to SYS, whose read + * guard (pa < SEMA_PA_MAX) still rejects nonsense. + */ + *ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + rc = 0; + } + spin_unlock_irqrestore(&svc_promo_lock, flags); + if (!rc && svc_promo_hit_log < 64) { + svc_promo_hit_log++; + pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n", + va, *addr, *ap); + } + return rc; +} + +/* * VA shadow: page-granular backing for device VAs whose VA->PA translation * cannot be derived from the guest's page tables. * @@ -2984,6 +3273,52 @@ struct svc_ipin_alt { static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ static u32 svc_pushalt_log; +/* r707x: per-client push-read attempt witness (pump read site). */ +static u32 svc_rdw_log; +/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd). + * GR channels only and capped: a hot CE channel re-running this every + * ~16 ms burned the whole budget before the channel under diagnosis ever + * appeared. Each scan reports its decision, a scan that found nothing + * reportable logs a sweep summary. */ +static u32 svc_userdp_log; +/* r829: bounded witness for the zero-entry holds (bad ring translations). */ +static u32 svc_zero_entry_log; +static u8 svc_read_last_src; +/* r715: PA the winning source of the last push read resolved to. */ +static u64 svc_read_last_pa; +/* + * r820: set when the last svc_read_va_push_ex chunk was answered by a + * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to + * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes + * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the + * whole small push is zeros - unparseable as a method stream, but exactly + * right from a translation the driver itself declared. + */ +static bool svc_read_last_promote; +/* r715: per-client push-decode branch witness (pump read site). */ +static u32 svc_pushdec_log; +/* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ +static u32 svc_pushfail_log; +/* + * SESSION 132: the util-sweep witnesses are per-CLIENT budgets (reset in + * svc_session_flush), not per-boot: on a multi-client boot the driver-init + * reads of an earlier client exhausted the boot budget and hid the later + * clients' rows (the same fix the r707x/r715 pump witnesses got). + */ +static u32 svc_util_dec_log; +static u32 svc_gr_util_log; +static struct { u64 va; u64 h; } svc_util_dec_keys[32]; +/* + * SESSION 143 (r803): per-channel sweep VISIBILITY. The r802 evidence could + * not say whether the Vulkan client's copy-class channels (0xbeef0101/2, + * eng 11/12) were swept at all: the per-pass log capped at 24 rows and the + * early return (no gpfifo/entries) prints nothing. These one-shot (per + * client, reset in svc_session_flush) keys name the first sweep of EVERY + * channel - begin or skip + reason - so "channel absent from the log" can + * never again be confused with "channel not in the table". + */ +static u32 svc_util_seen_keys[32]; +static u32 svc_util_seen_n; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -3265,11 +3600,56 @@ static bool svc_resolve_in_progress(u64 va) * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ +/* + * r835: the VAS handle of the channel currently being serviced (0 = none). + * Channel-scoped VA resolution consults this VAS's page tables FIRST: the + * global pin/lat/map layers are VA-keyed and every client's RM window sits + * at the same VAs (0x1200xxxxx), so a lookup outside the owning client's + * page tables can answer ANOTHER client's page - the X server's ring + * resolved to the GL client's (zero) page and its pushes decoded empty, + * wedging X's readback wait exactly like the RC watchdog's ring starvation. + */ +static u32 svc_cur_vas; + +static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + int i; + + if (!svc_cur_vas) + return -ENOENT; + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 pa = 0; + int a = SVC_APERTURE_SYS; + + if (!svc.vas[i].valid || + svc.vas[i].h_vaspace != svc_cur_vas) + continue; + if (svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, + va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc.vas[i].ap, &pa, &a, false)) + continue; + *addr = pa + (va & (SEMA_PAGE_SIZE - 1)); + *ap = a; + return 0; + } + return -ENOENT; +} + static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; + /* + * r835: the decoded channel's own page tables answer first. When a + * channel is being serviced (svc_cur_vas set) its VAS is the only + * authoritative source for its VAs; the VA-keyed layers below may + * hold another client's pages for the same numeric VA. A failed + * walk falls through to the old chain unchanged. + */ + if (!svc_chanvas_lookup(vgpu, va, addr, ap)) + return 0; if (!svc_pin_lookup_any(va, addr, ap)) { return 0; } @@ -3313,6 +3693,23 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } + /* + * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not + * the first. The promoted (va, pa) pair names a BOOKING, and even its + * base page can belong to a different buffer than the one mapped at + * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA; + * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's + * separately mapped 4 KB progress semaphore - writing the release to + * the booking's pa left the driver's own polled page at zero and the + * X server failed at its 10 s wait while the release value sat in the + * wrong page). The page-table walks describe the real mappings and + * must answer first; the promote record is only a fallback for VAs + * the walks cannot resolve at all. + */ + if (!svc_promo_lookup(va, addr, ap)) { + svc_lat_store(va, *addr, *ap); + return 0; + } return -ENOENT; } @@ -6330,10 +6727,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); - if (push && push_len >= 16) { + if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; + svc_gr_push_dump++; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { @@ -6401,6 +6799,31 @@ static bool svc_fwd_defer_arm(void) gsp_phys_work_defer_release_enabled(); } +/* + * r824: the release deferral must be owned by a candidate of THIS push. + * svc_fwd_defer_arm() answers globally (any unoffered candidate of the + * boot's ring); an unrelated lingering candidate then parked the release + * of a push that has no forwarding at all. Observed: the X/GL DMA test's + * progress-tracker release (the kernel's progress semaphore) was recorded + * into svc_fwd_rel and dropped at the next pass's reset, every pass, so + * the semaphore never advanced and Xorg failed at its 10 s wait - while + * every other stage of the chain (latch, NOP acceptance, decode) had + * just started working. A release whose push owns no candidate cannot + * wait for a forwarded copy that will never exist; it applies inline. + */ +static bool svc_fwd_cand_owns_push(u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + /* * P6b wave-1.5 deferred guest releases: every release group the current @@ -6954,6 +7377,20 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 *done_out, u32 rows, u32 dpitch, u32 spitch, u32 owner_ch, u32 owner_idx, u64 owner_hash) { + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. SESSION 132: one record per COPY OP, + * not per row - a 480-row frame copy logged 480 rows and exhausted + * the per-session budget (128) inside the first op, hiding every + * later copy (including any SASS upload). + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu rows=%u dpitch=0x%x spitch=0x%x dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, rows, dpitch, spitch, + dst_phys, dst_aperture, src_phys, src_aperture); + } if (rows > 1 && dpitch) { u64 done = 0; int rc = 0; @@ -7003,17 +7440,6 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; - /* - * fail4: name every copy the executor is asked to run (pre- - * resolution), so a copy that fails or is held before any witness - * still has an entry record. - */ - if (gsp_sema_dbg && svc.copyent_log < 128) { - svc.copyent_log++; - pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", - svc.cur_ch, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture); - } snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, owner_idx, owner_hash); @@ -8848,18 +9274,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len) memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* - * UVM's large page-table pushes begin with a - * zero-filled header region (observed 0x1004 - * bytes) before the first method, so zeros before - * the first method are leading padding; zeros - * after one are trailing padding and end the - * stream. + * NVC06F_DMA_NOP is 0x00000000: a zero word is + * NOP padding. UVM's large page-table pushes begin + * with a zero-filled header region (observed 0x1004 + * bytes) and the GR submit pushes carry zero gaps + * between method groups (observed r715: the 1432- + * and 17316-byte pushes pad between the context + * setup and the dispatch groups), so zeros are + * skipped anywhere in the stream, not only before + * the first method or at the tail. The exact-end + * requirement and the valid-header walk below are + * the false-positive arbiters. */ - if (!methods) { - off += 4; - continue; - } - break; /* trailing zero padding */ + off += 4; + continue; } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; @@ -8879,16 +9307,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len) return false; off += 4 + count * 4; break; + case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */ + case 2: /* GRP2_USE_TERT */ case 4: /* IMMD_DATA_METHOD: header only */ - off += 4; - break; case 7: /* END_PB_SEGMENT: header only */ + /* + * r715: the class header defines op 0 as + * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT - + * 4-byte header-only formats. The GR submit + * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK, + * value 0xfff) around their semaphore groups; the + * old parser rejected the first one on every big + * push, so the whole push read implausible and + * the entries were skipped without decode + * (session 124's consume-without-decode). + */ off += 4; break; default: return false; } - if (++methods > 1024) + if (++methods > 8192) return false; } while (off < len && buf[off] == 0) @@ -8947,6 +9386,127 @@ static unsigned long svc_put_log_at[32]; */ static u32 svc_userd_dbg; static u32 svc_userd_dbg_ch[32]; +/* SESSION 134: per-CE-channel budget of the first-submission ring captures + * (see svc_ce_firstsub_capture) and the start of an unlatched CE channel's + * completion hold (see svc_ce_hold_ms). Session state; reset by + * svc_session_flush. */ +static u32 svc_ce_firstsub_n[32]; +static unsigned long svc_ce_hold_since[32]; +/* + * SESSION 134c: the FB offset each CE rank's first-submission capture latched + * (0 = none). The shadow scan is rank-blind on its own - it takes the first + * ring-shaped page in the shadow - and on r779 that latched rank 0's block + * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported + * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was + * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from + * rank 0's applied release), so the client's rank-1 tracker wait never + * completed and the client spun forever (victory_mail stall, three boots). + * The capture now prefers the rank's convention offset and refuses any + * offset another CE rank already latched. Session state; reset by + * svc_session_flush. + */ +static u64 svc_ce_firstsub_off[32]; +/* + * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR + * autopin fallback scan (SESSION 133c) and the CE first-submission + * capture (SESSION 134c) each solved the same problem only within their + * own class; a live-but-foreign ring block passes every shape gate, so a + * channel whose own ring is not written yet latches another channel's + * block permanently and never reads its own entries. Observed on + * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE + * rank-0 block (fb 0x80000, userd 0x90000; e.g. + * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape + * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE + * handle decoded the identical entry[35] raw words from pa=0x10350641c), + * so no GR launch was ever captured and victory_mail hung after DEVICE. + * A channel that cannot see its own block must stay unlatched (and retry + * next pass) rather than steal a foreign one. Keyed by the FB ring + * offset; claims are session state, reset by svc_session_flush. + */ +struct svc_fb_claim { + u32 h; /* owning channel handle */ + u64 ring_off; /* FB ring offset */ + u64 userd_off; /* FB USERD offset (0 = unknown) */ +}; +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL +static struct svc_fb_claim svc_fb_claim_tab[40]; +static u32 svc_fb_claim_n; + +static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off) +{ + u32 i; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == h) { + svc_fb_claim_tab[i].ring_off = ring_off; + svc_fb_claim_tab[i].userd_off = userd_off; + return; + } + } + if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) { + svc_fb_claim_tab[svc_fb_claim_n].h = h; + svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off; + svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off; + svc_fb_claim_n++; + } +} + +/* + * true = the candidate FB block (ring at ring_off, USERD at userd_off if + * known) is already claimed by a DIFFERENT channel, or is a live CE + * rank's convention block even before that rank latched, i.e. it must not + * be latched or stolen. Same-channel re-latches pass. + */ +static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h, + u64 ring_off, u64 userd_off) +{ + u32 i; + unsigned long flags; + int k; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == self_h) + continue; + if (svc_fb_claim_tab[i].ring_off == ring_off) + return true; + if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off) + return true; + } + /* + * SESSION 144b: reserve every live CE rank's convention block even + * before that rank has latched. r809: CE rank 1's block at fb + * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran + * (rank 1 latched through the plain autopin path, not the + * first-submission capture) and the GR channel stole it again. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) { + u64 conv; + + if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu || + !svc.chan[k].engine_type || + svc.chan[k].h_object == self_h || + svc.chan[k].ce_index >= 32) + continue; + conv = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index; + if (ring_off == conv || + (userd_off && userd_off == conv + 0x10000)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return true; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +static void svc_fb_claim_reset(void) +{ + svc_fb_claim_n = 0; + memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab)); +} + /* * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD * probe fails at discovery. Session/reset-scoped, scheduled on the first @@ -10014,8 +10574,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (st->qmd_n > svc_gr_qmd_seen) svc_gr_qmd_seen = st->qmd_n; - if (st->qmd_have == 3) - st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + /* + * SESSION 127 C5C0 translator: the Vulkan driver writes + * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a + * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word + * 0x320 run). Accept the B-only form with an implicit zero upper + * half - the old A+B requirement left qmd_dev_va at 0 and the + * candidate logged with qmd=0x0 (the field is identity/logging + * only, but the 0x0 read confused every prior session's forensics). + */ + if (st->qmd_have & 2) + st->qmd_dev_va = + (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) | + st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is @@ -10037,15 +10608,85 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; - for (i = 0; i < have; i++) { - u32 idx = (svc_grce_dst_ring_next - 1 - i) % - SVC_GRCE_DST_MAX; - const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - - recs[n].dst_va = rec->dst_va; - recs[n].len = rec->len; - memcpy(recs[n].data, rec->data, rec->len); - n++; + /* + * SESSION 127 C5C0 translator: the rec ring is session-wide + * and mixes every client's LOAD_INLINE_DATA destinations. A + * stale rec whose page this QMD does not name made the fill + * refuse the whole launch (GR fill ENOENT + * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan + * client's cancel page inherited from the driver-init + * phase). Keep only recs whose page is one of the pages the + * captured QMD names (the QMD buffer, the parameter bank, + * the program, the pointer pairs) - exactly the pages the + * backend's inline-served fill can use; keep everything when + * the QMD names none (fail-open). + */ + { + u64 keep[12]; + u32 nkeep = 0, k; + + if (st->qmd_dev_va) + keep[nkeep++] = st->qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[32] | + ((u64)(st->qmd[33] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[48] | + ((u64)(st->qmd[49] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) { + u32 w = 34 + 2 * k; + u64 pv = (u64)st->qmd[w] | + ((u64)(st->qmd[w + 1] & 0x1ffff) + << 32); + + if (pv) + keep[nkeep++] = pv & + ~(u64)(SEMA_PAGE_SIZE - 1); + } + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = + &svc_grce_dst_ring[idx]; + u64 rpage = rec->dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + bool keepit = !nkeep; + u32 j; + + for (j = 0; j < nkeep; j++) + if (keep[j] == rpage) { + keepit = true; + break; + } + /* + * SESSION 127 (gated): a stale session-wide + * rec whose page resolves nowhere made the + * fill refuse the whole launch; keep it when + * it resolves (the CUDA inline-served + * pages), drop only unresolvable unnamed + * ones. With the translator off, the + * historical keep-all behaviour stands. + */ + if (!keepit && gsp_sema_gr_xlate) { + u64 rpa = 0; + u32 rap = SVC_APERTURE_SYS; + + if (svc_resolve_va(vgpu, rpage, &rpa, + &rap) && + svc_walk_vaspaces(vgpu, rpage, + &rpa, &rap)) + continue; + } else if (!keepit) { + continue; + } + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; + } } /* * C10 (GAP-2 A5): duplicate destinations in the newest-first @@ -10202,6 +10843,60 @@ static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, h_object, push_va, off, len); } +/* + * r828: the guest kernel's RC watchdog (krcWatchdog_IMPL, 7 s notify + * timeout) posts a notifier cycle on its own channel (object 0x31415900) + * through the class2d subchannel (NVA06F_SUBCHANNEL_2D = 3): SET_NOTIFY_A/B + * name the notifier VA, NOTIFY (WRITE_ONLY) is the GPU's "write a zeroed + * NvNotification" - the driver set the status word to 0xFFFF and declares + * "GPU is probably locked" when it never comes back as 0. Observed: robust + * channels enabled by the GL client (GZDOOM), then NVRM krcWatchdog every + * 8 s and Xorg dead on its screen readback. The emulation must author the + * notification write, exactly like svc_notify_complete does for the + * work-submit token. + */ +#define SVC_2D_SUBCH 3 /* NVA06F_SUBCHANNEL_2D */ +#define SVC_2D_SET_NOTIFY_A 0x0104 /* payload: addr bits [56:32] */ +#define SVC_2D_SET_NOTIFY_B 0x0108 /* payload: addr bits [31:0] */ +#define SVC_2D_NOTIFY 0x010c /* payload: 0 = WRITE_ONLY */ + +static u64 svc_notify2d_done; +static u32 svc_notify2d_log; + +static void svc_notify2d_write(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + /* 16 bytes = the NvNotification (status word at +14). */ + u8 zero[16] = { 0 }; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!ch->notify2d_va) + return; + if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { + if (gsp_sema_dbg && svc_notify2d_log < 16) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", + ch->h_object, ch->notify2d_va); + } + return; + } + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return; + svc_notify2d_done++; + /* + * r834: pin the page that accepted the ack; the same stale-map + * defense as the push-side pin in the decode. + */ + svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1), + pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap)); + if (gsp_sema_dbg && svc_notify2d_log < 16) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n", + ch->h_object, ch->notify2d_va, pa); + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -10219,6 +10914,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * and the matmul r322 starved/hung note). */ svc.cur_ch = ch->h_object; + /* r835: decode-scoped VAS for channel-first VA resolution. */ + svc_cur_vas = ch->h_vaspace; /* * r34x: the acquire-wait capture is PER RING ENTRY. It used to be * reset only at SET_OBJECT, so a decode pass over a later entry (or @@ -10752,6 +11449,48 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + if (subch == SVC_2D_SUBCH && count >= 1 && addr >= 0x0100 && + addr <= 0x010c) { + /* + * r828: the RC watchdog's notifier cycle (class2d + * methods on subchannel 3). See svc_notify2d_write: + * without the zeroed notification the guest's + * krcWatchdog declares the GPU locked after 7 s and + * the X/GL stack wedges on screen readback. + */ + switch (addr) { + case SVC_2D_SET_NOTIFY_A: + ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32; + /* + * r834: this method stream reached us through + * a REAL read (svc_read_last_pa); pin its page + * so the watchdog's push translation cannot go + * stale later - the content-scanned map answers + * a zero page once the injected pin ages out, + * staging exactly the starvation the ack must + * not depend on. + */ + if (svc_read_last_pa) + svc_pin_store(push_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + svc_read_last_pa & + ~(u64)(SEMA_PAGE_SIZE - 1), + SVC_APERTURE_SYS); + break; + case SVC_2D_SET_NOTIFY_B: + ch->notify2d_va |= (u64)v0; + break; + case SVC_2D_NOTIFY: + if (count == 1 && v0 == 0) /* WRITE_ONLY */ + svc_notify2d_write(vgpu, ch); + break; + default: + break; + } + off += 4 + count * 4; + continue; + } + if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; @@ -10844,7 +11583,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; + /* + * r824: the deferral needs a + * candidate that OWNS this push + * (see svc_fwd_cand_owns_push) - + * a release from a push with no + * forwarding has nothing to wait + * for and applies inline. + */ if (svc_fwd_defer_arm() && + svc_fwd_cand_owns_push(push_va) && !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); @@ -12655,6 +13403,17 @@ static void svc_gr_defer_reset(void); static void svc_session_flush(void) { + /* r707x: per-client witness budgets. */ + svc_pushfail_log = 0; + svc_rdw_log = 0; + svc_pushdec_log = 0; + /* SESSION 132: util-sweep witnesses + their dedup keys are per-client. */ + svc_util_dec_log = 0; + svc_gr_util_log = 0; + memset(svc_util_dec_keys, 0, sizeof(svc_util_dec_keys)); + /* SESSION 143: the per-channel sweep-visibility keys are per-client. */ + svc_util_seen_n = 0; + memset(svc_util_seen_keys, 0, sizeof(svc_util_seen_keys)); unsigned long flags; u32 i; @@ -12673,6 +13432,7 @@ static void svc_session_flush(void) */ { bool live = false; + bool forced = READ_ONCE(svc_sess_flush_forced); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) @@ -12681,10 +13441,15 @@ static void svc_session_flush(void) break; } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (live) { + if (live && !forced) { pr_info("sema-svc: session flush skipped: live channel(s) present (deferred flush raced a new session registration)\n"); return; } + if (forced) { + svc_sess_flush_forced = 0; + pr_info("sema-svc: session flush (forced) executing with live=%d channel(s)\n", + live); + } } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ @@ -12760,6 +13525,7 @@ static void svc_session_flush(void) svc_pb_last = 0; svc_pb_last_n = 0; svc_gr_ring_done = 0; + svc_gr_ring_scan_done = 0; svc_ring_n = 0; /* * The learned tables are session state too: the primary VA->PA pin @@ -12785,6 +13551,16 @@ static void svc_session_flush(void) * name the previous client's pages (ISSUE-0002 fan-out). */ memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); + /* + * r820: PROMOTE_CTX records name one client's kernel buffers, and a + * new client's channels re-announce their own (the same VAs can back + * with different PAs across sessions): session state. A flush skips + * entirely while any channel is live, so a running client's records + * cannot be wiped under it. + */ + memset(svc_promo, 0, sizeof(svc_promo)); + svc_promo_seq = 0; + svc_promo_hit_log = 0; memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; /* r551: delivered-release marks are per-session (same handles reused). */ @@ -12869,6 +13645,10 @@ static void svc_session_flush(void) memset(svc_put_log_at, 0, sizeof(svc_put_log_at)); svc_userd_dbg = 0; memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch)); + memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n)); + memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since)); + memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off)); + svc_fb_claim_reset(); svc_userd_sweep_log = 0; memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at)); memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n)); @@ -13835,7 +14615,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; - if (op == 4 || op == 7) /* header-only methods */ + /* + * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2 + * (GRP2_USE_TERT) are 4-byte header-only formats per the class + * header; the GR submit pushes start with 0x0001fff0 + * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page + * was rejected here and the read fell through to the walk. + */ + if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */ return true; if (op != 1 && op != 3 && op != 5) return false; @@ -13884,12 +14671,30 @@ static u32 svc_pushpg_split_log; * validated candidate). Reads stay defensively copied through * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. */ +/* r707d: bounded push-read failure witness (see below). */ +static u32 svc_pushfail_log; +/* + * r707e: last content-backed ring pair resolved by any channel. The VAS + * candidate verify is retried with it on EVERY pump pass: the parked + * COPY_SERVER_PDES candidate for the client's user VAS parks at + * device-create and must register before the submit's first push read + * (a failed GR entry read is never retried with a fresh VAS), but the ring + * resolution and the park race within one pass - the every-pass retry + * closes that race in a single boot. + */ +static u64 svc_lastgood_ring_va; +static u64 svc_lastgood_ring_pa; + + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) { u8 *p = buf; + /* r820: per-read; the decode reads it right after the call. */ + svc_read_last_promote = false; + while (len) { u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, len, @@ -13899,6 +14704,7 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, bool have_win, done = false; bool first = (p == buf); int last_rc = -ENOENT; + int trusted_rc = 0; u32 src; have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); @@ -13952,7 +14758,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, default: /* learned cache / discovered map / page * directories / targeted rm_mem resolve */ - if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + trusted_rc = svc_lookup_trusted(vgpu, va, + &pa, &ap); + if (trusted_rc) continue; break; } @@ -14013,10 +14821,47 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, */ if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; + svc_read_last_src = (u8)src; + svc_read_last_pa = pa; + /* + * r820: name whether the trusted answer came from a + * driver-declared PROMOTE_CTX buffer; the decode uses + * it for the zero-NOP acceptance (see the flag). + */ + if (src == 4) { + u64 promo_pa = 0; + int promo_ap = SVC_APERTURE_SYS; + + svc_read_last_promote = + !svc_promo_lookup(page_va, &promo_pa, + &promo_ap); + } done = true; } - if (!done) + if (!done) { + svc_read_last_src = 0xff; + if (svc_pushfail_log < 96) { + u64 ipa = 0, apa = 0; + int iap = 0, aap = 0; + bool has_ipin = + (svc_ipin_lookup(va, &ipa, &iap) == 0); + bool has_alt = + (svc_ipin_alt_lookup(va, &apa, &aap) == 0); + + svc_pushfail_log++; + pr_info("sema-pushfail: va=0x%llx chunk=0x%llx page=0x%llx last_rc=%d trusted_rc=%d ipin=%d pa=0x%llx alt=%d pa=0x%llx win=%d pa=0x%llx\n", + (unsigned long long)va, + (unsigned long long)chunk, + (unsigned long long)page_va, + last_rc, trusted_rc, has_ipin, + (unsigned long long)ipa, + has_alt, + (unsigned long long)apa, + have_win, + (unsigned long long)win_pa); + } return last_rc; + } p += chunk; va += chunk; len -= chunk; @@ -14081,6 +14926,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, : SVC_APERTURE_FB; /* + * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on + * this function's bounded scan re-runs (userd_scans < 8); a pass + * cadence stretched by a rebuild can spend every scan before the VAS + * candidate registers (r820: the ring resolved at +1 s but no latch + * line followed). One bounded row per call names the state the scan + * decides on. + */ + if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n", + ch->h_object, ch->engine_type, ch->userd_scans, + rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as, + ch->userd_ok ? 1 : 0); + } + + /* * ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it * while it is still idle. In a second session the registered userdMem * offset (pool-relative, see below) aliases a DIFFERENT rank's real @@ -14276,12 +15137,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * USERD cannot pass that test. One scan per channel (the result * persists through the pump's write-back), logged on latch. */ - if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + /* + * r820: the latch scan is capped per channel so a channel that + * never anchors cannot sweep forever. But the resolution the scan + * needs can arrive LATE: the Xorg GL channel's ring resolves only + * when the client's VAS page directory registers (COPY_SERVER_PDES + * age-out), and r820 showed every one of the 8 pre-resolution scans + * spent before that moment (the pass cadence stretched by a + * rebuild), so the 10 s DMA-test window closed with the ring + * resolved and no retry left. Once ring_pa is non-zero the scan is + * cheap (the sweep validates against a resolved ring), so it may + * keep retrying up to a wider cap; the client's own timeout still + * bounds the channel's life. + */ + if (!ch->engine_type && ring_pa && + ch->userd_scans < (ring_pa ? 64u : 8u)) { u64 off, lim; u8 *base = vgpu->bars[1].vaddr; u64 best = 0, best_dist = ~0ULL; u32 best_put = 0, best_get = 0; bool best_pending = false; + u32 gate_n = 0, vfail_n = 0; ch->userd_scans++; lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); @@ -14292,23 +15168,33 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 k; u64 dist; bool pending; + int rck = -1; + u64 rkva = 0; + u32 rklen = 0; memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); if (!hp || hp > ch->gpfifo_entries || hg > hp) continue; + gate_n++; /* Validate: every claimed entry decodes to a * pushbuffer VA in the ring's window. */ for (k = 0; k < hp && k < 64; k++) { u8 e[8]; u64 eva; u32 elen, a0, a1; + int rdrc; - if (svc_read_va(vgpu, + rdrc = svc_read_va(vgpu, ch->gpfifo_va + (u64)k * 8, - e, sizeof(e))) { + e, sizeof(e)); + if (rdrc) { ok = false; + rck = (int)k; + /* -1: read failure rc; else the + * window check below. */ + rkva = (u64)rdrc; break; } memcpy(&a0, e, 4); @@ -14329,11 +15215,29 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, SVC_GR_RAM_WIN / 2 || elen < 4 || elen > 0x100000) { ok = false; + rck = (int)k; + rkva = eva; + rklen = elen; break; } } - if (!ok) + if (!ok) { + /* + * r820 diag: name why a scanned + * block was rejected (read rc or + * the entry VA/len window). + */ + vfail_n++; + if (gsp_sema_dbg && !ch->engine_type && + svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n", + off, hp, rck, + (long long)rkva, + rkva, rklen); + } continue; + } /* * Several blocks can validate: a previous * session leaves its old USERD holding the same @@ -14373,6 +15277,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, best_pending ? "" : ", idle"); return 0; } + /* + * r820 diag: the sweep found nothing adoptable. gates + * counts put/get-coherent blocks seen, vfails the ones + * rejected by the ring-entry validation. + */ + if (gsp_sema_dbg && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx\n", + ch->h_object, ch->userd_scans, + gate_n, vfail_n, best); + } } } return -ENOENT; @@ -14401,6 +15316,293 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) +/* + * SESSION 134: CE first-submission capture. + * + * The CE (copy-engine) rings are placed by the guest's allocator at a + * location that varies per boot and is NOT reliably described by the + * channel's registered gpfifo VA through the discovered page tables (r763 + * matched the fixed FB convention and the sweep decoded the inline-SASS + * entry; r765-r772 read zeros everywhere, the CE channels never latched, + * the driver's inline LAUNCH_DMA code downloads never executed and the + * frames stayed black). The logged instance where ring content and USERD + * were both live is the channel's first notifier clear - the guest's + * doorbell for its first submission, processed by the pump in the same + * pass that writes the ring. At that instant this hook: + * 1. reads entry0 through the channel's own translation (svc_lookup), + * 2. reads entry0 through the trusted walk (the VAS may have just + * landed since the registration read), + * 3. scans the shadow (0x100 step, first 64 MB, the GR-scan bound) for + * a ring-shaped page whose entry0 names a 1MB-aligned push VA in the + * CE window [0x120000000,0x121000000), and for every hit runs the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate + * whose USERD reads coherent (or idle 0/0) is PINNED and LATCHED + * exactly like svc_ring_autopin's CE path. + * Bounded: at most 4 captures per CE channel, 16 witness rows per boot; + * resets with the session (svc_session_flush). + */ +static u64 svc_ce_firstsub_log; + +/* + * SESSION 134: bounded completion hold for unlatched CE channels. + * + * The clear write in svc_notify_complete is what makes the guest treat the + * submission as consumed; the driver then recycles the ring and its content + * is gone before the client's VAS lands (sema_vas_cand_ms), leaving the CE + * ring latch with nothing readable (r769-r774: rrc=-2 at the first + * submission, the walk answering a zero page later). Holding the notifier + * busy for a bounded window keeps the driver's submission in flight - and + * with it the ring content that the regular per-pass latch path + * (svc_chan_userd through svc_lookup) needs - until the VAS registers and + * the ring reads coherently. 0 = off (the pre-SESSION-134 behaviour). + */ +static u64 svc_ce_hold_ms; +module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644); +MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)"); + +/* + * SESSION 134c rank-aware first-submission capture (see the capture below). + * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at + * ring + 0x10000 (observed for every CE channel across boots; the same + * convention svc_chan_userd applies in its regular per-pass latch). + */ +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL + +/* + * One candidate offset for the CE first-submission capture: validate the + * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose + * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like + * svc_ring_autopin's CE path. Returns true when latched. + */ +static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u8 *sbase, + u64 lim, u64 off, u32 *scan_hits, + u64 *latch_off, u64 *latch_upa, + u32 *latch_put, u32 *latch_get, + bool own_block) +{ + static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; + u32 h0, h1, hlen; + u64 hva; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if ((hva & 0xfffff) || hva < 0x120000000ULL || + hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000) + return false; + (*scan_hits)++; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + int prc; + + if (upa + 4 > lim) + continue; + prc = svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3); + /* + * SESSION 144d: the rank's OWN convention block (own_block, + * the +0x10000 candidate at the rank's convention offset) is + * latched even when its stale content probes incoherent + * (g > p) or idle - the guest rewrites put/get on its next + * submission and the pump's per-channel last_get processes + * entry0 exactly once. r813: client #2's convention block at + * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the + * capture instant, the coherence gate rejected it, and the + * never-used 0x82000 zero block was latched instead, so the + * client's first push never drained and it hung. The gates + * below stay for every heuristic candidate (other deltas and + * the fallback scan), preserving the SESSION 135 protection. + */ + if (!own_block || ui != 0) { + if (prc && prc != -EAGAIN) + continue; + } + /* + * SESSION 135: at the FIRST-SUBMISSION instant the channel's + * submission is by definition unconsumed - the notifier clear + * this capture runs before is what would let the driver treat + * it as consumed. A candidate whose USERD reads a consumed + * state (get >= put && put > 0) cannot be this channel's live + * just-submitted work; it is a PREVIOUS session's leftover + * block surviving in VRAM (r785: the capture latched the dead + * CUDA client's caf00012 block at fb 0x80000, put=1 get=1, + * for the Vulkan channel beef0101 - the decoded entry was the + * old session's, the client stalled on its expectations). + * Idle 0/0 stays acceptable (svc_userd_probe's own rule). + */ + /* + * SESSION 144c: this consumed-state rejection is for FOREIGN + * blocks found by the heuristic deltas. The rank's OWN + * convention block (+0x10000) legitimately carries the + * previous session's consumed state until the guest + * re-initialises the reused allocation - rejecting it made + * the r812 second client fall through to a never-used + * 0x82000 block and stall forever (cefirst row: off=0x80000 + * upa=0x82000 put=0 get=0 while the client waited for the + * first push's completion). The pump's per-channel last_get + * still processes entry0 exactly once, so latching the + * convention block with stale put/get is correct. + */ + if (!own_block && p3 && p3 == g3) + continue; + /* latch exactly like the CE autopin */ + svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + *latch_off = off; + *latch_upa = upa; + *latch_put = p3; + *latch_get = g3; + return true; + } + return false; +} + +static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 ring_pa = 0, wpa = 0; + u32 e0 = 0, e1 = 0, walk_e0 = 0, walk_e1 = 0; + u32 scan_hits = 0, latched = 0; + u64 latch_off = 0, latch_upa = 0; + u32 latch_put = 0, latch_get = 0; + int ring_ap = 0, wap = SVC_APERTURE_SYS; + int rrc = -1, wrc = -1; + + if (ch->engine_type == 0 || ch->ce_index >= 32) + return; + if (svc_ce_firstsub_n[ch->ce_index] >= 4) + return; + svc_ce_firstsub_n[ch->ce_index]++; + + if (sbase && lim >= 0x1000) { + /* 1. the channel's own translation at this instant. */ + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + if (!rrc && ring_pa) { + u8 e[8]; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + } + } + /* 2. the trusted walk at this instant. */ + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap)) { + u8 e[8]; + + wrc = 0; + if (!svc_read_ap(vgpu, wpa, svc_ap_of(wap), e, + sizeof(e))) { + memcpy(&walk_e0, e, 4); + memcpy(&walk_e1, e + 4, 4); + } + } + /* 3. shadow scan for a CE-window ring page. */ + { + /* + * SESSION 134c rank-aware order: try the rank's + * convention offset first, then the bounded scan, + * and never accept an offset another CE rank already + * latched. The rank-blind first-hit latched rank 0's + * block for rank 1 on r779 (both rows read + * off=0x80000 upa=0x90000), so the rank-1 channel's + * first release decoded from rank 0's ring/page and + * was stale-dropped (cur=1 already) - the client's + * rank-1 tracker wait never completed and it spun + * forever. A rank whose block is not in the shadow + * at this instant now simply stays unlatched for the + * regular per-pass CE latch instead of stealing a + * foreign block. + */ + u64 exp = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * ch->ce_index; + u64 off; + + latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, + lim, exp, &scan_hits, + &latch_off, &latch_upa, + &latch_put, &latch_get, + true); + for (off = 0; !latched && off + 0x90 <= lim; + off += 0x100) { + unsigned int ri; + bool foreign = false; + + if (off == exp) + continue; + for (ri = 0; ri < 32; ri++) { + if (ri == ch->ce_index || + !svc_ce_firstsub_off[ri]) + continue; + if (svc_ce_firstsub_off[ri] == off) { + foreign = true; + break; + } + } + if (foreign) + continue; + /* + * SESSION 144: symmetric protection - a CE + * fallback scan must not steal a block another + * channel (CE rank or GR) already claims. + */ + if (svc_fb_claim_foreign(vgpu, ch->h_object, off, + off + 0x10000)) + continue; + latched = svc_ce_firstsub_try_off(vgpu, ch, + sbase, lim, + off, &scan_hits, + &latch_off, + &latch_upa, + &latch_put, + &latch_get, + false); + } + if (latched) { + svc_ce_firstsub_off[ch->ce_index] = latch_off; + /* + * SESSION 144: publish the claim so a GR rank + * whose own ring is not written yet cannot + * latch this CE block (r805-r808: every CUDA + * GR rank stole the CE rank-0 block and the + * launch was never captured). + */ + svc_fb_claim_add(ch->h_object, latch_off, + latch_upa); + } + } + } + + if (gsp_sema_dbg && svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u n=%u rrc=%d rpa=0x%llx rap=%d e0=0x%x e1=0x%x wrc=%d wpa=0x%llx wap=%d we0=0x%x we1=0x%x scan_hits=%u latch=%u off=0x%llx upa=0x%llx put=%u get=%u\n", + ch->h_object, ch->ce_index, + svc_ce_firstsub_n[ch->ce_index], rrc, ring_pa, + ring_ap, e0, e1, wrc, wpa, wap, walk_e0, walk_e1, + scan_hits, latched, latch_off, latch_upa, + latch_put, latch_get); + } + if (latched) + pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + latch_off, latch_upa, latch_put, latch_get); +} + static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -14453,6 +15655,41 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, return; /* + * SESSION 134: a visible doorbell means a submission just landed - + * for a CE-class channel this is the one instant its ring content + * and USERD are both live. Capture/snapshot it BEFORE the clear + * write below (the clear is what makes the guest treat the + * submission as complete and recycle the ring), see + * svc_ce_firstsub_capture. + */ + if (ch->engine_type) + svc_ce_firstsub_capture(vgpu, ch); + + /* + * SESSION 134: keep an unlatched CE submission open so the ring + * content survives until the latch can read it (see + * svc_ce_hold_ms). The regular per-pass latch path retries while + * the hold runs; the hold expires un-latched into the old + * behaviour, so a channel the emulation can never resolve cannot + * strand the guest. + */ + if (ch->engine_type && !ch->userd_ok && svc_ce_hold_ms && + ch->ce_index < 32) { + if (!svc_ce_hold_since[ch->ce_index]) + svc_ce_hold_since[ch->ce_index] = jiffies; + if (time_before(jiffies, + svc_ce_hold_since[ch->ce_index] + + msecs_to_jiffies(svc_ce_hold_ms))) + return; + if (svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u completion hold expired (%llu ms) unlatched; clearing\n", + ch->h_object, ch->ce_index, svc_ce_hold_ms); + } + svc_ce_hold_since[ch->ce_index] = jiffies; + } + + /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ @@ -14495,8 +15732,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ -#define SVC_CE_RING_FB_BASE 0x80000ULL -#define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */ @@ -14532,6 +15767,62 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool gsp_sema_gr_rings = true; /* + * SESSION 132 dead-entry guard for ring candidates. + * + * The r759 false positive: a vidmem candidate (0xa0000) whose first GPFIFO + * entry (push 0x120100000/40) passed the plausibility gate - the USERD + * probe hit the channel's REAL USERD at 0xb0000, which the VRAM shape scan + * had just found 0x10000 after the candidate by coincidence - was pinned + * as the GR ring while the client's own page tables resolve the ring VA to + * a guest-RAM page (0x11623d000) whose entry0 is live (push 0x4000000/ + * 0x8bc, decoded by the util sweep). The pump then read the pinned page, + * could not fetch the entry's push (all-zero through every source), and + * skipped the entry after 39 failed rescues - the live ring stayed + * undrained and every upload behind it invisible. + * + * A candidate whose first entry cannot deliver a method stream is not this + * channel's ring. The push head is read through the client's own page + * tables (guest truth); a push that cannot be resolved at all does NOT + * mark the entry dead (the VAS may not be registered yet on an early + * pass, and vidmem-backed pushes are not in the client's tables at all). + */ +static bool svc_ring_entry_dead(struct nv_vgpu_device *vgpu, u32 e0, u32 e1) +{ + u64 va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 len = ((e1 >> 10) & 0x1fffff) << 2; + u8 head[16]; + + if (!va || len < 8 || len > 0x10000) + return false; /* unusable entry; the gate rejects it */ + if (svc_read_va_trusted(vgpu, va, head, sizeof(head))) + return false; /* cannot tell: do not reject */ + return !memchr_inv(head, 0, sizeof(head)); +} + +/* + * SESSION 132: does the client's own page table (VAS walk only - never the + * pin/lat/map sources, so a bad pin cannot vote for itself) resolve the + * ring VA to a page whose first entry is LIVE? Used to reject a dead + * candidate only when a live alternative provably exists. + */ +static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 we[8]; + u32 w0, w1; + + if (svc_walk_vaspaces(vgpu, gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return false; /* walk cannot resolve: no verdict */ + if (svc_read_ap(vgpu, wpa, wap, we, sizeof(we))) + return false; + memcpy(&w0, we, 4); + memcpy(&w1, we + 4, 4); + return !svc_ring_entry_dead(vgpu, w0, w1); +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -14558,6 +15849,14 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *len < 4 || *len > 0x10000) return false; /* + * SESSION 132: a candidate whose entry0 push provably reads all-zero + * through the client's own page tables cannot deliver a method + * stream (the pump's rescue path eventually skips exactly these) - + * reject it so the autopin falls through to a live candidate. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) + return false; + /* * Gate: the USERD must be readable. rc==0 is an active get/put * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a * freshly created channel is not rejected. -EINVAL (implausible @@ -14567,6 +15866,20 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *get = 0; prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, ch->gpfifo_entries, put, get); + /* + * SESSION 132: the driver places the GPFIFO and the USERD + * independently (r759: ring in guest RAM 0x11623d000, USERD in + * vidmem 0xb0000), so a walk-resolved ring cannot pass a + * same-aperture USERD probe. Accept the channel's own USERD when + * one is already known - the put/get liveness gate is unchanged. + */ + if (prc != 0 && prc != -EAGAIN && ch->userd_ok && ch->userd_pa && + ch->userd_pa != cand_pa + SVC_GR_USERD_OFF) { + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, ch->userd_pa, ch->userd_as, + ch->gpfifo_entries, put, get); + } return prc == 0 || prc == -EAGAIN; } @@ -15597,6 +16910,58 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * r705h: host-triggered pdb discovery. The automatic latch + * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM + * ring scan, which sweeps [0,6GB) before the client's pushes exist and then + * gives up for the boot; and per-client VAS state is flushed at session end. + * This entry point lets a host tool that CONTENT-LOCATED a verified + * {ring VA -> ring PA} pair (the ring page's GPFIFO entries are its + * fingerprint) hand the pair to the service while the client is alive, so: + * 1. every parked COPY_SERVER_PDES candidate is verified against the pair + * (fast: a walk each), and + * 2. if none verifies, the vidmem walk-root sweep anchors on the pair. + * Registered VASes make the client's pushbuffer VAs walkable and the pump + * can then decode the submit's pushes (the fence release among them). + */ +static u64 svc_pdb_disc_va; +static u64 svc_pdb_disc_pa; +static bool svc_pdb_disc_pending; + +/* + * r707r: bounded channel rewind (debug). After the VAS pdb anchor lands, + * entries skipped before it never re-decode on their own (their unread + * window expires into a skip), so the release they carry would be lost for + * that put epoch. A host tool asks for a rewind (sema_chan_rewind: a hex + * h_object, or "all"); the pump re-processes the channel from slot 0 with + * the consumed-slot bitmap and the per-entry rescue state cleared. + */ +static u32 svc_rewind_h; +static bool svc_rewind_all; +static bool svc_rewind_pending; + +int gsp_sema_service_chan_rewind(u32 h_object) +{ + if (!svc.task) + return -ENODEV; + svc_rewind_h = h_object; + svc_rewind_all = (h_object == 0xffffffffu); + svc_rewind_pending = true; + wake_up_process(svc.task); + return 0; +} + +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa) +{ + if (!svc.task) + return -ENODEV; + svc_pdb_disc_va = ring_va & ~0xfffULL; + svc_pdb_disc_pa = ring_pa & ~0xfffULL; + svc_pdb_disc_pending = true; + wake_up_process(svc.task); + return 0; +} + +/* * r496f: discover a kernel-RM client VAS's page directory in the emulated * VRAM by verification. The CPU RM builds internal-VAS page tables in * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally @@ -15677,6 +17042,45 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, return; /* wait until the CE channels are known */ /* + * SESSION 148 perf (r858 evidence): the RC-watchdog channel's ring + * page sits at (notifier_pa page + 0x400) in EVERY observed boot - + * r848: notif 0x11b689420 -> ring 0x11b689400; r853: 0x11b8a6420 + * -> 0x11b8a6400; r858: 0x102e04420 -> 0x102e04400. The registered + * notifier PA is guest truth from the channel alloc, so this + * derivation is deterministic; the trusted walk resolved the same + * ring 14 s LATE on r858 (77280 registered, 77294 walk) and, on the + * failing boots, inside that window the driver's DMA test read + * zeros/garbage -> "Failed to initialize DMA" and guest vmap-lock + * livelocks followed. Content-gated by the standard candidate probe + * (entry0 + USERD validation), so a wrong derivation costs a failed + * probe, not a wrong pin. + */ + if (ch->h_object == 0x31415900u && ch->notifier_pa) { + u64 rpa = (ch->notifier_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) + + 0x400; + + if (svc_gr_ring_candidate(vgpu, ch, rpa, SVC_APERTURE_SYS, + &va, &len, &put, &get)) { + svc_pin_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + if (ch->userd_reg) + ch->userd_ok = true; + else { + ch->userd_pa = rpa + SVC_GR_USERD_OFF; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + } + svc_gr_ring_base = rpa; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=1 (notifier-derived; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + rpa, va, len, put, get); + return; + } + } + + /* * Guest truth first: the channel registration carries the USERD * memdesc base for THIS session's allocation, and the ring sits one * USERD block before it (observed for every GR channel across @@ -15731,11 +17135,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->gpfifo_entries, &put, &get)) continue; + /* + * SESSION 132: reject a dead candidate when + * the client's own page tables provably + * resolve the ring VA to a live page (the + * r759 vidmem false positive). + */ + if (svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va)) + continue; svc_pin_store(ch->gpfifo_va, cand, cap); svc_lat_store(ch->gpfifo_va, cand, cap); ch->userd_as = cap; ch->userd_ok = true; svc_gr_ring_base = cand; + if (cap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, cand, + ch->userd_pa); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -15766,13 +17182,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, - &put, &get)) { + &put, &get) && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; + svc_fb_claim_add(ch->h_object, pa, + pa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -15804,7 +17224,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && - va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { /* * Best effort only: a wrapped GP_PUT * legitimately exceeds the entry count, so a @@ -15819,10 +17241,20 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get); svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); - ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; - ch->userd_as = svc_ap_of(pin_ap); - ch->userd_ok = true; + /* r831: keep a registered real-PA userd. */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else { + ch->userd_pa = + pin_pa + SVC_GR_USERD_OFF; + ch->userd_as = svc_ap_of(pin_ap); + ch->userd_ok = true; + } svc_gr_ring_base = base; + if (svc_ap_of(pin_ap) == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, pin_pa, + pin_pa + + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -15854,10 +17286,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get)) { svc_pin_store(ch->gpfifo_va, wpa, wap); svc_lat_store(ch->gpfifo_va, wpa, wap); - ch->userd_pa = wpa + SVC_GR_USERD_OFF; - ch->userd_as = wap; - ch->userd_ok = true; + /* + * r831: the ring+0x2000 userd convention is a guess + * for the CUDA GR layout; a channel whose REGISTERED + * userd is a real PA (the RC watchdog) keeps its own + * (the candidate gate above probes it as fallback). + */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else { + ch->userd_pa = wpa + SVC_GR_USERD_OFF; + ch->userd_as = wap; + ch->userd_ok = true; + } svc_gr_ring_base = wpa; + if (wap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, wpa, + wpa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -15867,6 +17312,185 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * SESSION 133: bounded shadow ring scan fallback. + * + * The CE-class rings live in VIDMEM and the FB convention only + * matches when the guest's VRAM allocator happens to place them at + * the conventional offsets (r763: matched, fb 0x80000/0xa0000, the + * channel decoded; r765: read zeros, the ring VA resolved to a + * stale RAM page whose entry0 was dead, the CE channel never + * latched and its inline SASS copies never decoded - black + * frames). When every deterministic candidate has failed, scan the + * shadow for a ring-shaped page: entry0 must name a 1 MB aligned + * push VA in the client window with a sane length, must not be + * provably dead (svc_ring_entry_dead - the r759 false-positive + * guard), and the USERD at the observed deltas (+0x10000 for the + * CE family, +0x2000 for GR) must read a coherent put/get. One + * bounded scan per rank per boot; a false candidate still costs + * only a failed probe. + */ + if (!test_bit(ch->gr_index, &svc_gr_ring_scan_done)) { + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 off; + + /* + * SESSION 133b: the rings are NOT page-aligned (r767: the + * USERD shape scan found the channel blocks at 0xfee00/ + * 0xff000/0xff200, so the rings sit at unaligned offsets like + * 0xefe00); step by 0x100 and RETRY (rate-limited) - a first + * pass can run before the client has written the ring or the + * USERD, and a consumed once-bit would never rescan. + */ + if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last + + msecs_to_jiffies(1500))) + return; + svc_gr_ring_scan_last = jiffies; + if (sbase && lim > 0x1000) { + static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF, + 0x1000 }; + + /* + * SESSION 133c pass 1: find the USERD by its OWN + * shape (put@+0x8c, get@+0x88 - the find_userd + * layout) and derive the ring from it. The USERD + * words appear at channel creation and stay; the + * ring entry may be written later, so this pass is + * the stabler trigger. Each hit is validated by the + * derived ring's entry0 (shape + not-dead), so a + * coincidental put/get pair costs a failed check. + */ + for (off = 0; off + 0x90 <= lim; off += 0x100) { + u32 up, ug; + unsigned int ui; + + memcpy(&up, sbase + off + 0x8c, 4); + if (up < 1 || up > 65535) + continue; + memcpy(&ug, sbase + off + 0x88, 4); + if (ug > up) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 roff; + u32 h0, h1, p3 = 0, g3 = 0; + u64 hva; + u32 hlen; + + if (off < udeltas[ui]) + continue; + roff = off - udeltas[ui]; + memcpy(&h0, sbase + roff, 4); + memcpy(&h1, sbase + roff + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || + hva >= SEMA_VA_HI || hlen < 4 || + hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + /* + * SESSION 144: never latch a foreign + * block. Without this every CUDA GR + * rank stole the CE rank-0 block while + * its own ring was unwritten + * (r805-r808); staying unlatched here + * lets the FB-convention path latch the + * real ring once the client writes it. + */ + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + roff, off)) + continue; + if (svc_userd_probe(vgpu, off, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = roff; + svc_fb_claim_add(ch->h_object, roff, off); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd-shape scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, roff, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + /* pass 2: ring-entry-shaped pages (below) */ + for (off = 0; off + 8 <= lim; off += 0x100) { + u32 h0, h1; + u64 hva; + u32 hlen; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || hva >= SEMA_VA_HI || + hlen < 4 || hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + + if (upa + 4 > lim) + continue; + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + off, upa)) + continue; + if (svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = off; + svc_fb_claim_add(ch->h_object, off, + upa); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (shadow scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, off, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -15970,8 +17594,24 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * entry and pushbuffer reads resolve. Sliced so the service thread * stays responsive; gives up per channel after * SVC_GR_RAM_PASS_MAX slices. - */ - if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + * + * SESSION 147 (r840 evidence): the scan must NOT run for the kernel + * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel + * allocation whose VA is always resolvable through the driver- + * published VAS walk - the walk is the correct, deterministic + * source. On r840 the walk lost a registration race, the scan ran + * first and latched a FOREIGN page (pa=0x101cc3000, garbage + * entries: raw=0xffff88f10403e000), and the bad latch poisoned + * every later entry/push read: the driver's DMA test never saw its + * push complete and Xorg died with "Failed to initialize DMA" + * (the same boot passed on r835/836/838 where the trusted walk + * won the race). Gating the scan keeps the per-pass walk retry + * alive for the watchdog; the walk landing late is indistinguishable + * from the r838 flow, and never landing is no worse than a poisoned + * latch. + */ + if (ch->h_object != 0x31415900u && + ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && !test_bit(ch->gr_index, &svc_gr_ring_done)) { u64 hit_pa = 0; @@ -16075,6 +17715,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + /* + * SESSION 144b: record the block ownership here too - CE ranks that + * latch through this plain autopin path (not the first-submission + * capture) were invisible to the GR fallback scan, which then stole + * their block (r809: CE rank 1 at fb 0xa0000). + */ + svc_fb_claim_add(ch->h_object, pa, pa + 0x10000); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", @@ -16599,6 +18246,65 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, u32 ncand = 0, i; /* + * SESSION 148 perf (measured on r843): content-verified full-chain + * rescue, usable by GR channels too. The CE-only gate below refused + * every GR push outright, yet the same svc_lookup the diagnostic + * probe uses resolves the stalled entries correctly (X's frame + * pushes at 0x120000514 -> vidmem shadow pa=0x200514 with plausible + * stream content) while the pump's own read went through a stale + * shadowing translation, failed plausibility, and skipped the entry + * - one full retry window per push, the measured 22.4 s/frame + * glxgears cadence. Resolve via svc_lookup, verify the bytes parse + * as a method stream, then pin the verified page so the pump's + * re-read lands on it. Cheap: one chain lookup + one page read. + */ + if (entry_va && entry_len && entry_len <= SEMA_PAGE_SIZE) { + u64 mpa = 0; + int map = SVC_APERTURE_SYS; + u8 *vb = kmalloc(entry_len, GFP_KERNEL); + int lrc = -ENOMEM, rrc = -ENOMEM; + bool plaus = false; + + if (vb) { + lrc = svc_lookup(vgpu, entry_va, &mpa, &map); + if (!lrc) + rrc = svc_read_ap(vgpu, mpa, map, vb, + entry_len); + if (!rrc) + plaus = svc_method_stream_plausible(vb, + entry_len); + if (gsp_sema_trace_h && + (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: maprescue h=0x%x va=0x%llx len=%u lookup=%d pa=0x%llx ap=%d read=%d plaus=%d\n", + ch->h_object, entry_va, entry_len, + lrc, mpa, map, rrc, plaus); + } + if (!lrc && !rrc && plaus) { + svc_pin_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc_lat_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc.rescue_map_ok++; + if (svc.rescue_map_ok <= 8) + pr_info("sema-svc: rescue via map h=0x%x va=0x%llx -> pa=0x%llx ap=%d (verified stream)\n", + ch->h_object, entry_va, mpa, + map); + kfree(vb); + return 0; + } + kfree(vb); + } + } + + /* * The matcher needs only the channel rank (slot) and the release-push * shape; it does NOT need the pool semaphore translation that * expected_sem_payload is learned from. Requiring the learned payload @@ -16902,6 +18608,16 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, */ u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa; + if (gsp_sema_trace_h && (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pub h=0x%x put=%u last=%u armed=%d pa=0x%llx cnt=%u gate=%s\n", + ch->h_object, put, (u32)ch->last_get, ch->gr_pub_armed, + pa, ch->counter_last, + !pa ? "no-pa" : + (ch->last_get != put) ? "outstanding" : + (!ch->gr_pub_armed ? "unarmed" : "publish")); + } if (!pa) return; /* @@ -18534,6 +20250,156 @@ static void svc_fwd_hold_direct_note(void) } /* + * SESSION 127 C5C0 translator: pending GRCE copy-out list. + * + * The Vulkan compute client submits its frame work in the pre-built + * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch + * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A + * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into + * the client's sysmem readback buffer. Both pushes are only visible to the + * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main + * walk. + * + * Copy ordering: the copy must read the image AFTER the forwarded launch + * completed (the engine's output is copied back into the VA shadow on the + * completion edge). The sweep only records the copy; it is executed inside + * svc_fwd_hold_gr_release, i.e. after the backend published done and after + * the copy-out, immediately before the launch's guest release word is + * authored - so the client's fence never observes a pre-render image. + * A bounded fallback run (sweep, when no forward is owed) keeps the list + * from stalling when the launch was refused. + */ +#define SVC_GR_COPY_PEND 8 +struct svc_gr_copy_pend { + bool valid; + u32 h_object; + u64 push_va; + u64 hash; + u64 src; + u64 dst; + u32 bytes; /* per-row byte count */ + u32 rows; + u32 dpitch; + u32 spitch; + u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */ + u32 remap; /* SET_REMAP_COMPONENTS (logging) */ +}; +static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND]; +static u32 svc_gr_copy_pend_next; +static u64 svc_gr_copy_run_total; +static u64 svc_gr_copy_run_log; + +static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, + u64 src, u64 dst, u32 bytes, u32 rows, + u32 dpitch, u32 spitch, u32 dma, u32 remap) +{ + u32 i; + + if (!bytes || !rows || !src || !dst) + return; + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + + if (p->valid && p->push_va == push_va && p->src == src && + p->dst == dst && p->bytes == bytes && p->rows == rows) + return; /* already recorded (idempotent re-sweep) */ + } + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + if (p->valid) + continue; /* keep live ones; take a free slot */ + p->valid = true; + p->h_object = h_object; + p->push_va = push_va; + p->hash = hash; + p->src = src; + p->dst = dst; + p->bytes = bytes; + p->rows = rows; + p->dpitch = dpitch ? dpitch : bytes; + p->spitch = spitch ? spitch : bytes; + p->dma = dma; + p->remap = remap; + pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n", + h_object, push_va, src, dst, bytes, rows, p->dpitch, + p->spitch, dma, remap); + return; + } + /* All slots live: overwrite the oldest position. */ + { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n", + p->push_va); + memset(p, 0, sizeof(*p)); + } +} + +/* + * Execute and retire every pending GRCE copy. Called from the completion + * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep + * when no forward is owed. The executor is the same svc_copy() the CE/GCE + * op path uses (handles vidmem shadow sources and sysmem destinations). + */ +static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + u64 done = 0; + int rc; + + if (!p->valid) + continue; + rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, + SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, + p->rows, p->dpitch, p->spitch, p->h_object, 0, + p->hash); + svc_gr_copy_run_total++; + if (rc == 0 || rc == SVC_COPY_FORWARDED) { + p->valid = false; + if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, p->rows, rc); + } + } else if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, rc, done); + } + } +} + +/* + * Is this release VA owned by a live, not-yet-authored launch candidate + * captured from THIS push? (Only the launch's own push's release is + * withheld; a continuation fence entry elsewhere in the ring keeps the + * historical immediate-forward behaviour.) + */ +static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].rel_done && + svc_gr_cand_ring[ri].rel_va == va && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + +/* * Author a GR launch completion release (the QMD-level semaphore the * physical engine writes when the launch completes) in software, * forward-only. Used when a hold resolves without a successful hardware @@ -18581,6 +20447,14 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, { if (!svc_fwd_req.gr) return; + /* + * SESSION 127 C5C0 translator: the launch is complete and its output + * pages were copied back into the VA shadow on this edge - run any + * pending GRCE copy-outs (image -> linear readback buffer) BEFORE + * authoring the client's release word, so the fence can never clear + * on a pre-render image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -18718,6 +20592,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_push_va, waited); return; } + /* + * SESSION 127 C5C0 translator: this edge authors the guest release + * without a hold - run any pending GRCE copy-outs first, exactly like + * the hold's success edge, so the fence cannot clear on a pre-render + * image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -20062,8 +21943,34 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) goto note_done; } } - if (svc_resolve_va(vgpu, page, &pa, &ap)) - return; + if (svc_resolve_va(vgpu, page, &pa, &ap)) { + /* + * SESSION 127 C5C0 translator (gated): a client-VAS page. The + * Vulkan + * QMD's program, parameter and descriptor pages live in the + * client's own page tables (vidmem aperture) and the trusted + * sources - pins, learned cache, discovered map - do not + * carry them, which made the fill refuse the launch with + * -ENOENT. Resolve through the registered-VAS walk instead: + * the exact path walk_probe proves (r707w). + */ + u64 wpa = 0; + u32 wap = SVC_APERTURE_SYS; + + if (!gsp_sema_gr_xlate || + svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + return; + pa = wpa; + ap = svc_ap_of(wap); + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "vas"; + goto note_done; + } svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); /* @@ -20551,6 +22458,14 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes) if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* SESSION 139: float-shaped words are scalars, not device VAs + * (see the scan-site note; r789's 480.0f = 0x43f00000). */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < *n_cand; w++) if (wins[w] == wv) @@ -21056,18 +22971,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0x150u / 8u; k < 256; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; @@ -21075,6 +23005,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21109,18 +23056,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) { u64 v, wv; @@ -21130,6 +23092,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21142,19 +23121,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; - if (pp->qmd_word < 32 || pp->qmd_word > 47 || + /* SESSION 139: include word 48 (see the r788 finding: the + * word-34 pointer dedupes into the program's 4K page whose + * entry carries qmd_word=48; the old 32..47-even filter left + * that page unscanned). */ + if (pp->qmd_word < 32 || pp->qmd_word > 48 || (pp->qmd_word & 1) || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } /* * relfix PART 3: whole page, matching the rewrite's scan * range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff @@ -21167,6 +23165,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21179,6 +23194,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc_gr_scan_inline(wins, &n_cand, pp->va); } /* + * SESSION 140 (the n_data wiring): seed the candidate set with the + * launch's OWN QMD pointer pairs (words 34..47, already decoded into + * gr_qmd from the push's inline-QMD run). These ARE the launch's + * data-window pointers - the r792 evidence: the capture holds + * ptr[0]=0xdff0febc00 but the fill's page scans read only empty + * shadow pages (the QMD page's vidmem shadow is zero; its content + * never arrives - the invisible-vidmem-write class), so the pointer + * values could never become candidates and n_data stayed 0 (refused + * degenerate). The values need no page scan - use them directly. + */ + { + u32 w2; + + for (w2 = 0; w2 + 1 < 16; w2 += 2) { + u64 pv = (u32)svc_fwd_req.gr_qmd[34 + w2] | + ((u64)(svc_fwd_req.gr_qmd[35 + w2] & + 0x1ffffu) << 32); + u64 pw; + u32 fexp; + + if (pv < 0x10000000ULL || pv >= (1ULL << 47) || + (pv & 3)) + continue; + fexp = (u32)(pv >> 23) & 0xffu; + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + pw = pv & ~(u64)(SEMA_PAGE_SIZE - 1); + for (w = 0; w < n_cand; w++) + if (wins[w] == pw) + break; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) { + wins[n_cand++] = pw; + is_arg[n_cand - 1] = true; + if (svc_gr_pages_log < 64) { + svc_gr_pages_log++; + pr_info("sema-gr: qmd-ptr seed va=0x%llx (word %u) h=0x%x\n", + pw, 34 + w2, + svc_fwd_req.gr_h_object); + } + } + } + } + if (gsp_sema_dbg && svc_gr_pages_log < 48) { + svc_gr_pages_log++; + pr_info("sema-gr: fill cand n=%u pages=%u push=0x%llx\n", + n_cand, svc_fwd_req.gr_n_pages, + svc_fwd_req.gr_push_va); + for (w = 0; w < n_cand && w < 12; w++) + pr_info("sema-gr: win[%u] va=0x%llx arg=%u\n", + w, wins[w], is_arg[w]); + } + /* * r635: the launch-level CONTENT anchor for the host-side fallback, * computed BEFORE the span loop so candidate order cannot decide. * An argument candidate whose span already holds the client's own @@ -21817,7 +23884,12 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; - if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { + if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) || + (gsp_sema_gr_xlate && + !svc_walk_vaspaces(vgpu, + svc_gr_cand.qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &qpa, &qap))) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); @@ -21877,12 +23949,42 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); if (svc_gr_cand.prog_va) svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + /* + * SESSION 140: the QMD page itself carries the launch's pointer + * VALUES (QMD words 32..49 live inside it - the param, the + * pointer-region pairs and the program). Without this note the fill + * never scans it, so the pointer values (r790: word 34 = + * 0xdff0febc00) can never become candidates and n_data stays 0 + * (refused: degenerate data set). Noted under word 34 so the + * 32..48-even bank-page scan covers it. + */ + if (svc_gr_cand.qmd_dev_va) + svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34); for (i = 0; i < svc_gr_cand.n_pointer; i++) svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], svc_gr_cand.pointer_word[i]); for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + if (gsp_sema_dbg && svc_gr_pages_log < 24) { + svc_gr_pages_log++; + pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.param_va, + svc_gr_cand.prog_va, svc_gr_cand.n_pointer, + svc_gr_cand.n_inline, svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + for (i = 0; i < svc_gr_cand.n_pointer && i < 8; i++) + pr_info("sema-gr: ptr[%u] va=0x%llx word=%u\n", i, + svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_fwd_req.gr_n_pages && i < 12; i++) { + const struct svc_gr_page *pg = + &svc_fwd_req.gr_pages[i]; + + pr_info("sema-gr: page[%u] va=0x%llx pa=0x%llx ap=%u qmd_word=%u\n", + i, pg->va, pg->pa, pg->ap, pg->qmd_word); + } + } if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) return -E2BIG; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { @@ -21890,10 +23992,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (pg->pa) continue; - if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) - return -ENOENT; - if (!svc_gr_page_inline_covers(pg->va)) + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE || + !svc_gr_page_inline_covers(pg->va)) { + if (svc.fwd_gr_enoe_log < 8) { + svc.fwd_gr_enoe_log++; + pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n", + pg->va, pg->qmd_word, + svc_gr_cand.n_inline, + svc_gr_page_inline_covers(pg->va), + svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + } return -ENOENT; + } } /* * GRFIX FIX1c: the fill hosts the self-enforcing decision point and @@ -22480,11 +24591,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || - svc_gr_cand.h_object != ch->h_object) + svc_gr_cand.h_object != ch->h_object) { + if (svc.fwd_gr_nocand_log < 8) { + svc.fwd_gr_nocand_log++; + pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n", + ch->h_object, gsp_phys_gr_forward_enabled(), + svc_gr_cand.valid, svc_gr_cand.h_object, + svc_gr_cand.offered); + } return false; + } svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_nordy++; + if ((svc.fwd_gr_nordy & 0x3F) == 1) + pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n", + ch->h_object, gsp_phys_backend_gr_ready(), + gsp_phys_guest_mem_enabled(), + svc.fwd_gr_nordy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); @@ -22495,6 +24620,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { + svc.fwd_gr_holdbusy++; + if ((svc.fwd_gr_holdbusy & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n", + ch->h_object, svc_fwd_hold.ch_handle, + svc.fwd_gr_holdbusy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; /* @@ -22515,6 +24645,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_inflight++; + if ((svc.fwd_gr_inflight & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n", + ch->h_object, svc_fwd_state, + svc.fwd_gr_inflight); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if (!gsp_cuda_bridge_enabled()) { @@ -22541,7 +24676,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + if (svc.fwd_gr_refused < 8 || + (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); @@ -23508,8 +25644,1148 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, put, get, ch->submissions, waited, reason); } +/* + * --------------------------------------------------------------------- + * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class + * channel's PRE-BUILT ring continuation. + * + * The Vulkan client's fence waits never clear unaided because the + * driver's kick boundary (GP_PUT) stops at the first batch while the ring + * already carries the continuation entries it pre-built for the next + * submits - and the fence's own INC-5 RELEASE group (observed + * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of + * those entries, so the pump's GP_PUT-bounded walk never decodes it + * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever + * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word + * directly; this fold makes the service author it from the driver's own + * push bytes. + * + * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data, + * off since the false-positive corruption incidents), this sweep is + * CHANNEL-ANCHORED: it only walks entries reachable through the ring's + * validated GPFIFO translation, only accepts entries whose bytes pass the + * strict method-stream gate, and only applies the semaphore release + * groups it finds - forward-only, through the standard pending ledger + * (svc_pending_add). No consumption bookkeeping is touched (last_get, + * put_prev, slot_done_epoch and the replay state stay as the pump left + * them), so a later real kick of the same entries decodes them normally; + * repeated sweeps are idempotent (the pending ledger dedups (va,payload) + * and the apply path is forward-only). + * + * Gated by the same live-epoch arm as the counter publish + * (ch->gr_pub_armed) so a residual ring from a previous session cannot + * author releases before the session switch is proven. + * --------------------------------------------------------------------- + */ +unsigned gsp_sema_gr_ahead_max = 64; +module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_ahead, + "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)"); + + +static u32 svc_gr_ahead_log; +static u64 svc_gr_ahead_entries; +static u64 svc_gr_ahead_rels; +static u64 svc_gr_ahead_skips; +/* SESSION 127: releases withheld because a live candidate owns the word. */ +static u64 svc_gr_ahead_owned; +/* SESSION 127: bounded witness for the sweep-caller's launch request. */ +static u32 svc_gr_ahead_req_log; +/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */ +static u64 svc_gr_ahead_inline_writes; + +/* + * SESSION 129 util-channel witness: the utility channel's subchannel-2 + * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm + * fired for in-push payloads on the GR channel - i.e. the failure sits in + * the split (tail-served) shape. This struct carries the per-entry story + * out of the translator for one bounded, deduplicated witness line: + * which read source answered (attempt A = the channel path, 1 = the direct + * registered-VAS walk), whether the strict parse accepted the entry, and + * what the LID arm did (saw/fired/refused a still-zero payload). + */ +struct svc_util_wit { + u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */ + u32 pa_ok; /* attempt-A parse (method count) */ + u32 pb_ok; /* attempt-B parse (method count) */ + u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */ + u32 t_len; /* tail length served */ + u32 t0, t1; /* SESSION 130: first two words of the served tail */ + bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */ + bool fired; /* ... and authored its write */ + bool zero; /* ... but the tail payload was still all-zero (pending) */ + u32 dst; + u32 line_len; + u32 line_lc; + u32 count; +}; +static struct svc_util_wit svc_util_wit; +static u64 svc_util_pending_zero; + +/* + * SESSION 127 C5C0 translator, bounded tail materialization buffer: a + * LOAD_INLINE_DATA run whose payload overruns the entry end is served from + * the next ring entry (the driver splits large inline runs into a methods + * entry plus a dedicated data slot entry). 4 KB covers the observed + * param-bank write (1568 B) in full and the first page of the big uploads. + */ +#define SVC_AHEAD_TAIL_INLINE_MAX 32768 +static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; +static u64 svc_ahead_inline_trunc; + +/* + * SESSION 128: one-copy materialization bound for the utility/CE sweep. + * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch + * copy from a client-owned staging buffer into the vidmem shadow) moves + * the driver's code/constant uploads; the observed SASS/code-heap class is + * tens of KB, so 4 MB is a generous cap that still refuses a bogus + * geometry before it can walk the shadow. + */ +#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20) + +/* + * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an + * unprocessed copy-class channel carried - src bytes live in the client's + * staging buffer (guest RAM, resolved through the trusted walk like any + * operand page), dst is the vidmem page the driver staged the upload for + * (code heap / constant store). This is the emulated engine write for the + * copy the real CE would perform; it is once-only keyed by (push_va, + * content hash, dst, bytes) so re-scans of a stable ring are free. + */ +static u64 svc_ahead_copy_in_skip; +static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h, + u64 push_va, u64 hash, u64 src_va, u64 dst_va, + u32 row, u32 rows, u32 dma) +{ + static u8 cb[65536]; + u64 bytes = (u64)row * (rows ? rows : 1); + u64 done = 0; + u64 spa = 0, dpa = 0; + int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS; + int rc = 0; + bool src_ok, dst_ok; + + if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI || + dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI) + return; + /* + * Only the upload direction is materialized: the source must be a + * resolved client page in sysmem/guest RAM (the staging buffer the + * client CPU-wrote), and the destination must be vidmem (resolved to + * the FB aperture) or a not-yet-resolved VA whose bytes the VA + * shadow will hold. A resolved SYS destination is a copy-OUT (the + * engine reading vidmem into a client buffer): replaying it here + * would overwrite the client's buffer with shadow garbage before the + * producing launch ran, so it is skipped - the GR channels' copy-out + * machinery owns that direction. + */ + src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB; + dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) || + dap == SVC_APERTURE_FB; + if (!src_ok || !dst_ok) { + svc_ahead_copy_in_skip++; + if (svc_ahead_copy_in_skip <= 8) + pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n", + src_ok, dst_ok, push_va, src_va, dst_va, + bytes); + return; + } + if (bytes > SVC_AHEAD_COPY_IN_MAX) { + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n", + push_va, src_va, dst_va, bytes, + SVC_AHEAD_COPY_IN_MAX); + return; + } + if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes)) + return; + svc_inline_once_store(push_va, hash, dst_va, (u32)bytes); + while (done < bytes) { + u32 chunk = bytes - done > sizeof(cb) ? + (u32)sizeof(cb) : (u32)(bytes - done); + + rc = svc_read_va(vgpu, src_va + done, cb, chunk); + if (rc) { + rc = -EIO; + break; + } + rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk, + ch_h); + if (rc) + break; + done += chunk; + } + svc_gr_ahead_inline_writes++; + pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n", + src_va, dst_va, bytes, dma, done, rc, ch_h); +} + +/* One 4-byte word of a method stream that may continue into the tail. */ +static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + u32 off) +{ + u32 v = 0; + + if (off + 4 <= len) + memcpy(&v, buf + off, 4); + else if (tail && off >= len && (u64)(off - len) + 4 <= taillen) + memcpy(&v, tail + (off - len), 4); + return v; +} + +/* + * Strict method-stream parse with a one-entry tail (see the sweep comment). + * A run whose payload overruns this entry's end may be completed from the + * next ring entry; anything else invalid means the page is not a method + * stream (the same false-positive arbiter as svc_stream_full_ok). Returns + * the method count, or 0; *used_tail reports that the tail served a + * trailing data run (nothing after it is parsed). + */ +static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + bool *used_tail) +{ + u32 off = 0, methods = 0; + + *used_tail = false; + for (;;) { + u32 hdr, op, count; + u64 end; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 0xfff) + return 0; + end = (u64)off + 4 + (u64)count * 4; + if (end > len) { + if (!tail || (u64)(end - len) > taillen) + return 0; + *used_tail = true; + off = len; + goto done; + } + off = (u32)end; + break; + case 0: /* GRP0_USE_TERT */ + case 2: /* GRP2_USE_TERT */ + case 4: /* IMMD_DATA_METHOD */ + case 7: /* END_PB_SEGMENT */ + off += 4; + break; + default: + return 0; + } + if (++methods > 8192) + return 0; + } +done: + while (off < len && buf[off] == 0) + off++; + return (methods > 0 && (off == len || *used_tail)) ? methods : 0; +} + +/* + * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the + * original release extraction (svc_gr_ahead_push, below), it recognizes, for + * pre-built continuation entries: + * + * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word + * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum + * capture, so the physical GR forward (the CUDA QMD path) fires for a + * launch that never reaches the GP_PUT-bounded main walk - observed + * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`; + * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at + * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized + * into the VA shadow exactly like the main decode does - observed + * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08 + * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23 + * (1568 B -> 0x4b86020, payload in-push); + * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands + * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c + * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as + * pending copy-outs for the completion edge (svc_gr_copy_pend_run) - + * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000, + * 640x480 rows. + * + * Returns the number of release groups queued (as before). The buffer must + * pass the strict tail-aware parse first; a page that is not a method + * stream is not processed at all. + */ +static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + const u8 *buf, u32 len, const u8 *tail, + u32 taillen, u64 push_va, bool inline_only) +{ + static struct svc_push_state stq; /* service thread only */ + static u32 dw[66]; /* QMD window words */ + u32 off = 0, nrel = 0, methods, i; + u64 off_in = 0, off_out = 0; + u32 have_in = 0, have_out = 0; + u32 pitch_in = 0, pitch_out = 0; + u32 line_len = 0, line_count = 0; + u32 last_dma = 0, last_remap = 0; + u32 cp_subch = ~0u; + bool used_tail = false; + bool forward = gsp_phys_gr_forward_enabled(); + bool xlate = gsp_sema_gr_xlate; + + if (!xlate) { + /* Historical sweep: releases from exactly-covered entries. */ + tail = NULL; + taillen = 0; + } + methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail); + if (!methods) + return 0; + + memset(&stq, 0, sizeof(stq)); + stq.push_va = push_va; + + while (off + 4 <= len) { + u32 hdr, op, addr, count, subch; + u32 w0, w1, w2, w3, w4; + u64 end; + + memcpy(&hdr, buf + off, 4); + if (!hdr) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + addr = (hdr & 0x1fff) << 2; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + if (op == 0 || op == 2 || op == 4 || op == 7) { + off += 4; + continue; + } + if (op != 1 && op != 3 && op != 5) + break; + if (!count || count > 0xfff) + break; + end = (u64)off + 4 + (u64)count * 4; + /* + * Tail-served trailing run: process from the head word, then + * stop (the tail is a data slot, nothing parses after it). + */ + w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4); + w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8); + w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12); + w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16); + w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20); + + /* + * C5C0 inline-QMD window (mirrors the main decode's intercept): + * only accumulate once the run declares itself a QMD window. + */ + if (xlate && forward && !inline_only && subch >= 1 && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (stq.qmd_have || + (addr <= SVC_QMD_ADDR_B && + (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) { + u32 n = count > 66 ? 66 : count; + + for (i = 0; i < n; i++) + dw[i] = svc_ahead_w32(buf, len, tail, taillen, + off + 4 + 4 * i); + svc_qmd_accum(vgpu, ch, &stq, addr, n, + (const u8 *)dw, buf, len, true, false); + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the + * DMA write (the engine would do this; the emulation authors + * the bytes into the VA shadow so the forwarded launch's + * operand pages - parameter banks - are real). + */ + if (xlate && op == 3 && subch >= 1 && + addr == GRCE_M_LOAD_INLINE_DATA && + have_out && line_len) { + u64 nbytes = (u64)line_len * + (line_count ? line_count : 1); + u32 cap = count * 4; + u32 bytes, k, words; + u64 h = svc_push_hash(buf, len); + bool tail_served = end > len; + bool refuse_zero = false; + + if (nbytes > cap) + nbytes = cap; + bytes = (u32)nbytes; + if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) { + bytes = SVC_AHEAD_TAIL_INLINE_MAX; + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n", + push_va, off_out, nbytes, + SVC_AHEAD_TAIL_INLINE_MAX); + } + words = (bytes + 3) / 4; + for (k = 0; k < words; k++) + memcpy(svc_ahead_inline_buf + 4 * k, + &(u32){ svc_ahead_w32(buf, len, tail, + taillen, + off + 4 + 4 * k) }, + 4); + /* + * SESSION 129: a tail-served payload whose head is still + * all zeros means the tail source answered with an empty + * page (a stale pin shadow, or the driver has not staged + * the bytes yet) - authoring zeros over the destination + * would corrupt the code heap. Leave the write pending + * (no once-store, the sweep retries every pass) until a + * nonzero payload arrives. In-push payloads are never + * gated: the bytes are part of the flushed submit. + */ + if (inline_only && tail_served && bytes && + !memchr_inv(svc_ahead_inline_buf, 0, + min_t(u32, bytes, 16))) + refuse_zero = true; + if (inline_only) { + svc_util_wit.saw_lid = true; + svc_util_wit.dst = (u32)off_out; + svc_util_wit.line_len = line_len; + svc_util_wit.line_lc = line_count; + svc_util_wit.count = count; + if (refuse_zero) + svc_util_wit.zero = true; + } + if (refuse_zero) { + /* + * SESSION 130: a zero-headed tail-served + * payload is applied under a VARIANT once-key + * (h | 1) instead of being refused forever: + * the destination is currently empty, so + * authoring zeros is idempotent and correct + * for the driver's internal zero buffer (the + * observed 256 B utility i2m = exactly + * GF100_ZERO_BUFFER_SIZE), while a later + * non-zero payload for the same entry still + * matches its own (h) key and lands - the + * refusal can no longer stall the arm + * (r737/r739: `writes=0` with the payload + * page stably zero). + */ + svc_util_pending_zero++; + if (bytes && !svc_inline_once_seen(push_va, + h | 1, + off_out, + bytes)) { + svc_inline_once_store(push_va, h | 1, + off_out, bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, + off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + } else if (bytes && !svc_inline_once_seen(push_va, h, + off_out, + bytes)) { + svc_inline_once_store(push_va, h, off_out, + bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + /* + * Record the destination and the fresh payload bytes + * for the launch fill's inline-served pages, exactly + * like the main decode's LOAD_INLINE_DATA arm: the + * QMD capture later in this entry collects the ring + * and the fill can serve the parameter bank from it + * even while its shadow page is still empty. + */ + { + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + /* + * A refused (still-zero) tail payload is not + * recorded either: the next pass re-reads the + * tail and the real bytes land then. + */ + if (!refuse_zero) { + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } + } + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for + * subchannel 1 AND 2. The mem2mem subchannel (2, the + * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility + * channels drive) carries the same operand methods as the + * compute subchannel (1). This group MUST be parsed BEFORE + * the TURING-copy branch below: that branch's `subch >= 2` + * guard also matches subchannel 2 and would swallow the + * 0x180..0x18c methods unread, leaving have_out/line_len + * unset - the exact reason the utility channel's subch-2 + * LOAD_INLINE_DATA arm never fired (r737/r739 witness: + * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`). + */ + if (xlate && (subch == 1 || subch == 2) && + addr >= 0x180 && addr <= 0x18c && count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } else if (xlate && subch >= 2) { + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + if (addr >= 0x400 && addr <= 0x41c && n) { + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x400: + off_in = (off_in & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_in = 1; + break; + case 0x404: + off_in = (off_in & + ~0xffffffffULL) | + vv[k]; + have_in = 1; + break; + case 0x408: + off_out = (off_out & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x40c: + off_out = (off_out & + ~0xffffffffULL) | + vv[k]; + have_out = 1; + break; + case 0x410: + pitch_in = vv[k]; + break; + case 0x414: + pitch_out = vv[k]; + break; + case 0x418: + line_len = vv[k]; + break; + case 0x41c: + line_count = vv[k]; + break; + default: + break; + } + } + } else if (addr >= 0x700 && addr <= 0x708 && n) { + for (k = 0; k < n; k++) { + if (addr + 4 * k == 0x700) + last_remap = (last_remap & + ~0xffffu) | + (vv[k] & 0xffffu); + else if (addr + 4 * k == 0x708) + last_remap = (last_remap & + 0xffffu) | + ((vv[k] & 0xffffu) + << 16); + } + } else if (addr == 0x300 && count) { + u32 dma = w0; + + /* + * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a + * release-only / semaphore op - no data + * movement, nothing to record). + */ + last_dma = dma; + if ((dma & 0x3) && have_in && have_out && + line_len && line_count && + off_in && off_out) { + if (inline_only) + svc_ahead_copy_in(vgpu, + ch->h_object, push_va, + svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, + line_count, dma); + else if (forward) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, + line_count, + pitch_out ? pitch_out : + line_len * 4, + pitch_in, dma, + last_remap); + cp_subch = subch; + } + } + } + } + + if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u64 va; + + va = ((u64)(w1 & 0xff) << 32) | + (u64)(w0 & 0xfffffffc); + op = w4 & 0x7; + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && w2 && + va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { + /* + * A release targeting a live launch candidate's + * own QMD release word is owned by the + * completion edge (post-copy-out); sweeping it + * here would clear the client's fence before + * the engine output landed. + */ + if (xlate && forward && + svc_gr_ahead_owned_rel_va(va, push_va)) { + svc_gr_ahead_owned++; + } else { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? + 1 : 0); + nrel++; + } + } + } + if (end > len) + break; + off += 4 + count * 4; + } + (void)cp_subch; + return nrel; +} + +static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + static u8 abuf[2048]; + static u8 nbuf[2048]; + u32 i, maxent, nrel = 0, swept = 0, nskip = 0; + + if (!put || !ch->gpfifo_entries) + return; + maxent = min_t(u32, gsp_sema_gr_ahead_max, 512); + for (i = 0; i < maxent; i++) { + u32 idx = (put + i) % ch->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen, tail_len = 0; + u64 va; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + break; /* unwritten slot: end of the continuation */ + if (!va || !len || len > SVC_PUSH_MAX) + break; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + break; + rlen = min_t(u32, len, sizeof(abuf)); + if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) { + /* + * Unreadable here (pre-anchor translation) is a + * reason to stop this pass: the continuation is not + * proven coherent, so nothing is authored. + */ + nskip++; + break; + } + /* + * SESSION 127 C5C0 translator: read the next ring entry as + * the tail of a stream whose final inline run overruns this + * entry (the driver splits large LOAD_INLINE_DATA payloads + * into a dedicated data slot entry; observed beef0100 + * idx00->idx01 and idx08->idx09). + */ + { + u32 nidx = (put + i + 1) % ch->gpfifo_entries; + u8 nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 ne0, ne1, nlen; + u64 nva; + + if (!svc_read_va(vgpu, + ch->gpfifo_va + + (u64)nidx * SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent))) { + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, + sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, + nbuf, tail_len)) + tail_len = 0; + } + } + } + swept++; + nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, + tail_len, va, false); + } + svc_gr_ahead_entries += swept; + svc_gr_ahead_rels += nrel; + svc_gr_ahead_skips += nskip; + if (swept && (nrel || svc_gr_ahead_log < 64)) { + svc_gr_ahead_log++; + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, svc_gr_ahead_owned, + nskip, svc_gr_ahead_rels); + } +} + +/* + * SESSION 129: read a VA straight through the registered-VAS walks only + * (newest registration first), one page at a time - the exact path + * walk_probe and sema_probe prove. The util sweep's channel read can be + * shadowed by a stale injected pin / window guess for the utility VAs + * (observed r737: the subchannel-2 LID arm never fired although the push + * page content is real and the VAS walk resolves both the methods entry + * and its payload slot); this helper is the arbitration fallback that + * bypasses those sources entirely. + */ +static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t off = va & (SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used + * by the bounded util-sweep witness to select only the entries that can + * exercise the arm (one per distinct content, so a stable ring logs once). + */ +static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch) +{ + u32 k, n = 0; + + *first_subch = 0xff; + for (k = 0; k + 4 <= len; k += 4) { + u32 hdr; + + memcpy(&hdr, buf + k, 4); + if (((hdr >> 29) & 0x7) == 3 && + ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) { + if (*first_subch == 0xff) + *first_subch = (hdr >> 13) & 0x7; + n++; + } + } + return n; +} + +/* + * SESSION 128: utility-channel inline-download materialization. + * + * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D + * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered + * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in + * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA + * downloads they carry (the client's code-heap/SASS upload, command-stream + * staging) never land anywhere the emulation can see: the destinations + * stay zero in the vidmem shadow. + * + * This pass scans a bounded window of the channel's GPFIFO directly (the + * gpfifo VA resolves through the trusted walk even when the USERD does + * not) and feeds each entry + its successor (the source-view call carrying + * the payload) through the proven sweep decoder in inline-only mode: the + * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2), + * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by + * svc_grce_inline_write; releases, QMD windows and copy-pend records are + * skipped so nothing about the client's fence/completion state changes. + * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring + * costs only the reads. Requires sema_gr_xlate; bounded by + * sema_gr_util_scan entries per channel per pass (0 = off). + * + * SESSION 129 read arbitration: attempt A is the channel path (unchanged); + * when A fails to read, or reads content the strict parse rejects, attempt + * B re-reads the push and the tail through svc_util_walk_read and the + * parse arbitrates - a source that yields a method stream wins. The r737 + * evidence (arm never fired while the content is real) pointed at exactly + * this: a shadowed channel read consumed the entry without a parseable + * buffer while the walk resolves the true pages. + */ +static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ +/* + * The observed utility-channel pushes are the ~17 KB setup block and the + * two split uploads (256 B command stream + 12 KB code/heap payload); a + * 64 KB window covers every one of them with room to spare while keeping + * the two static buffers modest. + */ +#define SVC_UTIL_BUF (64 * 1024) + static u8 ubuf[SVC_UTIL_BUF]; + static u8 nbuf[SVC_UTIL_BUF]; + static u8 bbuf[SVC_UTIL_BUF]; + static u8 tbuf[SVC_UTIL_BUF]; + static u64 svc_gr_util_entries; + static u64 svc_gr_util_writes; + static u64 svc_util_tail_alt; /* SESSION 130: tails re-read through + * the direct walk because the + * channel read was zero-headed */ + static u64 svc_util_tail_try; /* ... and the walk attempts (alt is + * the subset that yielded non-zero) */ + u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64); + u32 swept = 0, nmade = 0; + bool first_sight = false; + + if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan) + return; + if (!ch->gpfifo_va || !ch->gpfifo_entries) { + /* SESSION 143: name every channel the sweep cannot cover. */ + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + pr_info("sema-utilsweep: skip h=0x%x eng=%u reason=no-gpfifo gpfifo=0x%llx entries=%u userd=%d\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0); + } + return; + } + { + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + first_sight = true; + pr_info("sema-utilsweep: begin h=0x%x eng=%u gpfifo=0x%llx entries=%u userd=%d cid=%u\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0, + ch->cid); + } + } + for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen; + u64 va, before; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + continue; /* unwritten slot */ + if (!va || !len || len > SVC_PUSH_MAX) + continue; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + continue; + /* + * SESSION 132 stale-translation arbitration: when the + * channel-path entry is dead (its push provably reads + * all-zero through the client's page tables) while the + * direct VAS walk answers a live entry on the same ring + * slot, the channel path is reading a stale page (the r759 + * pin false positive) - the walk's entry is the channel's + * ring. A dead entry with no live walk alternative is left + * to the existing push arbitration below. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) { + u8 went[SVC_GPFIFO_ENTRY_SIZE]; + + if (!svc_util_walk_read(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + went, sizeof(went))) { + u32 we0, we1; + u64 wva; + u32 wlen; + + memcpy(&we0, went, 4); + memcpy(&we1, went + 4, 4); + wva = ((u64)(we1 & 0xff) << 32) | + (u64)(we0 & 0xfffffffc); + wlen = ((we1 >> 10) & 0x1fffff) << 2; + if (!svc_ring_entry_dead(vgpu, we0, we1) && + wva && wlen && wlen <= SVC_PUSH_MAX && + wva >= SVC_LOW_REL_VA_LO && + wva < SEMA_VA_HI) { + e0 = we0; + e1 = we1; + va = wva; + len = wlen; + } + } + } + rlen = min_t(u32, len, sizeof(ubuf)); + /* + * SESSION 129 read arbitration + bounded witness (see the + * sweep's header comment). Attempt A: the channel path, + * unchanged. Attempt B: the direct registered-VAS walk, used + * when A cannot read or cannot parse; the strict parse then + * arbitrates which content is the method stream. + */ + { + u32 push_rc, parse_a = 0, parse_b = 0; + u32 tl_a = 0, tl_b = 0; + u32 lidn, lidsc = 0xff, slot; + u64 nva = 0; + u32 nlen = 0; + u64 hkey; + u8 *pbuf = ubuf; + bool ut = false, seen = false; + + push_rc = svc_read_va_push(vgpu, ch, va, ubuf, + rlen); + /* + * The payload of a trailing LOAD_INLINE_DATA run rides + * the next GPFIFO entry (the driver's source-view + * GPFIFO call, observed beef0100 idx00->idx01 and + * idx08->idx09): read it as the tail exactly like + * svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1; + + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + } + if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) { + tl_a = min_t(u32, nlen, sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, nbuf, + tl_a)) + tl_a = 0; + } + if (!push_rc) + parse_a = svc_ahead_parse(ubuf, rlen, nbuf, + tl_a, &ut); + if (push_rc || !parse_a) { + bool bpush = false; + + if (nva && nlen && nlen < SVC_PUSH_MAX) { + u32 want = min_t(u32, nlen, + sizeof(nbuf)); + + if (!svc_util_walk_read(vgpu, nva, + nbuf, + want)) + tl_b = want; + } + if (!svc_util_walk_read(vgpu, va, bbuf, + rlen)) { + bpush = true; + parse_b = svc_ahead_parse(bbuf, rlen, + nbuf, tl_b, + &ut); + /* + * A's buffer is authoritative only + * when A actually read it; otherwise + * (or when only B parses) B's walk + * content is the stream. + */ + if (push_rc || + (parse_b && !parse_a)) { + pbuf = bbuf; + tl_a = tl_b; + } + } + if (push_rc && !bpush) + continue; /* unreadable both ways */ + } + /* + * SESSION 130 tail-content arbitration: a tail-served + * payload whose channel head is still all-zero is not + * the staged bytes (the page the channel translation + * points at is empty, or the read is shadowed by a + * stale pin). Try the direct registered-VAS walk - + * the same path attempt B proves - before handing + * the tail to the LID arm, whose zero-headed refusal + * would otherwise stall the upload every pass. + */ + if (tl_a && nva && + !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) { + u32 want = tl_a; + + svc_util_tail_try++; + if (!svc_util_walk_read(vgpu, nva, tbuf, + want) && + memchr_inv(tbuf, 0, + min_t(u32, want, 16))) { + memcpy(nbuf, tbuf, want); + svc_util_tail_alt++; + } + } + memset(&svc_util_wit, 0, sizeof(svc_util_wit)); + if (tl_a >= 4) + memcpy(&svc_util_wit.t0, nbuf, 4); + if (tl_a >= 8) + memcpy(&svc_util_wit.t1, nbuf + 4, 4); + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf, + tl_a, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; + /* + * Bounded witness, one line per distinct LID-carrying + * (or unparseable) entry content: which attempt was + * used, the parses, the tail resolution and what the + * LID arm did. + */ + lidn = svc_util_lid_scan(pbuf, rlen, &lidsc); + if (!lidn && parse_a) + continue; + hkey = svc_push_hash(pbuf, rlen); + for (slot = 0; slot < 32; slot++) + if (svc_util_dec_keys[slot].h == hkey && + svc_util_dec_keys[slot].va == va) { + seen = true; + break; + } + if (seen || svc_util_dec_log >= 96) + continue; + slot = svc_util_dec_log & 31; + svc_util_dec_keys[slot].va = va; + svc_util_dec_keys[slot].h = hkey; + svc_util_dec_log++; + pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu/%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n", + ch->h_object, i, va, rlen, + pbuf == bbuf ? 1u : 0u, parse_a, parse_b, + nva, tl_a, svc_util_wit.t0, svc_util_wit.t1, + svc_util_tail_alt, svc_util_tail_try, + lidn, lidsc, + svc_util_wit.saw_lid ? 1 : 0, + svc_util_wit.fired ? 1 : 0, + svc_util_wit.zero ? 1 : 0, + svc_util_wit.dst, svc_util_wit.line_len, + svc_util_wit.line_lc, + rlen >= 4 ? *(const u32 *)pbuf : 0, + rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0); + } + } + svc_gr_util_entries += swept; + svc_gr_util_writes += nmade; + if (first_sight && !swept) + pr_info("sema-utilsweep: read=none h=0x%x gpfifo=0x%llx entries=%u\n", + ch->h_object, ch->gpfifo_va, ch->gpfifo_entries); + if (swept && (nmade || svc_gr_util_log < 128)) { + svc_gr_util_log++; + pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n", + ch->h_object, swept, nmade, svc_gr_util_entries, + svc_gr_util_writes); + } +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { + if (svc_pdb_disc_pending) { + svc_pdb_disc_pending = false; + pr_info("sema-pdb-disc: requested va=0x%llx pa=0x%llx\n", + svc_pdb_disc_va, svc_pdb_disc_pa); + svc_vas_candidates_verify(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + svc_xvas_pdb_discover(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + } /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* @@ -23523,6 +26799,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + if (svc_rewind_pending) { + svc_rewind_pending = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + if (!svc_rewind_all && c->h_object != svc_rewind_h) + continue; + pr_info("sema-chan: rewind h=0x%x last_get=%u put=%u (slot bitmap + rescue state cleared)\n", + c->h_object, + (unsigned int)c->last_get, + (unsigned int)c->put_prev); + bitmap_zero(c->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + c->last_get = 0; + c->rescue_fail = 0; + c->unread_va = 0; + c->unread_until = 0; + c->defer_since = 0; + } + } + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); /* * r507o stream-order: walk the CE channels BEFORE the GR channels. * The client's stream order places its H2D copies before the kernel @@ -23709,8 +27015,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_hold.ch_handle == ch->h_object) && !svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, - ch->userd_as, &pput, sizeof(pput))) + ch->userd_as, &pput, sizeof(pput))) { + if (gsp_sema_trace_h && + (u32)ch->h_object == + (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pass h=0x%x pput=%u last=%u armed=%d cnt=%u sub=%llu held=%llu starved=%llu\n", + ch->h_object, pput, + (u32)ch->last_get, + ch->gr_pub_armed, + ch->counter_last, + ch->submissions, + svc.gpfifo_held, + svc.gpfifo_starved); + } svc_counter_publish(vgpu, ch, pput); + } } /* @@ -23740,6 +27061,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!rrc) svc_pb_pair(vgpu, ch); + if (!rrc && ring_pa) { + svc_lastgood_ring_va = ch->gpfifo_va; + svc_lastgood_ring_pa = ring_pa; + } + /* + * r707f (FOLD): the moment ANY channel's ring resolves through + * the channel object, verify the parked COPY_SERVER_PDES VAS + * candidates against that content-backed pair. The candidates + * are parked at device-create time, before the client's compute + * channels first submit, so registering here lands the VAS page + * directory BEFORE the submit's entries get their first push + * read - the window the host-triggered path could not win (the + * entry's first read and the ring's first resolution happen in + * the same pass; a pair that arrives later cannot help, because + * a failed GR entry read is never retried with the fresh VAS). + */ + if (!rrc) + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + ring_pa); /* * ISSUE-0002 (r47 diag): run the deferred CE USERD sweep for a @@ -23760,6 +27100,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies + 27 * HZ; } + /* + * SESSION 128: utility/upload channels carry the engine's + * inline code/data downloads (LOAD_INLINE_DATA on the + * mem2mem subchannel) and the copy-class channels (eng=11/12) + * carry the driver's pitch copy-ins (staging -> code heap / + * constant store). With an unresolved USERD the put-bounded + * entry loop below never decodes them (the !userd_ok branch + * continues before it); with a resolved USERD the main + * decode's LOAD_INLINE_DATA/operand arms still only accept + * subchannel 1, so the subchannel-2 downloads are missed + * either way. Scan every channel's GPFIFO directly here and + * materialize the downloads (see svc_gr_util_sweep); the + * inline-only mode plus the once-only key make the overlap + * with the main loop and the GR sweep a no-op. + */ + if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate) + svc_gr_util_sweep(vgpu, ch); + if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get, @@ -23784,6 +27142,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r832: the RC watchdog's notifier ack must not depend on the + * ring reads. Its VA is known from the channel's first decode + * (ch->notify2d_va, captured from SET_NOTIFY_A/B) and the ack + * is the same zeroed NvNotification the decode path writes: + * the guest re-arms status=0xFFFF every cycle and checks it + * back to 0 within 7 s. Under heavy GL traffic the watchdog + * ring's translation can go stale for stretches (its entries + * then read as zeros and are consumed as empty), which starved + * exactly this write and wedged the guest with "GPU is + * probably locked" while every other channel kept running. + * Writing the ack on every pass that sees work pending is + * idempotent. + */ + if (!ch->engine_type && ch->notify2d_va && put != get) + svc_notify2d_write(vgpu, ch); + + /* * ISSUE-0002 diag: put snapshots for CE channels, bounded per * session. The throttle is indexed by rank (the channel struct * iterated here is a local copy, so a per-channel stamp never @@ -23977,6 +27352,50 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * SESSION 126 fence fold: sweep the ring continuation beyond + * GP_PUT for semaphore releases (see svc_gr_ahead_sweep). + * GR-class channels only, live-armed epochs only, and only + * with the ring translation proven through the channel + * object (rrc == 0). + */ + if (!ch->engine_type && ch->gr_pub_armed && + gsp_sema_gr_ahead_max && !rrc && ring_pa) { + svc_gr_ahead_sweep(vgpu, ch, put); + /* + * SESSION 127 C5C0 translator: offer a launch the + * sweep captured from the continuation (the Vulkan + * client's inline-QMD push lives beyond GP_PUT, so + * the main walk's per-entry request never sees it). + * svc_gr_request is a no-op without a matching + * unoffered candidate and keeps it on a busy + * verdict, so a bounded per-pass retry here is the + * sweep-captured launch's entire dispatch path. + */ + if (gsp_sema_gr_xlate && + gsp_phys_gr_forward_enabled()) { + if (svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + svc_gr_ahead_req_log < 8) { + svc_gr_ahead_req_log++; + pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n", + ch->h_object, + svc_gr_cand.push_va, + svc_gr_cand.offered); + } + (void)svc_gr_request(vgpu, ch); + } + /* + * Fallback copy-out retirement: when no forward is + * owed (the launch was already consumed or the + * backend is off), run any pending GRCE copies so + * they cannot sit forever (the completion edge is + * the primary path). + */ + if (gsp_sema_gr_xlate && !svc_gr_forward_pending()) + svc_gr_copy_pend_run(vgpu); + } + + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. * Runs before the entry loop so a wedged session epoch is * recovered (and logged) instead of spinning: see @@ -24018,6 +27437,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) get_before = ch->last_get; applied_before = svc.releases_applied; + /* r835: this channel's VAS answers its VAs first (see + * svc_chanvas_lookup). Refreshed per channel turn; a channel + * without a known VAS clears it back to the old chain. */ + svc_cur_vas = ch->h_vaspace; /* * C4 (GAP-1 H1): the GR walk witness - does put advance, and @@ -24182,6 +27605,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r829: a zero-length / zero-VA entry is a bad read + * (a stale or wrong translation), never work. The old + * path consumed it as an empty push and advanced + * GPGet; observed r830 on the RC watchdog channel: + * two bogus zero entries consumed through a + * mis-resolved ring page pushed the userd's GPGet past + * the driver's own GPPut, every later userd probe then + * read get>put and the channel's REAL userd was + * rejected - the channel stopped being serviced and + * the guest declared the GPU locked. Hold the entry + * instead (bounded; the per-channel counter persists + * through the pass snapshot); a corrected translation + * (the ring auto-pin lands within seconds) re-reads + * it. Past the cap the old consume-as-empty behavior + * resumes so an unresolvable ring cannot livelock the + * channel. + */ + if (va == 0 || len == 0) { + if (ch->zero_held < 512 && !svc_hold_bypass()) { + ch->zero_held++; + svc.gpfifo_held++; + if (gsp_sema_dbg && + svc_zero_entry_log < 16) { + svc_zero_entry_log++; + pr_info("sema-chan: zero-entry hold h=0x%x idx=%llu raw=%08x%08x (bad ring translation; not consumed)\n", + ch->h_object, idx, + e1, e0); + } + stalled = true; + continue; + } + } + + /* * Cheap wait for an entry already inside its * expensive-wait window: read + rescue + hold were * already paid for this VA, and a pin-batch rewind @@ -24229,9 +27686,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; - bool unreadable = - svc_read_va_push(vgpu, ch, va, - pb, len) != 0; + int rd_rc; + bool unreadable; + bool zero_nop = false; + bool displaced = false; + bool have_ipin = false; + + svc_read_last_src = 0xff; + svc_read_last_pa = 0; + rd_rc = svc_read_va_push(vgpu, ch, va, + pb, len); + unreadable = rd_rc != 0; + if (gsp_sema_dbg && svc_rdw_log < 6000 && + !ch->engine_type) { + svc_rdw_log++; + pr_info("sema-rdw: h=0x%x va=0x%llx len=%u rc=%d src=%u\n", + ch->h_object, va, len, + rd_rc, + (unsigned)svc_read_last_src); + } bool injected = false; bool allow_ops = (ch->engine_type != 0); @@ -24256,8 +27729,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 ipa; int iap; - injected = (svc_ipin_lookup(va, &ipa, - &iap) == 0); + have_ipin = + (svc_ipin_lookup(va, &ipa, + &iap) == 0); + /* + * r715: the entry's treatment + * follows the source that + * ACTUALLY answered the read, + * not a fresh page-granularity + * ipin lookup. The read falls + * through to the trusted walk + * (src 4) whenever an ipin's + * page fails its first-chunk + * guard, and taking the + * injected branch then skipped + * the only source that could + * answer: the entry was + * consumed without ever + * consulting the registered + * VAS walk (session 124). + */ + injected = have_ipin && + (svc_read_last_src == 0); /* * r496e: an all-zero push on a * content-scanned-ring channel is the @@ -24265,8 +27758,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * dword - NVA06F_DMA_NOP), not a bad * translation: accept it and let the * decode find no ops. + * r820/r826: the same push arrives on + * KERNEL channels (nvidia-push's + * __nvPushTestPushBuffer writes the + * NOP into the kernel pushbuffer); its + * promote entry is a booking, so the + * trusted walk answers - a SMALL + * all-zero GR push resolved by the + * trusted chain (src=4) is the NOP. + * Real launches are large (a QMD push + * is hundreds of bytes), so the size + * bound keeps an unread legitimate + * push from being consumed as empty. */ - bool zero_nop = ch->gr_ram_ring && + zero_nop = (ch->gr_ram_ring || + (!ch->engine_type && + svc_read_last_src == 4 && + len <= 16)) && len >= 4 && !memchr_inv(pb, 0, len); @@ -24299,7 +27807,82 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(fb); } } else { - unreadable = true; + /* + * r707v: a primary-source + * hit (injected pin, window + * guess, lat cache) whose + * content is implausible + * must not shadow the + * registered VAS walk - a + * stale pin otherwise makes + * every push read "resolve" + * to the wrong page, fail + * plausibility and turn the + * entry unreadable forever. + * Retry once with the pins + * bypassed so the trusted + * walk (svc_walk_vaspaces) + * answers; accept only a + * plausible stream. + */ + u8 *vb = kmalloc(len, + GFP_KERNEL); + + if (vb) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool ok = false; + + /* + * r707w: consult ONLY the + * registered VAS walk (the + * exact path walk_probe + * proves) - not + * svc_lookup_trusted, whose + * pin/lat hits would answer + * with the same wrong page + * again and keep the entry + * unreadable. + */ + int wr = svc_walk_vaspaces(vgpu, + va & ~0xfffULL, + &wpa, &wap); + int rr = wr ? wr : svc_read_ap(vgpu, + wpa + (va & 0xfff), + wap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + vb, len); + bool pl = !rr ? svc_method_stream_plausible(vb, len) + : false; + + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk rc=%d pa=0x%llx plaus=%d\n", + ch->h_object, va, + wr ? wr : rr, + wpa + (va & 0xfff), + pl ? 1 : 0); + } + if (!wr && !rr && pl) { + memcpy(pb, vb, len); + svc.pushpg_fallback++; + ok = true; + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk ok pa=0x%llx\n", + ch->h_object, va, + wpa + (va & 0xfff)); + } + } + kfree(vb); + if (!ok) + unreadable = true; + } else { + unreadable = true; + } } /* * Still implausible (or @@ -24315,18 +27898,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 *db = kmalloc(len, GFP_KERNEL); - if (db && - svc_push_read_displaced(vgpu, - ch->h_object, va, - db, len)) { - memcpy(pb, db, len); - injected = false; - unreadable = false; - } - kfree(db); - } - } - } + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + displaced = true; + } + kfree(db); + } + } + } /* * The entry's page was unreadable through @@ -24345,11 +27929,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, db, len); injected = false; unreadable = false; + displaced = true; } kfree(db); } /* + * r715 pushdec witness: name the branch + * the entry takes straight after the + * read, before the rescue/hold paths. + * Session 124 had an entry consumed + * between the read (rc=0 src=4) and the + * decode with no row naming why; this + * line makes the branch and the content + * (first 8 bytes) visible on every read. + * Budget: 512 rows per client (reset in + * svc_session_flush). + */ + if (gsp_sema_dbg && svc_pushdec_log < 2048) { + svc_pushdec_log++; + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + svc_method_stream_plausible(pb, len) ? 1 : 0, + injected ? 1 : 0, + have_ipin ? 1 : 0, + zero_nop ? 1 : 0, + displaced ? 1 : 0, + unreadable ? 1 : 0, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + + /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA @@ -25195,7 +28808,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && (svc_gr_qmd_seen > 0 || - all_zero) && + (all_zero && !zero_nop)) && (!entry_replay || svc_gr_cand_push_owed(va)) && !(svc_gr_cand.valid && @@ -25503,6 +29116,31 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * GR stale-skip clock: the + * rescue-failure skip gate below + * requires defer_since, yet every + * GR arming site is gated on a + * nonzero expected_sem_payload. A + * zero-expectation GR entry (its + * release was already delivered) + * therefore never satisfied the + * gate and was re-read on every + * pass while the channel ring + * stalled behind it (live + * evidence: one 96-byte entry + * re-read 362 times over ~18 s; + * ~8 s of ring stall per frame + * with frames at 0.045 FPS). Arm + * the clock at first sighting; the + * skip window itself stays + * governed by gsp_sema_gr_grace_ms. + */ + if (!ch->engine_type && + !ch->expected_sem_payload && + !ch->defer_since) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_gr_grace_ms + 1); + /* * Open the cheap-wait window * for this entry (see the * loop-top guard) so rewind @@ -26072,6 +29710,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_logged = ch->gr_ram_logged; svc.chan[k].gr_ram_ring = ch->gr_ram_ring; svc.chan[k].userd_scans = ch->userd_scans; + svc.chan[k].notify2d_va = ch->notify2d_va; + svc.chan[k].zero_held = ch->zero_held; + svc.chan[k].userd_reg = ch->userd_reg; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -26485,12 +30126,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } - pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", + pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, + ch->gr_pub_armed, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); @@ -26992,19 +30634,43 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) u32 put = 0; u64 delta; - if (!s || !s->userd_ok || !s->pa_ok || !s->entries) + if (!s) + return; + if (!s->userd_ok || !s->pa_ok || !s->entries) { + /* m112b: silent-return diagnostic (bounded per boot). */ + if (s->log_budget < 8) { + s->log_budget++; + pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n", + (void *)vgpu, (int)(s - svc_scrubs), + s->userd_ok, s->pa_ok, s->entries, + s->session_gen); + } return; + } if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { /* Retry through the alternate aperture (as registration did). */ if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB, - &put, sizeof(put))) + &put, sizeof(put))) { + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n", + s->userd_pa, s->userd_as); + } return; + } } - if (put >= s->entries) + if (put >= s->entries) { + /* m112b: junk-translation diagnostic (bounded). */ + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n", + put, s->entries, s->userd_pa); + } return; /* junk translation; retry next pass */ + } if (!s->seeded) { s->seeded = true; @@ -27205,6 +30871,26 @@ static void svc_stall_watch_check(void) svc_force_consume = 4; } +/* + * ISSUE-0002 client-session boundary aid: force a session flush that + * executes even while channels are live. The channel-close trigger only + * fires when the table empties, which never happens on a guest where the + * driver-global kernel channels (RC watchdog 0x31415900, the tiny notifier + * channel) stay valid; later client sessions then inherit the previous + * session's map/pin/pool discovery state and wedge (false translations -> + * failed rescues -> driver DMA-init timeout). Raised through the + * sema_force_flush module parameter so the boundary can be tested before + * wiring automatic detection. + */ +void gsp_sema_service_sess_flush_force(void) +{ + WRITE_ONCE(svc_sess_flush_forced, 1); + WRITE_ONCE(svc_sess_flush, 1); + if (READ_ONCE(svc.task)) + wake_up_process(svc.task); + pr_info("sema-svc: forced session flush requested\n"); +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; |