diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1401 |
1 files changed, 1327 insertions, 74 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8fd602c..cc508dc 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -786,6 +786,12 @@ static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ +/* SESSION 133: ranks that already consumed their one shadow ring scan. */ +static unsigned long svc_gr_ring_scan_done; +static unsigned long svc_gr_ring_scan_last; /* rate limit for the scan */ +/* SESSION 138: bounded witness for the GR request builder's page list and + * the fill's candidate set (one launch's wiring printed per budget slice). */ +static u32 svc_gr_pages_log; static bool svc_pb_first_push(const u8 *buf) { @@ -1866,15 +1872,48 @@ module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644); * SESSION 127 C5C0 translator gate. When 1, the sweep additionally * translates continuation entries into launch work (tail-aware method * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5 - * copy-out capture, candidate-owned release withholding). Default 0 keeps - * the historical sweep behaviour for the CUDA batteries; the Vulkan - * translator cycles pass sema_gr_xlate=1. Flip the default once the CUDA - * battery is proven clean with it on. + * copy-out capture, candidate-owned release withholding). + * + * SESSION 130 re-test of the r727 "translator regressions" (translator on, + * utility sweep OFF, r746): victory_saxpy and victory_math now PASS with + * checksums identical to the gate-off baseline (saxpy + * csum=981440890.500000), so the saxpy/math half of the r727 regression + * is FIXED (the SESSION 129/130 sweep changes); victory_mail PASSES. The + * remaining xlate-on regression is victory_multikernel: its launch is + * captured and submitted to the physical channel but the forward times + * out repeatedly (`GR LAUNCH FAILED: timeout after 2000 ms + * (qmd_rel=0xc0ffee01 want=0x600df00d ...) h=0x5c000019 + * push=0x20043c394`, `hold active`), so the client hangs - the default + * stays 0 until that forward path is fixed. The UTILITY SWEEP + * (`sema_gr_util_scan`) is the OTHER regression source: at 24 it broke + * saxpy (`bad=1047527`) and hung math on r745, so it stays opt-in + * (default 0) until its channel reads prefer the direct walk over pins. */ static bool gsp_sema_gr_xlate; module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644); MODULE_PARM_DESC(sema_gr_xlate, "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)"); +/* + * SESSION 128 utility-channel inline-download materialization + * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload + * channels (the MEM2MEM/A140 + 2D utilities, registered but with an + * unresolved USERD so the pump's put-bounded entry loop never decodes + * them) carry the engine-side code/data downloads: the + * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180.. + * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose + * payload the engine fetches from the next GPFIFO entry (the + * source-view call). Without this pass those destinations stay zero in + * the vidmem shadow - observed: the Vulkan client's code-heap/SASS + * upload (12 KB) and command-stream staging (256 B) never land, so the + * forwarded launch has no program content. Bounded: scan up to N + * entries per channel per pass, materialize once-only keyed by + * (push_va, content hash, dst, bytes) via the same + * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate. + */ +static u32 gsp_sema_gr_util_scan; +module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644); +MODULE_PARM_DESC(sema_gr_util_scan, + "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)"); MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)"); static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) @@ -3056,6 +3095,15 @@ static u64 svc_read_last_pa; static u32 svc_pushdec_log; /* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ static u32 svc_pushfail_log; +/* + * SESSION 132: the util-sweep witnesses are per-CLIENT budgets (reset in + * svc_session_flush), not per-boot: on a multi-client boot the driver-init + * reads of an earlier client exhausted the boot budget and hid the later + * clients' rows (the same fix the r707x/r715 pump witnesses got). + */ +static u32 svc_util_dec_log; +static u32 svc_gr_util_log; +static struct { u64 va; u64 h; } svc_util_dec_keys[32]; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -7027,6 +7075,20 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 *done_out, u32 rows, u32 dpitch, u32 spitch, u32 owner_ch, u32 owner_idx, u64 owner_hash) { + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. SESSION 132: one record per COPY OP, + * not per row - a 480-row frame copy logged 480 rows and exhausted + * the per-session budget (128) inside the first op, hiding every + * later copy (including any SASS upload). + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu rows=%u dpitch=0x%x spitch=0x%x dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, rows, dpitch, spitch, + dst_phys, dst_aperture, src_phys, src_aperture); + } if (rows > 1 && dpitch) { u64 done = 0; int rc = 0; @@ -7076,17 +7138,6 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; - /* - * fail4: name every copy the executor is asked to run (pre- - * resolution), so a copy that fails or is held before any witness - * still has an entry record. - */ - if (gsp_sema_dbg && svc.copyent_log < 128) { - svc.copyent_log++; - pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", - svc.cur_ch, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture); - } snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, owner_idx, owner_hash); @@ -9033,6 +9084,26 @@ static unsigned long svc_put_log_at[32]; */ static u32 svc_userd_dbg; static u32 svc_userd_dbg_ch[32]; +/* SESSION 134: per-CE-channel budget of the first-submission ring captures + * (see svc_ce_firstsub_capture) and the start of an unlatched CE channel's + * completion hold (see svc_ce_hold_ms). Session state; reset by + * svc_session_flush. */ +static u32 svc_ce_firstsub_n[32]; +static unsigned long svc_ce_hold_since[32]; +/* + * SESSION 134c: the FB offset each CE rank's first-submission capture latched + * (0 = none). The shadow scan is rank-blind on its own - it takes the first + * ring-shaped page in the shadow - and on r779 that latched rank 0's block + * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported + * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was + * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from + * rank 0's applied release), so the client's rank-1 tracker wait never + * completed and the client spun forever (victory_mail stall, three boots). + * The capture now prefers the rank's convention offset and refuses any + * offset another CE rank already latched. Session state; reset by + * svc_session_flush. + */ +static u64 svc_ce_firstsub_off[32]; /* * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD * probe fails at discovery. Session/reset-scoped, scheduled on the first @@ -12826,6 +12897,10 @@ static void svc_session_flush(void) svc_pushfail_log = 0; svc_rdw_log = 0; svc_pushdec_log = 0; + /* SESSION 132: util-sweep witnesses + their dedup keys are per-client. */ + svc_util_dec_log = 0; + svc_gr_util_log = 0; + memset(svc_util_dec_keys, 0, sizeof(svc_util_dec_keys)); unsigned long flags; u32 i; @@ -12931,6 +13006,7 @@ static void svc_session_flush(void) svc_pb_last = 0; svc_pb_last_n = 0; svc_gr_ring_done = 0; + svc_gr_ring_scan_done = 0; svc_ring_n = 0; /* * The learned tables are session state too: the primary VA->PA pin @@ -13040,6 +13116,9 @@ static void svc_session_flush(void) memset(svc_put_log_at, 0, sizeof(svc_put_log_at)); svc_userd_dbg = 0; memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch)); + memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n)); + memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since)); + memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off)); svc_userd_sweep_log = 0; memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at)); memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n)); @@ -14621,6 +14700,243 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) +/* + * SESSION 134: CE first-submission capture. + * + * The CE (copy-engine) rings are placed by the guest's allocator at a + * location that varies per boot and is NOT reliably described by the + * channel's registered gpfifo VA through the discovered page tables (r763 + * matched the fixed FB convention and the sweep decoded the inline-SASS + * entry; r765-r772 read zeros everywhere, the CE channels never latched, + * the driver's inline LAUNCH_DMA code downloads never executed and the + * frames stayed black). The logged instance where ring content and USERD + * were both live is the channel's first notifier clear - the guest's + * doorbell for its first submission, processed by the pump in the same + * pass that writes the ring. At that instant this hook: + * 1. reads entry0 through the channel's own translation (svc_lookup), + * 2. reads entry0 through the trusted walk (the VAS may have just + * landed since the registration read), + * 3. scans the shadow (0x100 step, first 64 MB, the GR-scan bound) for + * a ring-shaped page whose entry0 names a 1MB-aligned push VA in the + * CE window [0x120000000,0x121000000), and for every hit runs the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate + * whose USERD reads coherent (or idle 0/0) is PINNED and LATCHED + * exactly like svc_ring_autopin's CE path. + * Bounded: at most 4 captures per CE channel, 16 witness rows per boot; + * resets with the session (svc_session_flush). + */ +static u64 svc_ce_firstsub_log; + +/* + * SESSION 134: bounded completion hold for unlatched CE channels. + * + * The clear write in svc_notify_complete is what makes the guest treat the + * submission as consumed; the driver then recycles the ring and its content + * is gone before the client's VAS lands (sema_vas_cand_ms), leaving the CE + * ring latch with nothing readable (r769-r774: rrc=-2 at the first + * submission, the walk answering a zero page later). Holding the notifier + * busy for a bounded window keeps the driver's submission in flight - and + * with it the ring content that the regular per-pass latch path + * (svc_chan_userd through svc_lookup) needs - until the VAS registers and + * the ring reads coherently. 0 = off (the pre-SESSION-134 behaviour). + */ +static u64 svc_ce_hold_ms; +module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644); +MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)"); + +/* + * SESSION 134c rank-aware first-submission capture (see the capture below). + * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at + * ring + 0x10000 (observed for every CE channel across boots; the same + * convention svc_chan_userd applies in its regular per-pass latch). + */ +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL + +/* + * One candidate offset for the CE first-submission capture: validate the + * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose + * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like + * svc_ring_autopin's CE path. Returns true when latched. + */ +static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u8 *sbase, + u64 lim, u64 off, u32 *scan_hits, + u64 *latch_off, u64 *latch_upa, + u32 *latch_put, u32 *latch_get) +{ + static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; + u32 h0, h1, hlen; + u64 hva; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if ((hva & 0xfffff) || hva < 0x120000000ULL || + hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000) + return false; + (*scan_hits)++; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + int prc; + + if (upa + 4 > lim) + continue; + prc = svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3); + if (prc && prc != -EAGAIN) + continue; + /* + * SESSION 135: at the FIRST-SUBMISSION instant the channel's + * submission is by definition unconsumed - the notifier clear + * this capture runs before is what would let the driver treat + * it as consumed. A candidate whose USERD reads a consumed + * state (get >= put && put > 0) cannot be this channel's live + * just-submitted work; it is a PREVIOUS session's leftover + * block surviving in VRAM (r785: the capture latched the dead + * CUDA client's caf00012 block at fb 0x80000, put=1 get=1, + * for the Vulkan channel beef0101 - the decoded entry was the + * old session's, the client stalled on its expectations). + * Idle 0/0 stays acceptable (svc_userd_probe's own rule). + */ + if (p3 && p3 == g3) + continue; + /* latch exactly like the CE autopin */ + svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + *latch_off = off; + *latch_upa = upa; + *latch_put = p3; + *latch_get = g3; + return true; + } + return false; +} + +static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 ring_pa = 0, wpa = 0; + u32 e0 = 0, e1 = 0, walk_e0 = 0, walk_e1 = 0; + u32 scan_hits = 0, latched = 0; + u64 latch_off = 0, latch_upa = 0; + u32 latch_put = 0, latch_get = 0; + int ring_ap = 0, wap = SVC_APERTURE_SYS; + int rrc = -1, wrc = -1; + + if (ch->engine_type == 0 || ch->ce_index >= 32) + return; + if (svc_ce_firstsub_n[ch->ce_index] >= 4) + return; + svc_ce_firstsub_n[ch->ce_index]++; + + if (sbase && lim >= 0x1000) { + /* 1. the channel's own translation at this instant. */ + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + if (!rrc && ring_pa) { + u8 e[8]; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + } + } + /* 2. the trusted walk at this instant. */ + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap)) { + u8 e[8]; + + wrc = 0; + if (!svc_read_ap(vgpu, wpa, svc_ap_of(wap), e, + sizeof(e))) { + memcpy(&walk_e0, e, 4); + memcpy(&walk_e1, e + 4, 4); + } + } + /* 3. shadow scan for a CE-window ring page. */ + { + /* + * SESSION 134c rank-aware order: try the rank's + * convention offset first, then the bounded scan, + * and never accept an offset another CE rank already + * latched. The rank-blind first-hit latched rank 0's + * block for rank 1 on r779 (both rows read + * off=0x80000 upa=0x90000), so the rank-1 channel's + * first release decoded from rank 0's ring/page and + * was stale-dropped (cur=1 already) - the client's + * rank-1 tracker wait never completed and it spun + * forever. A rank whose block is not in the shadow + * at this instant now simply stays unlatched for the + * regular per-pass CE latch instead of stealing a + * foreign block. + */ + u64 exp = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * ch->ce_index; + u64 off; + + latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, + lim, exp, &scan_hits, + &latch_off, &latch_upa, + &latch_put, &latch_get); + for (off = 0; !latched && off + 0x90 <= lim; + off += 0x100) { + unsigned int ri; + bool foreign = false; + + if (off == exp) + continue; + for (ri = 0; ri < 32; ri++) { + if (ri == ch->ce_index || + !svc_ce_firstsub_off[ri]) + continue; + if (svc_ce_firstsub_off[ri] == off) { + foreign = true; + break; + } + } + if (foreign) + continue; + latched = svc_ce_firstsub_try_off(vgpu, ch, + sbase, lim, + off, &scan_hits, + &latch_off, + &latch_upa, + &latch_put, + &latch_get); + } + if (latched) + svc_ce_firstsub_off[ch->ce_index] = latch_off; + } + } + + if (gsp_sema_dbg && svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u n=%u rrc=%d rpa=0x%llx rap=%d e0=0x%x e1=0x%x wrc=%d wpa=0x%llx wap=%d we0=0x%x we1=0x%x scan_hits=%u latch=%u off=0x%llx upa=0x%llx put=%u get=%u\n", + ch->h_object, ch->ce_index, + svc_ce_firstsub_n[ch->ce_index], rrc, ring_pa, + ring_ap, e0, e1, wrc, wpa, wap, walk_e0, walk_e1, + scan_hits, latched, latch_off, latch_upa, + latch_put, latch_get); + } + if (latched) + pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + latch_off, latch_upa, latch_put, latch_get); +} + static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -14673,6 +14989,41 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, return; /* + * SESSION 134: a visible doorbell means a submission just landed - + * for a CE-class channel this is the one instant its ring content + * and USERD are both live. Capture/snapshot it BEFORE the clear + * write below (the clear is what makes the guest treat the + * submission as complete and recycle the ring), see + * svc_ce_firstsub_capture. + */ + if (ch->engine_type) + svc_ce_firstsub_capture(vgpu, ch); + + /* + * SESSION 134: keep an unlatched CE submission open so the ring + * content survives until the latch can read it (see + * svc_ce_hold_ms). The regular per-pass latch path retries while + * the hold runs; the hold expires un-latched into the old + * behaviour, so a channel the emulation can never resolve cannot + * strand the guest. + */ + if (ch->engine_type && !ch->userd_ok && svc_ce_hold_ms && + ch->ce_index < 32) { + if (!svc_ce_hold_since[ch->ce_index]) + svc_ce_hold_since[ch->ce_index] = jiffies; + if (time_before(jiffies, + svc_ce_hold_since[ch->ce_index] + + msecs_to_jiffies(svc_ce_hold_ms))) + return; + if (svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u completion hold expired (%llu ms) unlatched; clearing\n", + ch->h_object, ch->ce_index, svc_ce_hold_ms); + } + svc_ce_hold_since[ch->ce_index] = jiffies; + } + + /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ @@ -14715,8 +15066,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ -#define SVC_CE_RING_FB_BASE 0x80000ULL -#define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */ @@ -14752,6 +15101,62 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool gsp_sema_gr_rings = true; /* + * SESSION 132 dead-entry guard for ring candidates. + * + * The r759 false positive: a vidmem candidate (0xa0000) whose first GPFIFO + * entry (push 0x120100000/40) passed the plausibility gate - the USERD + * probe hit the channel's REAL USERD at 0xb0000, which the VRAM shape scan + * had just found 0x10000 after the candidate by coincidence - was pinned + * as the GR ring while the client's own page tables resolve the ring VA to + * a guest-RAM page (0x11623d000) whose entry0 is live (push 0x4000000/ + * 0x8bc, decoded by the util sweep). The pump then read the pinned page, + * could not fetch the entry's push (all-zero through every source), and + * skipped the entry after 39 failed rescues - the live ring stayed + * undrained and every upload behind it invisible. + * + * A candidate whose first entry cannot deliver a method stream is not this + * channel's ring. The push head is read through the client's own page + * tables (guest truth); a push that cannot be resolved at all does NOT + * mark the entry dead (the VAS may not be registered yet on an early + * pass, and vidmem-backed pushes are not in the client's tables at all). + */ +static bool svc_ring_entry_dead(struct nv_vgpu_device *vgpu, u32 e0, u32 e1) +{ + u64 va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 len = ((e1 >> 10) & 0x1fffff) << 2; + u8 head[16]; + + if (!va || len < 8 || len > 0x10000) + return false; /* unusable entry; the gate rejects it */ + if (svc_read_va_trusted(vgpu, va, head, sizeof(head))) + return false; /* cannot tell: do not reject */ + return !memchr_inv(head, 0, sizeof(head)); +} + +/* + * SESSION 132: does the client's own page table (VAS walk only - never the + * pin/lat/map sources, so a bad pin cannot vote for itself) resolve the + * ring VA to a page whose first entry is LIVE? Used to reject a dead + * candidate only when a live alternative provably exists. + */ +static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 we[8]; + u32 w0, w1; + + if (svc_walk_vaspaces(vgpu, gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return false; /* walk cannot resolve: no verdict */ + if (svc_read_ap(vgpu, wpa, wap, we, sizeof(we))) + return false; + memcpy(&w0, we, 4); + memcpy(&w1, we + 4, 4); + return !svc_ring_entry_dead(vgpu, w0, w1); +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -14778,6 +15183,14 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *len < 4 || *len > 0x10000) return false; /* + * SESSION 132: a candidate whose entry0 push provably reads all-zero + * through the client's own page tables cannot deliver a method + * stream (the pump's rescue path eventually skips exactly these) - + * reject it so the autopin falls through to a live candidate. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) + return false; + /* * Gate: the USERD must be readable. rc==0 is an active get/put * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a * freshly created channel is not rejected. -EINVAL (implausible @@ -14787,6 +15200,20 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *get = 0; prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, ch->gpfifo_entries, put, get); + /* + * SESSION 132: the driver places the GPFIFO and the USERD + * independently (r759: ring in guest RAM 0x11623d000, USERD in + * vidmem 0xb0000), so a walk-resolved ring cannot pass a + * same-aperture USERD probe. Accept the channel's own USERD when + * one is already known - the put/get liveness gate is unchanged. + */ + if (prc != 0 && prc != -EAGAIN && ch->userd_ok && ch->userd_pa && + ch->userd_pa != cand_pa + SVC_GR_USERD_OFF) { + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, ch->userd_pa, ch->userd_as, + ch->gpfifo_entries, put, get); + } return prc == 0 || prc == -EAGAIN; } @@ -16003,6 +16430,15 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->gpfifo_entries, &put, &get)) continue; + /* + * SESSION 132: reject a dead candidate when + * the client's own page tables provably + * resolve the ring VA to a live page (the + * r759 vidmem false positive). + */ + if (svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va)) + continue; svc_pin_store(ch->gpfifo_va, cand, cap); svc_lat_store(ch->gpfifo_va, cand, cap); ch->userd_as = cap; @@ -16038,7 +16474,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, - &put, &get)) { + &put, &get) && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; @@ -16076,7 +16514,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && - va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { /* * Best effort only: a wrapped GP_PUT * legitimately exceeds the entry count, so a @@ -16139,6 +16579,165 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * SESSION 133: bounded shadow ring scan fallback. + * + * The CE-class rings live in VIDMEM and the FB convention only + * matches when the guest's VRAM allocator happens to place them at + * the conventional offsets (r763: matched, fb 0x80000/0xa0000, the + * channel decoded; r765: read zeros, the ring VA resolved to a + * stale RAM page whose entry0 was dead, the CE channel never + * latched and its inline SASS copies never decoded - black + * frames). When every deterministic candidate has failed, scan the + * shadow for a ring-shaped page: entry0 must name a 1 MB aligned + * push VA in the client window with a sane length, must not be + * provably dead (svc_ring_entry_dead - the r759 false-positive + * guard), and the USERD at the observed deltas (+0x10000 for the + * CE family, +0x2000 for GR) must read a coherent put/get. One + * bounded scan per rank per boot; a false candidate still costs + * only a failed probe. + */ + if (!test_bit(ch->gr_index, &svc_gr_ring_scan_done)) { + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 off; + + /* + * SESSION 133b: the rings are NOT page-aligned (r767: the + * USERD shape scan found the channel blocks at 0xfee00/ + * 0xff000/0xff200, so the rings sit at unaligned offsets like + * 0xefe00); step by 0x100 and RETRY (rate-limited) - a first + * pass can run before the client has written the ring or the + * USERD, and a consumed once-bit would never rescan. + */ + if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last + + msecs_to_jiffies(1500))) + return; + svc_gr_ring_scan_last = jiffies; + if (sbase && lim > 0x1000) { + static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF, + 0x1000 }; + + /* + * SESSION 133c pass 1: find the USERD by its OWN + * shape (put@+0x8c, get@+0x88 - the find_userd + * layout) and derive the ring from it. The USERD + * words appear at channel creation and stay; the + * ring entry may be written later, so this pass is + * the stabler trigger. Each hit is validated by the + * derived ring's entry0 (shape + not-dead), so a + * coincidental put/get pair costs a failed check. + */ + for (off = 0; off + 0x90 <= lim; off += 0x100) { + u32 up, ug; + unsigned int ui; + + memcpy(&up, sbase + off + 0x8c, 4); + if (up < 1 || up > 65535) + continue; + memcpy(&ug, sbase + off + 0x88, 4); + if (ug > up) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 roff; + u32 h0, h1, p3 = 0, g3 = 0; + u64 hva; + u32 hlen; + + if (off < udeltas[ui]) + continue; + roff = off - udeltas[ui]; + memcpy(&h0, sbase + roff, 4); + memcpy(&h1, sbase + roff + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || + hva >= SEMA_VA_HI || hlen < 4 || + hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + if (svc_userd_probe(vgpu, off, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = roff; + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd-shape scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, roff, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + /* pass 2: ring-entry-shaped pages (below) */ + for (off = 0; off + 8 <= lim; off += 0x100) { + u32 h0, h1; + u64 hva; + u32 hlen; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || hva >= SEMA_VA_HI || + hlen < 4 || hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + + if (upa + 4 > lim) + continue; + if (svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = off; + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (shadow scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, off, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -21014,6 +21613,14 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes) if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* SESSION 139: float-shaped words are scalars, not device VAs + * (see the scan-site note; r789's 480.0f = 0x43f00000). */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < *n_cand; w++) if (wins[w] == wv) @@ -21538,6 +22145,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21593,6 +22217,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21607,7 +22248,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const u8 *page; u32 k; - if (pp->qmd_word < 32 || pp->qmd_word > 47 || + /* SESSION 139: include word 48 (see the r788 finding: the + * word-34 pointer dedupes into the program's 4K page whose + * entry carries qmd_word=48; the old 32..47-even filter left + * that page unscanned). */ + if (pp->qmd_word < 32 || pp->qmd_word > 48 || (pp->qmd_word & 1) || !pp->pa) continue; kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : @@ -21630,6 +22275,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -21641,6 +22303,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + if (gsp_sema_dbg && svc_gr_pages_log < 48) { + svc_gr_pages_log++; + pr_info("sema-gr: fill cand n=%u pages=%u push=0x%llx\n", + n_cand, svc_fwd_req.gr_n_pages, + svc_fwd_req.gr_push_va); + for (w = 0; w < n_cand && w < 12; w++) + pr_info("sema-gr: win[%u] va=0x%llx arg=%u\n", + w, wins[w], is_arg[w]); + } /* * r635: the launch-level CONTENT anchor for the host-side fallback, * computed BEFORE the span loop so candidate order cannot decide. @@ -22351,6 +23022,25 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + if (gsp_sema_dbg && svc_gr_pages_log < 24) { + svc_gr_pages_log++; + pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.param_va, + svc_gr_cand.prog_va, svc_gr_cand.n_pointer, + svc_gr_cand.n_inline, svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + for (i = 0; i < svc_gr_cand.n_pointer && i < 8; i++) + pr_info("sema-gr: ptr[%u] va=0x%llx word=%u\n", i, + svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_fwd_req.gr_n_pages && i < 12; i++) { + const struct svc_gr_page *pg = + &svc_fwd_req.gr_pages[i]; + + pr_info("sema-gr: page[%u] va=0x%llx pa=0x%llx ap=%u qmd_word=%u\n", + i, pg->va, pg->pa, pg->ap, pg->qmd_word); + } + } if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) return -E2BIG; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { @@ -24057,6 +24747,36 @@ static u64 svc_gr_ahead_skips; static u64 svc_gr_ahead_owned; /* SESSION 127: bounded witness for the sweep-caller's launch request. */ static u32 svc_gr_ahead_req_log; +/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */ +static u64 svc_gr_ahead_inline_writes; + +/* + * SESSION 129 util-channel witness: the utility channel's subchannel-2 + * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm + * fired for in-push payloads on the GR channel - i.e. the failure sits in + * the split (tail-served) shape. This struct carries the per-entry story + * out of the translator for one bounded, deduplicated witness line: + * which read source answered (attempt A = the channel path, 1 = the direct + * registered-VAS walk), whether the strict parse accepted the entry, and + * what the LID arm did (saw/fired/refused a still-zero payload). + */ +struct svc_util_wit { + u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */ + u32 pa_ok; /* attempt-A parse (method count) */ + u32 pb_ok; /* attempt-B parse (method count) */ + u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */ + u32 t_len; /* tail length served */ + u32 t0, t1; /* SESSION 130: first two words of the served tail */ + bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */ + bool fired; /* ... and authored its write */ + bool zero; /* ... but the tail payload was still all-zero (pending) */ + u32 dst; + u32 line_len; + u32 line_lc; + u32 count; +}; +static struct svc_util_wit svc_util_wit; +static u64 svc_util_pending_zero; /* * SESSION 127 C5C0 translator, bounded tail materialization buffer: a @@ -24065,10 +24785,98 @@ static u32 svc_gr_ahead_req_log; * entry plus a dedicated data slot entry). 4 KB covers the observed * param-bank write (1568 B) in full and the first page of the big uploads. */ -#define SVC_AHEAD_TAIL_INLINE_MAX 4096 +#define SVC_AHEAD_TAIL_INLINE_MAX 32768 static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; static u64 svc_ahead_inline_trunc; +/* + * SESSION 128: one-copy materialization bound for the utility/CE sweep. + * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch + * copy from a client-owned staging buffer into the vidmem shadow) moves + * the driver's code/constant uploads; the observed SASS/code-heap class is + * tens of KB, so 4 MB is a generous cap that still refuses a bogus + * geometry before it can walk the shadow. + */ +#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20) + +/* + * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an + * unprocessed copy-class channel carried - src bytes live in the client's + * staging buffer (guest RAM, resolved through the trusted walk like any + * operand page), dst is the vidmem page the driver staged the upload for + * (code heap / constant store). This is the emulated engine write for the + * copy the real CE would perform; it is once-only keyed by (push_va, + * content hash, dst, bytes) so re-scans of a stable ring are free. + */ +static u64 svc_ahead_copy_in_skip; +static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h, + u64 push_va, u64 hash, u64 src_va, u64 dst_va, + u32 row, u32 rows, u32 dma) +{ + static u8 cb[65536]; + u64 bytes = (u64)row * (rows ? rows : 1); + u64 done = 0; + u64 spa = 0, dpa = 0; + int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS; + int rc = 0; + bool src_ok, dst_ok; + + if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI || + dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI) + return; + /* + * Only the upload direction is materialized: the source must be a + * resolved client page in sysmem/guest RAM (the staging buffer the + * client CPU-wrote), and the destination must be vidmem (resolved to + * the FB aperture) or a not-yet-resolved VA whose bytes the VA + * shadow will hold. A resolved SYS destination is a copy-OUT (the + * engine reading vidmem into a client buffer): replaying it here + * would overwrite the client's buffer with shadow garbage before the + * producing launch ran, so it is skipped - the GR channels' copy-out + * machinery owns that direction. + */ + src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB; + dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) || + dap == SVC_APERTURE_FB; + if (!src_ok || !dst_ok) { + svc_ahead_copy_in_skip++; + if (svc_ahead_copy_in_skip <= 8) + pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n", + src_ok, dst_ok, push_va, src_va, dst_va, + bytes); + return; + } + if (bytes > SVC_AHEAD_COPY_IN_MAX) { + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n", + push_va, src_va, dst_va, bytes, + SVC_AHEAD_COPY_IN_MAX); + return; + } + if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes)) + return; + svc_inline_once_store(push_va, hash, dst_va, (u32)bytes); + while (done < bytes) { + u32 chunk = bytes - done > sizeof(cb) ? + (u32)sizeof(cb) : (u32)(bytes - done); + + rc = svc_read_va(vgpu, src_va + done, cb, chunk); + if (rc) { + rc = -EIO; + break; + } + rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk, + ch_h); + if (rc) + break; + done += chunk; + } + svc_gr_ahead_inline_writes++; + pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n", + src_va, dst_va, bytes, dma, done, rc, ch_h); +} + /* One 4-byte word of a method stream that may continue into the tail. */ static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, u32 off) @@ -24172,7 +24980,7 @@ done: */ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, const u8 *buf, u32 len, const u8 *tail, - u32 taillen, u64 push_va) + u32 taillen, u64 push_va, bool inline_only) { static struct svc_push_state stq; /* service thread only */ static u32 dw[66]; /* QMD window words */ @@ -24236,7 +25044,7 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch * C5C0 inline-QMD window (mirrors the main decode's intercept): * only accumulate once the run declares itself a QMD window. */ - if (xlate && forward && subch >= 1 && + if (xlate && forward && !inline_only && subch >= 1 && addr >= SVC_QMD_ADDR_A && addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (stq.qmd_have || @@ -24269,6 +25077,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch u32 cap = count * 4; u32 bytes, k, words; u64 h = svc_push_hash(buf, len); + bool tail_served = end > len; + bool refuse_zero = false; if (nbytes > cap) nbytes = cap; @@ -24288,10 +25098,69 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch taillen, off + 4 + 4 * k) }, 4); - if (bytes && !svc_inline_once_seen(push_va, h, off_out, - bytes)) { + /* + * SESSION 129: a tail-served payload whose head is still + * all zeros means the tail source answered with an empty + * page (a stale pin shadow, or the driver has not staged + * the bytes yet) - authoring zeros over the destination + * would corrupt the code heap. Leave the write pending + * (no once-store, the sweep retries every pass) until a + * nonzero payload arrives. In-push payloads are never + * gated: the bytes are part of the flushed submit. + */ + if (inline_only && tail_served && bytes && + !memchr_inv(svc_ahead_inline_buf, 0, + min_t(u32, bytes, 16))) + refuse_zero = true; + if (inline_only) { + svc_util_wit.saw_lid = true; + svc_util_wit.dst = (u32)off_out; + svc_util_wit.line_len = line_len; + svc_util_wit.line_lc = line_count; + svc_util_wit.count = count; + if (refuse_zero) + svc_util_wit.zero = true; + } + if (refuse_zero) { + /* + * SESSION 130: a zero-headed tail-served + * payload is applied under a VARIANT once-key + * (h | 1) instead of being refused forever: + * the destination is currently empty, so + * authoring zeros is idempotent and correct + * for the driver's internal zero buffer (the + * observed 256 B utility i2m = exactly + * GF100_ZERO_BUFFER_SIZE), while a later + * non-zero payload for the same entry still + * matches its own (h) key and lands - the + * refusal can no longer stall the arm + * (r737/r739: `writes=0` with the payload + * page stably zero). + */ + svc_util_pending_zero++; + if (bytes && !svc_inline_once_seen(push_va, + h | 1, + off_out, + bytes)) { + svc_inline_once_store(push_va, h | 1, + off_out, bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, + off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + } else if (bytes && !svc_inline_once_seen(push_va, h, + off_out, + bytes)) { svc_inline_once_store(push_va, h, off_out, bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; (void)svc_grce_inline_write(vgpu, off_out, svc_ahead_inline_buf, bytes, @@ -24310,12 +25179,19 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; - rec->dst_va = off_out; - rec->len = min_t(u32, bytes, - GSP_BRIDGE_INLINE_BYTES); - memcpy(rec->data, svc_ahead_inline_buf, - rec->len); - svc_grce_dst_ring_next++; + /* + * A refused (still-zero) tail payload is not + * recorded either: the next pass re-reads the + * tail and the real bytes land then. + */ + if (!refuse_zero) { + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } } if (end > len) break; @@ -24323,8 +25199,49 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch continue; } - /* TURING_DMA_COPY_A operand runs (subch >= 2). */ - if (xlate && subch >= 2) { + /* + * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for + * subchannel 1 AND 2. The mem2mem subchannel (2, the + * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility + * channels drive) carries the same operand methods as the + * compute subchannel (1). This group MUST be parsed BEFORE + * the TURING-copy branch below: that branch's `subch >= 2` + * guard also matches subchannel 2 and would swallow the + * 0x180..0x18c methods unread, leaving have_out/line_len + * unset - the exact reason the utility channel's subch-2 + * LOAD_INLINE_DATA arm never fired (r737/r739 witness: + * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`). + */ + if (xlate && (subch == 1 || subch == 2) && + addr >= 0x180 && addr <= 0x18c && count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } else if (xlate && subch >= 2) { + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ u32 k, n = count > 4 ? 4 : count; u32 vv[4] = { w0, w1, w2, w3 }; @@ -24394,52 +25311,35 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch * movement, nothing to record). */ last_dma = dma; - if (forward && (dma & 0x3) && have_in && - have_out && line_len && line_count && + if ((dma & 0x3) && have_in && have_out && + line_len && line_count && off_in && off_out) { - svc_gr_copy_pend_add(ch->h_object, - push_va, svc_push_hash(buf, - len), - off_in, off_out, - line_len * 4, line_count, - pitch_out ? pitch_out : + if (inline_only) + svc_ahead_copy_in(vgpu, + ch->h_object, push_va, + svc_push_hash(buf, + len), + off_in, off_out, line_len * 4, - pitch_in, dma, last_remap); - cp_subch = subch; - } - } - } else if (xlate && subch == 1 && addr >= 0x180 && - addr <= 0x18c && count) { - u32 k, n = count > 4 ? 4 : count; - u32 vv[4] = { w0, w1, w2, w3 }; - - /* GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c). */ - for (k = 0; k < n; k++) { - switch (addr + 4 * k) { - case 0x180: - line_len = vv[k]; - break; - case 0x184: - line_count = vv[k]; - break; - case 0x188: - off_out = (off_out & 0xffffffffULL) | - ((u64)(vv[k] & 0x1ffff) - << 32); - have_out = 1; - break; - case 0x18c: - off_out = (off_out & - ~0xffffffffULL) | vv[k]; - have_out = 1; - break; - default: - break; + line_count, dma); + else if (forward) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + off_in, off_out, + line_len * 4, + line_count, + pitch_out ? pitch_out : + line_len * 4, + pitch_in, dma, + last_remap); + cp_subch = subch; + } } } } - if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { + if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) { u64 va; va = ((u64)(w1 & 0xff) << 32) | @@ -24548,7 +25448,7 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, } swept++; nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, - tail_len, va); + tail_len, va, false); } svc_gr_ahead_entries += swept; svc_gr_ahead_rels += nrel; @@ -24561,6 +25461,341 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, } } +/* + * SESSION 129: read a VA straight through the registered-VAS walks only + * (newest registration first), one page at a time - the exact path + * walk_probe and sema_probe prove. The util sweep's channel read can be + * shadowed by a stale injected pin / window guess for the utility VAs + * (observed r737: the subchannel-2 LID arm never fired although the push + * page content is real and the VAS walk resolves both the methods entry + * and its payload slot); this helper is the arbitration fallback that + * bypasses those sources entirely. + */ +static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t off = va & (SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used + * by the bounded util-sweep witness to select only the entries that can + * exercise the arm (one per distinct content, so a stable ring logs once). + */ +static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch) +{ + u32 k, n = 0; + + *first_subch = 0xff; + for (k = 0; k + 4 <= len; k += 4) { + u32 hdr; + + memcpy(&hdr, buf + k, 4); + if (((hdr >> 29) & 0x7) == 3 && + ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) { + if (*first_subch == 0xff) + *first_subch = (hdr >> 13) & 0x7; + n++; + } + } + return n; +} + +/* + * SESSION 128: utility-channel inline-download materialization. + * + * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D + * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered + * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in + * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA + * downloads they carry (the client's code-heap/SASS upload, command-stream + * staging) never land anywhere the emulation can see: the destinations + * stay zero in the vidmem shadow. + * + * This pass scans a bounded window of the channel's GPFIFO directly (the + * gpfifo VA resolves through the trusted walk even when the USERD does + * not) and feeds each entry + its successor (the source-view call carrying + * the payload) through the proven sweep decoder in inline-only mode: the + * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2), + * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by + * svc_grce_inline_write; releases, QMD windows and copy-pend records are + * skipped so nothing about the client's fence/completion state changes. + * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring + * costs only the reads. Requires sema_gr_xlate; bounded by + * sema_gr_util_scan entries per channel per pass (0 = off). + * + * SESSION 129 read arbitration: attempt A is the channel path (unchanged); + * when A fails to read, or reads content the strict parse rejects, attempt + * B re-reads the push and the tail through svc_util_walk_read and the + * parse arbitrates - a source that yields a method stream wins. The r737 + * evidence (arm never fired while the content is real) pointed at exactly + * this: a shadowed channel read consumed the entry without a parseable + * buffer while the walk resolves the true pages. + */ +static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ +/* + * The observed utility-channel pushes are the ~17 KB setup block and the + * two split uploads (256 B command stream + 12 KB code/heap payload); a + * 64 KB window covers every one of them with room to spare while keeping + * the two static buffers modest. + */ +#define SVC_UTIL_BUF (64 * 1024) + static u8 ubuf[SVC_UTIL_BUF]; + static u8 nbuf[SVC_UTIL_BUF]; + static u8 bbuf[SVC_UTIL_BUF]; + static u8 tbuf[SVC_UTIL_BUF]; + static u64 svc_gr_util_entries; + static u64 svc_gr_util_writes; + static u64 svc_util_tail_alt; /* SESSION 130: tails re-read through + * the direct walk because the + * channel read was zero-headed */ + static u64 svc_util_tail_try; /* ... and the walk attempts (alt is + * the subset that yielded non-zero) */ + u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64); + u32 swept = 0, nmade = 0; + + if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan || + !ch->gpfifo_va || !ch->gpfifo_entries) + return; + for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen; + u64 va, before; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + continue; /* unwritten slot */ + if (!va || !len || len > SVC_PUSH_MAX) + continue; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + continue; + /* + * SESSION 132 stale-translation arbitration: when the + * channel-path entry is dead (its push provably reads + * all-zero through the client's page tables) while the + * direct VAS walk answers a live entry on the same ring + * slot, the channel path is reading a stale page (the r759 + * pin false positive) - the walk's entry is the channel's + * ring. A dead entry with no live walk alternative is left + * to the existing push arbitration below. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) { + u8 went[SVC_GPFIFO_ENTRY_SIZE]; + + if (!svc_util_walk_read(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + went, sizeof(went))) { + u32 we0, we1; + u64 wva; + u32 wlen; + + memcpy(&we0, went, 4); + memcpy(&we1, went + 4, 4); + wva = ((u64)(we1 & 0xff) << 32) | + (u64)(we0 & 0xfffffffc); + wlen = ((we1 >> 10) & 0x1fffff) << 2; + if (!svc_ring_entry_dead(vgpu, we0, we1) && + wva && wlen && wlen <= SVC_PUSH_MAX && + wva >= SVC_LOW_REL_VA_LO && + wva < SEMA_VA_HI) { + e0 = we0; + e1 = we1; + va = wva; + len = wlen; + } + } + } + rlen = min_t(u32, len, sizeof(ubuf)); + /* + * SESSION 129 read arbitration + bounded witness (see the + * sweep's header comment). Attempt A: the channel path, + * unchanged. Attempt B: the direct registered-VAS walk, used + * when A cannot read or cannot parse; the strict parse then + * arbitrates which content is the method stream. + */ + { + u32 push_rc, parse_a = 0, parse_b = 0; + u32 tl_a = 0, tl_b = 0; + u32 lidn, lidsc = 0xff, slot; + u64 nva = 0; + u32 nlen = 0; + u64 hkey; + u8 *pbuf = ubuf; + bool ut = false, seen = false; + + push_rc = svc_read_va_push(vgpu, ch, va, ubuf, + rlen); + /* + * The payload of a trailing LOAD_INLINE_DATA run rides + * the next GPFIFO entry (the driver's source-view + * GPFIFO call, observed beef0100 idx00->idx01 and + * idx08->idx09): read it as the tail exactly like + * svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1; + + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + } + if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) { + tl_a = min_t(u32, nlen, sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, nbuf, + tl_a)) + tl_a = 0; + } + if (!push_rc) + parse_a = svc_ahead_parse(ubuf, rlen, nbuf, + tl_a, &ut); + if (push_rc || !parse_a) { + bool bpush = false; + + if (nva && nlen && nlen < SVC_PUSH_MAX) { + u32 want = min_t(u32, nlen, + sizeof(nbuf)); + + if (!svc_util_walk_read(vgpu, nva, + nbuf, + want)) + tl_b = want; + } + if (!svc_util_walk_read(vgpu, va, bbuf, + rlen)) { + bpush = true; + parse_b = svc_ahead_parse(bbuf, rlen, + nbuf, tl_b, + &ut); + /* + * A's buffer is authoritative only + * when A actually read it; otherwise + * (or when only B parses) B's walk + * content is the stream. + */ + if (push_rc || + (parse_b && !parse_a)) { + pbuf = bbuf; + tl_a = tl_b; + } + } + if (push_rc && !bpush) + continue; /* unreadable both ways */ + } + /* + * SESSION 130 tail-content arbitration: a tail-served + * payload whose channel head is still all-zero is not + * the staged bytes (the page the channel translation + * points at is empty, or the read is shadowed by a + * stale pin). Try the direct registered-VAS walk - + * the same path attempt B proves - before handing + * the tail to the LID arm, whose zero-headed refusal + * would otherwise stall the upload every pass. + */ + if (tl_a && nva && + !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) { + u32 want = tl_a; + + svc_util_tail_try++; + if (!svc_util_walk_read(vgpu, nva, tbuf, + want) && + memchr_inv(tbuf, 0, + min_t(u32, want, 16))) { + memcpy(nbuf, tbuf, want); + svc_util_tail_alt++; + } + } + memset(&svc_util_wit, 0, sizeof(svc_util_wit)); + if (tl_a >= 4) + memcpy(&svc_util_wit.t0, nbuf, 4); + if (tl_a >= 8) + memcpy(&svc_util_wit.t1, nbuf + 4, 4); + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf, + tl_a, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; + /* + * Bounded witness, one line per distinct LID-carrying + * (or unparseable) entry content: which attempt was + * used, the parses, the tail resolution and what the + * LID arm did. + */ + lidn = svc_util_lid_scan(pbuf, rlen, &lidsc); + if (!lidn && parse_a) + continue; + hkey = svc_push_hash(pbuf, rlen); + for (slot = 0; slot < 32; slot++) + if (svc_util_dec_keys[slot].h == hkey && + svc_util_dec_keys[slot].va == va) { + seen = true; + break; + } + if (seen || svc_util_dec_log >= 96) + continue; + slot = svc_util_dec_log & 31; + svc_util_dec_keys[slot].va = va; + svc_util_dec_keys[slot].h = hkey; + svc_util_dec_log++; + pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu/%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n", + ch->h_object, i, va, rlen, + pbuf == bbuf ? 1u : 0u, parse_a, parse_b, + nva, tl_a, svc_util_wit.t0, svc_util_wit.t1, + svc_util_tail_alt, svc_util_tail_try, + lidn, lidsc, + svc_util_wit.saw_lid ? 1 : 0, + svc_util_wit.fired ? 1 : 0, + svc_util_wit.zero ? 1 : 0, + svc_util_wit.dst, svc_util_wit.line_len, + svc_util_wit.line_lc, + rlen >= 4 ? *(const u32 *)pbuf : 0, + rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0); + } + } + svc_gr_util_entries += swept; + svc_gr_util_writes += nmade; + if (swept && (nmade || svc_gr_util_log < 24)) { + svc_gr_util_log++; + pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n", + ch->h_object, swept, nmade, svc_gr_util_entries, + svc_gr_util_writes); + } +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { if (svc_pdb_disc_pending) { @@ -24871,6 +26106,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies + 27 * HZ; } + /* + * SESSION 128: utility/upload channels carry the engine's + * inline code/data downloads (LOAD_INLINE_DATA on the + * mem2mem subchannel) and the copy-class channels (eng=11/12) + * carry the driver's pitch copy-ins (staging -> code heap / + * constant store). With an unresolved USERD the put-bounded + * entry loop below never decodes them (the !userd_ok branch + * continues before it); with a resolved USERD the main + * decode's LOAD_INLINE_DATA/operand arms still only accept + * subchannel 1, so the subchannel-2 downloads are missed + * either way. Scan every channel's GPFIFO directly here and + * materialize the downloads (see svc_gr_util_sweep); the + * inline-only mode plus the once-only key make the overlap + * with the main loop and the GR sweep a no-op. + */ + if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate) + svc_gr_util_sweep(vgpu, ch); + if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get, |