diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 5482 |
1 files changed, 4875 insertions, 607 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 487bb4d..dcce4b5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -93,6 +93,13 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) #define KVA_HI 0xffffc00000000000ULL /* + * r534: guest-read granularity for the full-RAM discovery scan + * (svc_scan_pass). Hoisted here because the scan is defined before the + * push-apply window that used to own the define. + */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ + +/* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed @@ -160,6 +167,15 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -260,6 +276,10 @@ struct svc_channel { * window (observed: one 8 s grace per push, * serializing the runtime init) */ DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); + u8 slot_pend_tries[SVC_GPFIFO_MAX_BITS]; /* fail4 r308: bounded + * re-holds of a denied device-dst copy slot */ + u32 counter_gen; /* r311: session generation that authored the + * completion word (guards the residue zero) */ /* GPFIFO slots already consumed in the current * put epoch (the client's put value since it * last wrapped). A pin-batch rewind replays @@ -330,6 +350,10 @@ struct svc_channel { * publish targets counter_pa instead, so both * families of the counter page get written * (SA-COUNTER FORENSICS-grv7-counter F1) */ + u64 release_va; /* r378: the release VA release_pa was learned + * from, kept so a late write (stale-recover + * zero) can re-validate the target through + * gsp_sema_write_guard before it lands */ u32 counter_last; /* last value published to counter_pa */ u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ @@ -377,6 +401,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -394,6 +428,11 @@ struct svc_channel { u64 gr_hold_dead_va; /* launch push VA the bound belongs to */ unsigned long gr_hold_dead_since; /* first-hold stamp (waited_ms) */ bool gr_hold_exp_logged; /* one GRFILL-EXPIRED line per launch */ + u64 opdef_hold_dead_va; /* r361: launch push VA the opdef bound + * belongs to (slot-keyed `since` is + * defeated by rewind replays) */ + unsigned long opdef_hold_deadline; /* absolute opdef hold expiry */ + bool opdef_hold_exp_logged; /* one OPDEF-EXPIRED line per launch */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -407,6 +446,49 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -417,19 +499,7 @@ struct svc_channel { u64 wd_put; /* put at the window start */ u64 wd_sub; /* submissions at the window start */ unsigned long wd_since; /* window start (0 = no window) */ - bool wd_epoch_done; /* the stale-epoch reset ran for this put */ - u32 wd_nocap_put; /* force-replay (no capture) for slots below - * this while the stale range is re-walked; 0 = - * none */ - bool wd_chanswitch; /* STALLFREE: registered; first pump pass - * must reconcile the binding before the walk - * (drop dead opdef slots whose since predates - * the binding; log the transition) */ - unsigned long wd_newbind_until; /* STALLFREE: freshness window of a - * new binding; inside it an - * unmapped/undecoded GR entry is held - * (not silently consumed) and an - * immediate rebuild is requested */ + unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ }; /* @@ -485,6 +555,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -814,6 +890,20 @@ struct sema_svc { u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ + u64 fill_ram_log; /* r361 physical-RAM fill build budget */ + u64 pushtrace_all; /* fail4: de-windowed push decode + * trace (every push, bounded) */ + u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ + u64 dmatrace_log; /* fail4: denied-transfer trace */ + u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ + u64 snap_gate_log; /* fail4 r311: capture-gate witness */ + u64 copyent_log; /* fail4: svc_copy entry trace */ + u64 bigrun_log; /* fail4: big method-data run headers */ + u64 lid_hdr_log; /* fail4: LOAD_INLINE_DATA headers */ + u64 skiprun_log; /* fail4: silently skipped data runs */ + u64 mystery_log; /* fail4: moderate op=1 runs on big pushes */ + u64 next_hdr_log; /* fail4: header after each big LID run */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ @@ -1019,19 +1109,6 @@ struct sema_svc { * stale-replay wedge watchdog (zero on a green run). */ u64 gr_stale_wedge; - /* - * STALLFREE r229/r230: set by the pump when a new binding's GR entry - * is unmapped/empty; the service loop consumes it by running - * svc_rebuild() before the next pump pass, so the page translations - * land within ~1 s and the pending entry re-evaluates. - */ - bool rebuild_now; - u32 gr_pending_map; /* GR-STALL-pending-map lines - * emitted (bounded proof) */ - u32 gr_newbind_log; /* GR-STALL-newbind window-armed - * lines emitted (bounded proof) */ - u32 gr_rebuild_log; /* GR-STALL-rebuild consumption - * lines emitted (bounded proof) */ u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -1293,7 +1370,10 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int k; u64 keep_counter_pa = 0; u64 keep_release_pa = 0; + u64 keep_release_va = 0; u8 keep_gr_index = 0, keep_ce_index = 0; + bool keep_pub_armed = false; + u32 keep_counter_gen = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); bool same = ch->valid && ch->vgpu == desc->vgpu && ch->h_object == desc->h_object; @@ -1316,8 +1396,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same_gen) { keep_counter_pa = ch->counter_pa; keep_release_pa = ch->release_pa; + keep_release_va = ch->release_va; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; + /* + * r311: a same-session re-registration must not + * forget that this session already proved itself + * (gr_pub_armed) - losing it re-arms the residual + * watchdog, and its recovery zeroes a completion + * word this live session authored, which the + * client's UVM reads as a downward jump and never + * completes on (r311 vector_add: pub-adv/zero + * 3 s cycles with the client polling forever). + */ + keep_pub_armed = ch->gr_pub_armed; + keep_counter_gen = ch->counter_gen; bitmap_copy(keep_rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); } @@ -1326,24 +1419,17 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ ch->gr_hold_dead_va = 0; /* GRFIX: no launch bound armed */ ch->gr_hold_exp_logged = false; + ch->opdef_hold_dead_va = 0; /* r361: no opdef bound armed */ + ch->opdef_hold_exp_logged = false; ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ - /* - * STALLFREE: a genuinely new binding - a fresh channel or a - * same-handle rebinding across a session/context boundary - * (!same_gen) - must reconcile on its first pump pass: drop - * opdef slots a dead context left under this handle (only - * ones whose first-pending stamp predates the binding, so a - * slot the fresh binding creates can never be touched) and - * log the transition when the ring already carries content. - * A same-generation RM re-issue preserves state deliberately - * (m148/m176) and is not reconciled. - */ - ch->wd_chanswitch = !same_gen; - if (same_gen) + if (same_gen) { bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); + ch->gr_pub_armed = keep_pub_armed; + ch->counter_gen = keep_counter_gen; + } /* * Rank among the live CE channels OF THIS GUEST. The RM hands * out object handles in creation order and the guest's UVM @@ -1392,6 +1478,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same) { ch->counter_pa = keep_counter_pa; ch->release_pa = keep_release_pa; + ch->release_va = keep_release_va; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } @@ -1500,6 +1587,8 @@ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ +static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ +static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -1568,6 +1657,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + return; + } + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -1586,12 +1759,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1815,8 +2016,18 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return -ENOENT; } - /* PD0: one dual entry per index (big half then small half). */ + /* + * PD0: one dual entry per index (big half then small half). + * Geometry per the driver's own GMMU format (gp10x family): + * PD3 [48:47], PD2 [46:38], PD1 [37:29], PD0 [28:21], PT + * [20:12]. PD0 is 8 bits - an earlier 0x1ff experiment here was + * reverted; the observed +2 MB client-page phase is NOT a PD0 + * index error. + */ idx = (va >> 21) & 0xff; + if (trace) + pr_info("sema-walk: PD0 table=0x%llx idx=%u va=0x%llx tap=%d\n", + table, idx, va, tap); if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, @@ -1826,6 +2037,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; + if (trace) + pr_info("sema-walk: PD0 big=0x%016llx small=0x%016llx\n", + big, small); /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); @@ -1837,6 +2051,12 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, *addr += va & 0xfff; return 0; } + if (trace) + pr_info("sema-walk: small miss sap=%d t=0x%llx\n", + sap, t); + } else if (trace) { + pr_info("sema-walk: small skip sap=%d small&1=%llu\n", + sap, small & 1); } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ @@ -1869,6 +2089,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return 0; } } + if (trace) + pr_info("sema-walk: no leaf va=0x%llx big=0x%llx small=0x%llx\n", + va, big, small); return -ENOENT; } @@ -1897,6 +2120,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -2022,8 +2265,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -3032,6 +3290,16 @@ static u32 svc_pool_pa_n; static u32 svc_pool_pa_next; static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; static u32 svc_pool_pa_logged_n; +/* + * fail4: the completion-hint window was 10 s. It outlived short-lived + * clients: after a client freed its RM events the window kept re-posting + * hint events every 0.5 s, and a post already queued in the status ring + * could be read after the guest's own free processing - the guest ISR + * then ran a freed RC callback and the guest panicked in interrupt + * (rm_run_rc_callback+0x0). 1.5 s still covers a waiter that arrives + * after the edge while shrinking the post-free exposure ~7x. + */ +#define SVC_POST_HINT_MS 1500 static u64 svc_pool_pa_miss_log; static u32 svc_pool_pa_hit_log; static u32 svc_pool_wr_log; @@ -3080,6 +3348,14 @@ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, } } +/* r372: landing-table entries repaired from a fresh pin (stale-ledger class). */ +static u64 svc_ledger_pin_repairs; +/* r372: pool alt fan-outs skipped because the primary pin moved on. */ +static u64 svc_pool_alt_skipped; +/* r372: bounded page-note resolution diagnostics (va/pa/ap/branch). */ +static u32 svc_gr_pagenote_log; +static u32 svc_fbhpa_log; + /* * Fan a pool release out to the last displaced translation of the same page * (see svc_ipin_alt): the primary target may be the stale one when two host @@ -3103,6 +3379,27 @@ static void svc_pool_alt_apply(struct nv_vgpu_device *vgpu, u32 h, u64 va, return; if ((pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return; /* never write an identity translation */ + /* + * r372 (stale-ledger corruption class): fan out ONLY while the + * current primary pin for this VA still names the displaced page. + * Once a newer pin has displaced it, the old page belongs to a + * buffer the guest has freed - writing it lands on whatever the + * guest kernel reused the page for. A missing current pin is + * equally unprovable: skip. The primary release write already + * succeeded either way, so this is a lost best-effort, not a + * failure. + */ + { + u64 cpa = 0; + int cap = SVC_APERTURE_SYS; + + if (svc_ipin_lookup(va, &cpa, &cap) || + ((cpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + svc_pool_alt_skipped++; + return; + } + } if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) return; if (cur >= val) @@ -3325,11 +3622,50 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. + * + * r536: the read-back proof does NOT catch a moved pool whose + * OLD page is still readable - the write then lands on the old + * page while the client polls the new one and the tracker + * starves (observed: "pending release starved va=0x12100000x + * tries=64" for ~45 s after the pool moved, with the stale pin + * pa=0x120b30000 still readable; the fresh pin pa=0x130e9d000 + * arrived 45 s later). The client's OWN page tables are the + * GPU/UVM ground truth for where the tracker reads, so when a + * structural walk resolves a DIFFERENT page than the pin, the + * walk wins (both are read-back proven at the exact offset). */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) || + !svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + walk_hit = true; + if (walk_hit && + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa)) { + u32 wprobe = 0; + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), + &wprobe, sizeof(wprobe))) { + svc_pool_pa_store(va, exact, wap); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-pin"; + svc_pool_pa_note(h, va, exact, + "walk-over-pin"); + *addr = exact; + *ap = wap; + return 0; + } + } svc_tr_clear_failure(page); if (src) *src = "pin"; @@ -3537,6 +3873,10 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) } static void svc_pending_add(u64 va, u32 payload, u32 op); +/* r34x: defined with the pending ring / opdef slots (later in the file); + * the acquire gate drains/appraises withheld releases before it holds. */ +static void svc_apply_pending(struct nv_vgpu_device *vgpu); +static bool svc_wait_pend_release(u64 va); /* * F1: GR release-word authorship checks. Defined next to the svc_fwd_req @@ -3700,76 +4040,92 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, u64 gpa; u32 nonz = 0; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { - u32 off; + /* + * r534: read guest RAM in SVC_SCAN_CHUNK (64 KB) blocks instead of + * one 4 KB page per vfio_dma_rw call. The full 6 GB scan is the + * rebuild's whole cost (1.5 M guest reads ~= 10-11 s per rebuild, + * once per 10 s: the pump got one pass per rebuild and the second + * client's resolutions could wait minutes). A 64 KB read amortizes + * the iommu walk + pin over 16 pages, cutting the scan to well under + * a second with identical pair/base detection (the inner page loop + * is byte-for-byte the old per-page body). + */ + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + u32 po; if (READ_ONCE(svc_sess_flush)) break; /* ISSUE-0002: a flush is waiting on this scan */ - if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; - for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { - u64 a, b; + for (po = 0; po < SVC_SCAN_CHUNK; po += SEMA_PAGE_SIZE) { + u8 *page = buf + po; + u64 pg = gpa + po; + u32 off; - memcpy(&a, buf + off, 8); - memcpy(&b, buf + off + 8, 8); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; - /* (a) self-pointer -> base candidate */ - if (a >= KVA_LO && a < KVA_HI && - (a & 0xfff) == ((gpa + off) & 0xfff)) { - u64 cb = a - gpa - off; - u32 i; + memcpy(&a, page + off, 8); + memcpy(&b, page + off + 8, 8); - if ((cb & 0x3fffffffULL) == 0) { - for (i = 0; i < *ncand; i++) { - if (cands[i].base == cb) { - cands[i].count++; - break; + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((pg + off) & 0xfff)) { + u64 cb = a - pg - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; } - } - if (i == *ncand && *ncand < 8) { - cands[*ncand].base = cb; - cands[*ncand].count = 1; - (*ncand)++; } } - } - /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ - if (a >= KVA_LO && a < KVA_HI && - b >= SEMA_VA_LO && b < SEMA_VA_HI && - (a & 0xfff) == 0 && (b & 0xfff) == 0 && - *npairs < PAIR_MAX) { - u8 ok = 0; + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + u8 ok = 0; - /* - * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] - * (offset 48) directly before vas[UVM_ID_GPU0] - * (offset 56), so a pair taken from a live rm_mem - * is preceded by the header {u32 type; bitmap - * mapped_on}. Validate it when it fits in the same - * page: freed allocations and unrelated tables - * that happen to hold KVA/VA pairs do not match, - * and a stale entry for the same VA must not - * shadow the live one. - */ - if (off >= 48) { - u32 type; - u64 mapped_on; - - memcpy(&type, buf + off - 48, 4); - memcpy(&mapped_on, buf + off - 40, 8); - if ((type == 0 || type == 1) && - (mapped_on & 0x3) == 0x3) - ok = 1; + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, page + off - 48, 4); + memcpy(&mapped_on, page + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; + (*npairs)++; } - pairs[*npairs].cpu_va = a; - pairs[*npairs].gpu_va = b; - pairs[*npairs].valid_hdr = ok; - (*npairs)++; } + nonz++; } - nonz++; } } @@ -3949,6 +4305,96 @@ static void svc_session_flush(void); /* defined next to the pending queue */ #define SVC_LEDGER_EVICT_RESET 256u static u32 svc_ledger_n; static u64 svc_ledger_diverges; +static u64 svc_ledger_pinwins; /* r336: pin-backed divergence wins */ +static u32 svc_d2hmap_log; /* D2H-MAP per-page destination budget */ +/* + * r336/r34x: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait + * state. The decoder captures EVERY acquire of the ring entry currently + * being decoded (svc_waits[], reset per entry at the top of + * svc_decode_stream), and svc_run_ops honors them before running any of + * the entry's work, so a copy cannot execute ahead of the dependency its + * push waits on (the streams2 readback-before-scale class). Per-entry + * capture is essential: the wait state used to leak across entries (a + * later launch's self-release wait, e.g. need=85 while the held push + * releases 84, gated an earlier entry that carries no acquire at all - + * the r340 over-hold). The re-decode re-parses the entry, so the state + * is refreshed on every pass. + * + * Semantics per op (class contract, see svc_wait_satisfied): + * 0 ACQUIRE = (v == p); 2 ACQ_STRICT_GEQ = v >= p; 3 ACQ_CIRC_GEQ = + * circular (s32)(v - p) >= 0; 4 ACQ_AND = (v & p) != 0; + * 5 ACQ_NOR = ~(v | p) != 0. + */ +#define SVC_WAIT_MAX 8 +/* + * r401: bounded acquire wait (fail-open), now a module parameter. The + * original 100 ms bound was shorter than a single GR launch's completion + * latency in this emulation (observed 0.5-1.6 s per launch), so a copy + * that waited on a launch's tracker (streams2: the D2H readback's + * ACQ_CIRC_GEQ on the scale channel's progress counter) gave up and ran + * BEFORE the launch wrote its output - the host read pre-scale data + * (bad=32767, c2 all wrong). 4 s covers several launch latencies; the + * bound still guarantees the entry is never parked forever (a genuinely + * unsatisfiable acquire fails open late, after the dependency's own work + * would have landed - late-but-correct instead of early-and-wrong). + */ +unsigned gsp_sema_wait_hold_ms = 4000; +struct svc_wait_op { + u64 va; + u32 payload; + u8 op; /* C56F SEM_EXECUTE operation */ + u32 off; /* method offset in the entry (order) */ +}; +static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; +static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; +static u32 svc_wait_ovf; /* acquires dropped past the array */ +static u64 svc_wait_held; +static u64 svc_wait_pass; +static u64 svc_wait_giveup; +static u32 svc_wait_log; /* per-session held-row print budget */ +static u32 svc_wait_pass_log; /* per-session resolved-row budget */ +static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ +static u32 svc_pitch_log; /* r402: 2D (pitched) op builds */ +static u32 svc_fill_raw_log; /* r365 FILL-RAW: raw dropped-fill dumps */ +static u32 svc_lowfill_log; /* r365b LOWFILL-WIT: below-floor fills */ +static u32 svc_tabcopy_log; /* r367 TABLE-COPY: table-page copies */ +static u32 svc_fill_noremap; /* r343 FILL: fail-closed holds */ +static u64 svc_wait_rel_va; /* this entry's own release (mirror) */ +static u32 svc_wait_rel_payload; +static u32 svc_wait_rel_off; /* its method offset (order check) */ +static bool svc_wait_rel_have; +/* + * Per-channel first-hold clock for the bounded wait: a gate evaluation + * that cannot satisfy the acquire arms a 100 ms budget per hold episode; + * expiry fails open (ops run) with a loud 'give-up' row. The table is + * tiny (a handful of channels), service-thread only, cleared per pass + * resolution / session flush. + */ +struct svc_wait_hold { + bool active; + u32 h; + unsigned long since; +}; +#define SVC_WAIT_HOLD_MAX 16 +static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX]; +static u32 svc_dst_div_log; /* r402: copy-dst ledger/probe divergence */ static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -3988,7 +4434,8 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + /* r534: sized for svc_scan_pass's 64 KB chunked guest reads. */ + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } @@ -4076,10 +4523,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_nocap_held, svc.gr_nocap_consumed, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, - svc.pool_alt_applied, svc.pool_residue); + svc.pool_alt_applied, svc.pool_residue, + svc_wait_held, svc_wait_pass, svc_wait_giveup, + svc_wait_ovf); if (svc.resc_run || svc_rescue_scan_capped) pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", svc.resc_run, svc_rescue_scan_capped); @@ -4097,11 +4546,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, svc_ledger_evictions, svc_ledger_evict_resets, - svc_ledger_diverges, - svc_ledger_reloc_diverges, svc_ledger_rewrites); + svc_ledger_diverges, svc_ledger_pinwins, + svc_ledger_reloc_diverges, svc_ledger_rewrites, + svc_ledger_pin_repairs, svc_pool_alt_skipped); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is @@ -4160,6 +4610,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 +#define CE_M_LINE_COUNT 0x41c +#define CE_M_PITCH_IN 0x410 +#define CE_M_PITCH_OUT 0x414 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -4234,6 +4687,19 @@ struct svc_memset_op { u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; + /* + * r402: 2D (pitched) transfers. rows > 1 selects row-wise mode: + * the op runs `rows` times with the destination advancing by + * dpitch (and the source by spitch) per row, each row carrying + * `bytes` (copy) or elem*count (fill) bytes. cudaMemset2D and + * cudaMemcpy2D issue exactly this shape (LINE_COUNT rows, + * PITCH_OUT/PITCH_IN in bytes); without it the fill/copy ran flat + * and the partial-width rows corrupted the neighbouring columns + * (victory_memset2d bad>12M, memcpy_roundtrip pitched class). + */ + u32 rows; + u32 dpitch; + u32 spitch; }; /* Decoder state carried across the methods of one push. */ @@ -4245,9 +4711,19 @@ struct svc_push_state { u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ + /* + * r402: 2D transfer geometry (CE class): PITCH_IN/PITCH_OUT in + * bytes and LINE_COUNT rows. A pitched fill/copy whose pitch + * exceeds the row length must run row-wise; a flat transfer leaves + * pitch == line_length (or no pitch method at all). + */ + u32 pitch_in, pitch_out; + u32 have_pitch_in, have_pitch_out; + u32 line_count; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; + u32 have_remap_a, have_remap_b, have_remap_c; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; @@ -4378,7 +4854,18 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * withholding the release corrupted the next session (r59: the late * memset overwrote refilled data after the client had passed the * release), so the two must move together. + * + * r383 (streams2 big-copy class): name the missed VA through the + * same stalled-VA demand channel the GR fill uses. The userspace pin + * stream watches the "stalled VA" row and pushes the VA to the + * in-guest scanner, so the pin lands on the next scanner pass + * instead of waiting out a full scan cadence. The hold itself is + * unchanged; this only shortens the window in which the entry sits + * op-pending (r377: a held H2D copy executed ~1.5 s late and read a + * staging page the client had since refilled with the next copy's + * bytes - the neighbor-data class). */ + svc_log_stalled_va(va & ~(u64)(SEMA_PAGE_SIZE - 1)); svc.op_tr_pending = 1; return -ENOENT; } @@ -4461,8 +4948,213 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined belo static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ +/* + * r384 (the guest boot-corruption lottery, root-caused 2026-10-20): + * page-table pages of the guest kernel's direct map were found ZEROED + * (two adjacent 4 KiB PTE pages at gpa 0x12d3c2000/0x3000 => two 2 MiB + * holes in the guest physmap, kernel objects inside them unreachable, + * multi-oops panic). The only emulation writers of page-sized ZERO runs + * are the virtual FILL executor (svc_fill / CE memsets), whose large + * >= SVC_VAS_FLOOR destinations resolve through the walk/scan when no + * pin and no ledger answer exists - and that resolver is demonstrably + * non-deterministic (four executions of the same 8 KiB engine-window + * clear resolved to four different guest pages, all refused by the + * below-floor guard). + * + * Two nets, mirroring the m169/r378 copy guards: + * 1. svc_fill_guard_hold(): a LARGE (>= 64 KiB) virtual fill whose only + * authority is the walk/scan (no pin, no ledger) is held through a + * short bounded window and its VA pushed to the pin-demand channel + * (the userspace pin stream injects the guest-pagemap truth), so the + * retried decode resolves through a real pin. Fail-open after the + * bound with the historical behaviour, like the copy guard. + * 2. svc_fill_target_is_pt(): a ZERO fill about to land in guest RAM on + * a page whose first 64 qwords are all PTE-shaped (present, NX, + * physical address in range) is REFUSED loudly. A page table page is + * never a legitimate zero-fill destination; this directly blocks the + * observed corruption shape even when some other resolver approved + * the target. Pin-backed resolutions are exempt (the guest's own + * pagemap is truth - if it says the VA maps the page, obey it). + */ +unsigned gsp_sema_fill_guard_ms = 800; +bool gsp_sema_fill_guard = true; +#define SVC_FILL_GUARD_SLOTS 16 +static struct { + u64 va_page; + unsigned long first; +} svc_fill_guard[SVC_FILL_GUARD_SLOTS]; +static u32 svc_fill_guard_next; +static u32 svc_fill_held; +static u32 svc_fill_pt_refused; +static u32 svc_fill_exec_log; /* r384 FILL-EXEC witness budget */ + +/* true = the guard may hold this fill; false = fail-open, run it. */ +static bool svc_fill_guard_hold(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i, slot = SVC_FILL_GUARD_SLOTS; + + for (i = 0; i < SVC_FILL_GUARD_SLOTS; i++) { + if (svc_fill_guard[i].va_page == page) { + if (time_before(now, svc_fill_guard[i].first + + msecs_to_jiffies( + gsp_sema_fill_guard_ms))) + return true; + if (svc_fill_guard[i].first) { + svc_fill_guard[i].first = 0; + pr_info("sema-svc: fill guard released without a pin va=0x%llx (using the walk translation)\n", + page); + } + return false; + } + if (!svc_fill_guard[i].va_page) + slot = i; + } + if (slot == SVC_FILL_GUARD_SLOTS) + slot = svc_fill_guard_next++ % SVC_FILL_GUARD_SLOTS; + svc_fill_guard[slot].va_page = page; + svc_fill_guard[slot].first = now; + return true; +} + +/* + * A guest-kernel page table page: the first SVC_PT_SCAN qwords are all + * present PTEs (bit 0), non-executable (bit 63 - kernel data mappings), + * with a physical address inside the 34-bit guest range. Random payload + * qwords pass all three conditions with probability far below the 75% + * threshold at SVC_PT_SCAN=64; a genuine table passes at 100%. + */ +#define SVC_PT_SCAN 64u +#define SVC_PT_MIN 48u +static bool svc_fill_target_is_pt(struct nv_vgpu_device *vgpu, u64 addr) +{ + u64 ents[SVC_PT_SCAN]; + u32 i, hits = 0; + + if (addr & (SEMA_PAGE_SIZE - 1)) + return false; /* only a page-aligned table shape */ + if (svc_read(vgpu, addr, ents, sizeof(ents))) + return false; + for (i = 0; i < SVC_PT_SCAN; i++) { + u64 e = ents[i]; + + if (!(e & 1ull)) + continue; + if (!(e & (1ull << 63))) + continue; + if ((e & 0x000ffffffffff000ull) == 0) + continue; + if ((e & 0x000ffffffffff000ull) >= (1ull << 34)) + continue; + hits++; + } + return hits >= SVC_PT_MIN; +} + +/* + * r391 per-page destination resolution for a VIRTUAL fill (the r390 + * guest-panic class). A device VA's pages are independent guest allocations + * - the pin stream shows split runs (va=0x7c36c7400000 -> 29 pages at + * 0x131b63000, the next run at 0x131a80000) - so resolving only the fill's + * first page and writing `total` bytes contiguously past it writes over + * whatever memory follows the first run. On r390 a 383620-byte zero fill + * did exactly that: the contiguous write covered the guest kernel's + * page-table pages (0x131ba1000/0x131ba2000/0x131bab000), the direct map + * grew holes, and the guest died with "Fatal exception in interrupt" + * (unmap_page_range / cpuacct / psi faults). svc_copy has always translated + * virtual operands per 4 KB page; the fill must do the same. + * + * Authority order mirrors the top-level resolution (and svc_copy's per-page + * dst): a fresh pin (guest pagemap truth) wins and repairs the landing + * table, then the landing table (the page a previous write of this VA + * landed on), then the resolver chain. A page with no resolution returns + * -ENOENT: the caller fails the fill loudly instead of writing through a + * contiguous guess. + */ +static int svc_fill_dst_page_pa(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out, u32 *prov_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0, lpa = 0; + u32 aper = 0, lap = 0; + + if (!svc_pin_lookup_any(page, &pa, &aper)) { + *prov_out = 1; + if (!svc_ledger_lookup(page, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-page va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + page, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(page, pa, aper); + } + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_ledger_lookup(page, &pa, &aper)) { + *prov_out = 2; + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_op_translate(vgpu, page, &pa, &aper)) { + *prov_out = 3; + *pa_out = pa; + *ap_out = aper; + return 0; + } + return -ENOENT; +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, + u32 elem, u32 count, u32 phys, u32 aperture, + u64 *done_out); +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, + size_t total, u32 dst_phys, u32 dst_aperture, + u32 src_phys, u32 src_aperture, u64 *done_out, + u32 owner_ch, u32 owner_idx, u64 owner_hash); + +/* + * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per + * row with the destination advancing by dpitch. Each row keeps the whole + * per-page resolution, ledger-store and guard machinery of the 1D path + * (no new resolution semantics), and a failed row's retry simply re-runs + * the already-written rows with the same pattern - idempotent for fills. + */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, - u32 count, u32 phys, u32 aperture, u64 *done_out) + u32 count, u32 phys, u32 aperture, u64 *done_out, + u32 rows, u32 dpitch) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_fill_1d(vgpu, addr + (u64)r * dpitch, + value, elem, count, phys, aperture, + &row_done); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_fill_1d(vgpu, addr, value, elem, count, phys, aperture, + done_out); +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, + u32 count, u32 phys, u32 aperture, u64 *done_out) { u8 pat[8]; u8 bounce[256]; @@ -4483,16 +5175,47 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, if (!phys) { /* Virtual destination: the VA shadow first (its pages stay - * authoritative), then the unified landing table (the page a - * previous write of this VA landed on - always the - * authoritative location of the content), then the pinned - * map and the VA->PA map. */ + * authoritative), then a live pin (the guest's newest + * statement - r372: a re-pinned VA must not be written + * through a stale landing), then the unified landing table + * (the page a previous write of this VA landed on - always + * the authoritative location of the content), then the + * pinned map and the VA->PA map. */ u64 pa; u32 aper; + u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ if (svc_vas_has(addr)) { shadow = true; + } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { + u64 lpa = 0; + u32 lap = 0; + + prov = 1; + /* + * r372 pin-wins: repair the landing table when it + * still names the pre-re-pin page, so every later + * reader follows the pin too. + */ + if (!svc_ledger_lookup(addr, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + addr, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(addr, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper); + } + addr = pa; + aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; } else if (!svc_ledger_lookup(addr, &pa, &aper)) { + prov = 2; addr = pa; aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; @@ -4502,9 +5225,101 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, shadow = true; svc.vas_shadowed++; } else { + prov = 3; addr = pa; aperture = aper; } + /* + * r365b LOWFILL-WIT: a below-floor virtual fill (the guest's + * vidmem-window page-table writes) is a corruption risk: its + * resolution must be proven before the op is allowed to run + * (r365: 16-byte and 32 KB zero fills landed in guest RAM and + * killed the guest kernel). Log every resolution bounded. + */ + if (va0 < SVC_VAS_FLOOR && svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill: va=0x%llx shadow=%d resolved=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, shadow, addr, aperture, elem, count, + value); + } + /* + * r369: a below-floor fill may ONLY land in vidmem. The + * 0x204020000 engine-window clear resolved to a DIFFERENT + * guest-RAM page on every execution (9 runs, 9 pages) and + * the guest died with its task_struct corrupted (gmain NULL + * deref in __fdget) - each wrong resolution zeroed 8 KB of + * guest kernel memory. Vidmem resolutions (the page-table + * region, ap=VID) are the emulation's own shadow and stay + * allowed; a SYS resolution for a window VA is refused and + * logged loudly. + */ + if (va0 < SVC_VAS_FLOOR && !shadow && + aperture != CE_APERTURE_VID) { + pr_err_ratelimited("vgsp-sema: LOWFILL-REFUSED va=0x%llx pa=0x%llx ap=%u elem=%u count=%u (below-floor fill resolved outside vidmem)\n", + va0, addr, aperture, elem, count); + return -ENOENT; + } + /* + * r384 net 1 (pin-demand hold for unproven large fills): a + * >= 64 KiB virtual fill whose only authority is the + * walk/scan resolver (no pin, no landing-table entry) AND + * which would land in GUEST RAM (SYS aperture) is the class + * that zeroed the guest's page-table pages (see the guard + * commentary above svc_fill_guard_hold). Demand the VA + * through the pin channel and hold the op through a short + * bounded window so the retried decode resolves through the + * guest's own pagemap; fail open with the historical + * behaviour after the bound. Vidmem-shadow resolutions are + * the emulation's own memory and stay immediate. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + total >= (64u << 10) && prov >= 3 && + aperture == CE_APERTURE_SYS) { + svc_log_stalled_va(va0); + if (svc_fill_guard_hold(va0)) { + svc_fill_held++; + if (svc_fill_held <= 32) + pr_info("sema-svc: FILL-HOLD va=0x%llx resolved=0x%llx ap=%u total=%zu value=0x%llx (unproven large fill; pin demanded)\n", + va0, addr, aperture, total, + value); + return -ENOENT; + } + } + /* + * r384 net 2 (page-table-shape refusal): a ZERO fill about + * to land in guest RAM on a page whose first 64 qwords are + * all present+NX+in-range PTEs is refused loudly - a page + * table page is never a legitimate zero-fill destination. + * This blocks the observed corruption shape (two adjacent + * PTE pages zeroed => two 2 MiB holes in the guest physmap) + * even when some other resolver approved the target, and + * irrespective of how wrong the resolution is. Pin-backed + * resolutions are exempt: the guest's own pagemap is truth + * there. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + value == 0 && total >= SEMA_PAGE_SIZE && + aperture == CE_APERTURE_SYS && prov != 1 && + svc_fill_target_is_pt(vgpu, addr)) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill target is a page table)\n", + va0, addr, prov, total); + svc_log_stalled_va(va0); + return -ENOENT; + } + /* + * r384 witness: where every >= 4 KiB virtual fill actually + * lands (va, pa, aperture, resolution provenance, pattern) - + * the decisive line for the boot-corruption forensics (the + * large zero fills were invisible until now). Bounded. + */ + if (va0 >= SVC_VAS_FLOOR && !shadow && + total >= SEMA_PAGE_SIZE && svc_fill_exec_log < 64) { + svc_fill_exec_log++; + pr_info("sema-svc: FILL-EXEC va=0x%llx pa=0x%llx ap=%u prov=%u total=%zu value=0x%llx\n", + va0, addr, aperture, prov, total, value); + } } if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && @@ -4530,20 +5345,29 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, /* * C2 (data class) + m180 unification: a large virtual fill is a * write/read pair with the memset's readers like any H2D copy - - * record its landing pages (SYS and VID alike) so the engine fill - * and the D2H read of this VA use exactly the page the fill wrote. + * record its landing pages so the engine fill and the D2H read of + * this VA use exactly the page the fill wrote. Only the VID + * (shadow) case is contiguous; a SYS landing is recorded per page + * in the loop below (r391: the contiguous `addr + po` store also + * poisoned the table with pages the fill never wrote). */ if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && - total >= (64u << 10)) { - u32 cap = (aperture == CE_APERTURE_VID) ? - SVC_APERTURE_FB : SVC_APERTURE_SYS; + total >= (64u << 10) && aperture == CE_APERTURE_VID) { u64 po; for (po = 0; po < total; po += SEMA_PAGE_SIZE) - svc_ledger_store(va0 + po, addr + po, cap); + svc_ledger_store(va0 + po, addr + po, + SVC_APERTURE_FB); } - if (aperture == CE_APERTURE_VID) { + /* + * r391: a virtual device destination is translated per 4 KB page + * (see svc_fill_dst_page_pa) - never written through the first + * page's PA contiguously. The whole-range shadow write below only + * applies to a resolution that is contiguous by construction. + */ + if (aperture == CE_APERTURE_VID && + !(va0 >= SVC_VAS_FLOOR && !phys && !shadow)) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); @@ -4563,14 +5387,86 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; + u64 wpa = addr; + u32 wap = aperture; + bool per_page = !phys && !shadow && va0 >= SVC_VAS_FLOOR; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); + /* + * r391: resolve THIS page's own landing (see + * svc_fill_dst_page_pa) and clamp the chunk to the page + * boundary - a virtual VA's pages are independent guest + * allocations and must never be written through the first + * page's PA contiguously. + */ + if (per_page) { + u32 off = (u32)((va0 + o) & (SEMA_PAGE_SIZE - 1)); + u64 ppa = 0; + u32 pap = 0, prov = 0; + int prc; + + chunk = min_t(size_t, chunk, SEMA_PAGE_SIZE - off); + prc = svc_fill_dst_page_pa(vgpu, va0 + o, &ppa, + &pap, &prov); + if (prc) { + if (done_out) + *done_out = (u64)o; + /* + * r391: demand the exact page through the pin + * channel (the userspace stream pushes this VA + * to the in-guest scanner) and hold the fill + * for a bounded retry - never write through a + * contiguous guess past a missing page. + */ + svc_log_stalled_va((va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + pr_err_ratelimited("vgsp: FILL-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill (page 0x%llx unresolved)\n", + va0, (u64)total, (u64)o, + (va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + return -ENOENT; + } + wap = (pap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wpa = (ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) + off; + /* + * r391 net: the r384 page-table-shape refusal + * applied to EVERY page of a virtual zero fill. + * The r390 fill's first page was the client's + * buffer, the later pages were the guest's PTE + * pages - the first-page-only check could not see + * them. A page-table-shaped guest-RAM page is never + * a legitimate zero-fill target: skip it loudly + * instead of zeroing it. Pin-backed resolutions + * are exempt (the guest's pagemap is truth there). + */ + if (wap == CE_APERTURE_SYS && value == 0 && prov != 1 && + svc_fill_target_is_pt(vgpu, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill page is a page table; page skipped)\n", + va0 + o, wpa, prov, total); + o += chunk; + continue; + } + } for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); - if (svc_write(vgpu, addr + o, bounce, chunk)) { + if (wap == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(vgpu); + + if (!base || wpa + chunk > size) { + if (done_out) + *done_out = (u64)o; + return -EINVAL; + } + memcpy(base + wpa, bounce, chunk); + } else if (svc_write(vgpu, wpa, bounce, chunk)) { if (done_out) *done_out = (u64)o; if (total >= SVC_COPY_AUDIT_MIN) @@ -4578,6 +5474,15 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, va0, (u64)total, (u64)o, -EIO); return -EIO; } + /* + * r391: record this page's landing (the readers follow the + * table) - SYS and VID alike, exactly like the copy path. + */ + if (per_page) + svc_ledger_store(va0 + o, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (wap == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS); o += chunk; } if (done_out) @@ -4697,7 +5602,7 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; -struct svc_gr_cand { +struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ /* @@ -4733,7 +5638,105 @@ struct svc_gr_cand { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; -static struct svc_gr_cand svc_gr_cand; +/* + * fail4: the capture was a single slot; two concurrent launches (streams2's + * vadd pair on two channels) overwrote each other's candidate while the + * first was kept for a retry (the candidate-overwrite witness) and the + * engines ran crossed parameters (c1 == a2+b2). Keep up to four candidates + * side by side in a ring; the accessor macro addresses the current slot, + * so every existing use keeps working on the selected candidate, and + * capture/consume select the slot instead of clobbering it. + */ +#define SVC_GR_CAND_RING 4 +static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; +static u32 svc_gr_cand_pos; +#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) +/* + * r383: push-VA history of recent captures, beyond the (four-slot) + * candidate ring's lifetime. A replayed decode of a complete QMD asks + * "has this launch ever been captured?"; the ring alone can have rotated + * past it, and a wrong "no" would re-capture (and re-execute) an already + * forwarded launch. Bounded, service thread only. + */ +#define SVC_GR_CAP_HIST 8 +static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST]; +static u32 svc_gr_cap_hist_pos; + +/* + * r401: robust session-scoped "already captured" memory for the r383 + * replay-capture path. The 4-slot candidate ring + 8-VA history evicted a + * real capture within ~12 later captures, so a true replay of an + * already-forwarded launch (identical push VA and content) was + * re-captured and re-executed: the r399 soak log shows the same nine + * launch pushes re-submitted every ~10-11 s for minutes (idempotent + * kernels survive it; accumulators would not). The open-addressed table + * keeps the (push_va, push_hash) identity of every capture of the + * session, so a true replay always classifies as "seen" (LAUNCH-SKIP, + * exactly once) while a genuinely new push - even at the same VA with + * the same entry words - is captured through the normal + * `trusted && !replay` path and never consults this table. Cleared by + * svc_session_flush (a new client's identical-payload pushes restart + * their release counters, so a boot-scoped table would wrongly suppress + * them). Service-thread only. + */ +#define SVC_GR_SEEN_SLOTS 2048 +#define SVC_GR_SEEN_PROBE 8 +static struct { + u64 push_va; + u64 hash; +} svc_gr_seen[SVC_GR_SEEN_SLOTS]; +static u64 svc_gr_seen_stale; /* probe-chain evictions (telemetry) */ + +static u32 svc_gr_seen_mix(u64 push_va, u64 hash) +{ + u64 k = push_va * 0x9E3779B97F4A7C15ULL; + + k ^= hash + 0x9E3779B97F4A7C15ULL + (k << 6) + (k >> 2); + return (u32)(k % SVC_GR_SEEN_SLOTS); +} + +static bool svc_gr_seen_lookup(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va) + return false; + if (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash) + return true; + } + return false; +} + +static void svc_gr_seen_store(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va || + (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash)) { + svc_gr_seen[k].push_va = push_va; + svc_gr_seen[k].hash = hash; + return; + } + } + /* Full probe chain: replace the tail (oldest-ish) entry. */ + svc_gr_seen_stale++; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].push_va = + push_va; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].hash = + hash; +} static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); @@ -4769,7 +5772,7 @@ static_assert(sizeof(struct svc_gr_page) == 24, static u64 svc_push_hash(const u8 *p, u32 len); /* QMD field decode: the exact shape the host CUDA bridge uses. */ -static void svc_gr_qmd_decode(struct svc_gr_cand *c) +static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) { const u32 *q = c->qmd; u32 i; @@ -4814,6 +5817,25 @@ static void svc_gr_cand_capture(struct svc_channel *ch, * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined * later in the file and this site may not reference it). */ + /* + * fail4: never overwrite a DIFFERENT launch's unconsumed candidate - + * move to a free ring slot instead (same-push re-captures still + * refresh in place). + */ + if (svc_gr_cand.valid && !svc_gr_cand.offered && + svc_gr_cand.push_va != push_va) { + u32 ri; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + svc_gr_cand_pos = rp; + break; + } + } + } if (svc_gr_cand.valid && svc_cand_ovw_log < 64) { svc_cand_ovw_log++; pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n", @@ -4825,8 +5847,14 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; + /* r383: remember the identity beyond this ring slot's lifetime. */ + svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va; + svc_gr_cap_hist_pos++; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); + /* r401: the session-wide exactly-once memory (survives ring rotation; + * see svc_gr_seen_lookup). */ + svc_gr_seen_store(push_va, svc_gr_cand.push_hash); if (push && push_len >= 4) { u32 total = push_len / 4; u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; @@ -4919,8 +5947,14 @@ static bool svc_fwd_defer_arm(void) { if (!gsp_phys_guest_mem_enabled()) return false; - if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) - return true; + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered) + return true; + } return svc_fwd_cand.valid && gsp_phys_work_defer_release_enabled(); } @@ -5185,9 +6219,228 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va, return (rc_); \ } while (0) +/* + * fail4: pipeline-order source snapshot for held copies. The m169 + * device-dst guard defers a staged H2D (host->device) for up to 3 s; + * the driver's ping-pong staging buffers are overwritten by the next + * chunk meanwhile, so the deferred read returns the content of a later + * chunk (vector_add: device a[i] pages held a[i+2 MB] - exactly the + * 2x1 MB staging stride). A held copy snapshots its source at hold + * time and the retried execution copies from the snapshot, matching + * the hardware's in-order read of the staging. + */ +#define SVC_COPY_SNAP_N 16 +#define SVC_COPY_SNAP_MAX (4u << 20) +#define SVC_COPY_SNAP_TTL_MS 30000u +struct svc_copy_snap { + u64 dst, src; + size_t total; + /* + * Owner identity (r444): the channel, ring entry and push-content + * hash of the decode pass that captured this snapshot. A snapshot + * exists to serve ONE entry's retried copy (the held staging read); + * keying it by (dst,src,total) alone let a snapshot captured for an + * earlier entry poison a later copy with the same operand triple - + * the wedge_ab "bad = mutated-element count" class: step N's H2D + * executed from step N-1's snapshot because the re-decode of the + * completed step N-1 entry re-captured its (unchanged) source after + * its copy had already run, and step N's copy then consumed it. + * The pass hash changes whenever the guest rewrites the push, so a + * ring slot reused by a fresh push can never match a stale owner. + */ + u64 owner_hash; + u32 owner_ch; + u32 owner_idx; + u8 *buf; + unsigned long stamp; +}; +static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N]; + +static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) + return svc_copy_snaps[i].buf; + return NULL; +} + +static void svc_copy_snap_free(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + return; + } +} + +/* + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. + */ +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } +} + +/* Capture the source of a virtual-operand copy about to be held. */ +static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, + u64 src, size_t total, u32 owner_ch, + u32 owner_idx, u64 owner_hash) +{ + struct svc_copy_snap *s = NULL; + u8 *buf; + size_t o; + u32 i; + + if (!total || total > SVC_COPY_SNAP_MAX) + return; + if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, + owner_hash)) + return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (!svc_copy_snaps[i].buf) { + s = &svc_copy_snaps[i]; + break; + } + if (time_after(jiffies, svc_copy_snaps[i].stamp + + msecs_to_jiffies(SVC_COPY_SNAP_TTL_MS))) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + s = &svc_copy_snaps[i]; + break; + } + } + if (!s) + return; /* all slots live: keep the historical behaviour */ + buf = kmalloc(total, GFP_KERNEL); + if (!buf) + return; + for (o = 0; o < total; ) { + size_t chunk = min_t(size_t, 0x1000 - ((src + o) & 0xfff), + total - o); + u64 pa = 0; + u32 ap = 0; + + /* The source is a virtual host VA; resolve per page first. */ + if (svc_op_translate(vgpu, src + o, &pa, &ap) || + svc_read(vgpu, pa, buf + o, chunk)) { + kfree(buf); + return; + } + o += chunk; + } + s->dst = dst; + s->src = src; + s->total = total; + s->owner_ch = owner_ch; + s->owner_idx = owner_idx; + s->owner_hash = owner_hash; + s->stamp = jiffies; + s->buf = buf; + pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n", + dst, src, total, owner_ch, owner_idx, + owner_hash, + total >= 4 ? *(const u32 *)buf : 0, + total >= 8 ? *(const u32 *)(buf + 4) : 0, + total >= 12 ? *(const u32 *)(buf + 8) : 0, + total >= 16 ? *(const u32 *)(buf + 12) : 0); +} + +static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, + u64 pa, u32 off, u32 bytes, u64 *hpa, + u32 *out_off); + +/* r402 row-wise wrapper: see svc_fill. The source advances by spitch. */ +/* r402: read-only resolver probe (defined with the GR fill diagnostics). */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out) + u64 *done_out, u32 rows, u32 dpitch, u32 spitch, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, + src + (u64)r * spitch, total, + dst_phys, dst_aperture, src_phys, + src_aperture, &row_done, + owner_ch, owner_idx, owner_hash); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture, done_out, + owner_ch, owner_idx, owner_hash); +} + +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, + u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash) { u8 bounce[256]; size_t o; @@ -5198,12 +6451,28 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_prov = 0, src_prov = 0; bool bigcopy_this = false; bool audit = total >= SVC_COPY_AUDIT_MIN; + u8 *snap = NULL; static u64 bigcopy_log; + bool trace64 = (total == 65536u); + const char *dir = "?"; if (done_out) *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture); + } + snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (!dst_phys) { u64 pa; @@ -5213,12 +6482,68 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, dst_shadow = true; } else if (!svc_ledger_lookup(dst, &pa, &aper)) { /* + * r402 diagnostic (bounded): name the case where the + * ledger's landing page for a copy DESTINATION + * disagrees with the live resolver (pin/walk). The + * D2H staging writes failed on the class this + * witness detects: the writer follows the ledger + * while the guest's reader holds a different page, + * so the client reads stale data (streams2 c1 + * zeros / smiley 3-of-4 passes). Read-only probe + * (svc_gr_src_probe stores nothing). + */ + if (gsp_sema_dbg && svc_dst_div_log < 64 && + dst >= SVC_VAS_FLOOR) { + u64 wpa = 0; + u32 wap = 0; + const char *src = + svc_gr_src_probe(vgpu, dst, &wpa, &wap); + + if (wpa && + (((wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + ((wap == SVC_APERTURE_FB) != + (aper == SVC_APERTURE_FB)))) { + svc_dst_div_log++; + pr_err_ratelimited("vgsp-sema: DST-DIVERGE va=0x%llx ledger=0x%llx/%u probe=%s 0x%llx/%u (write follows ledger)\n", + dst, pa, aper, src, wpa, wap); + } + } + /* * m180 unification: a previous write of this VA is * the authority for where the content lives (and so * for where a new write lands); it also spares the * m169 stale-walk hold below, which exists precisely * because no pin was known. + * + * r376 pin-agreement: a FRESH pin that disagrees is + * newer than the recorded landing - the guest freed + * or moved the buffer - so the pin wins and the + * entry is repaired (the same rule the fills and the + * launch paths apply; without it a stale entry + * writes a freed page - the guest-corruption class). */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + dst, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(dst, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } dst = pa; dst_aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; @@ -5275,9 +6600,49 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", dst_orig, dst_prov); + /* + * fail4: the staging source is valid NOW; snapshot it so + * the deferred execution copies the pipeline-order bytes. + */ + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, src_orig, + total, owner_ch, owner_idx, + owner_hash); return -ENOENT; } /* + * r378: the same stale-table class for SYS destinations resolved by + * the walk/scan (prov >= 3): a FRESH injected pin that disagrees is + * the guest's newest statement, so writing through the resolved page + * lands on a page the guest has freed/re-homed (the guest-corruption + * class). Hold through the same bounded re-decode path instead of + * writing; a missing pin keeps the historical behaviour (the + * resolution is then the write authority). + */ + if (dst_virt && dst_aperture == CE_APERTURE_SYS && + dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && + gsp_sema_copy_dst_pin_guard) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_orig, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dst & ~(u64)(SEMA_PAGE_SIZE - 1))) || + svc_ap_of(iap) != SVC_APERTURE_SYS) && + svc_dst_guard_hold(dst_orig)) { + svc.copy_dst_hold++; + if (svc.copy_dst_hold <= 16) + pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u pin=0x%llx (stale SYS target)\n", + dst_orig, dst_prov, ipa); + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, + src_orig, total, + owner_ch, owner_idx, + owner_hash); + return -ENOENT; + } + } + /* * r231 READ-ORDER guard/witness: a device-src copy whose source VA * has a PENDING device-dst write hold reads the PRE-write content - * the writer's bytes land only when the guard resolves while the @@ -5301,6 +6666,49 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc_vas_has(src)) { src_shadow = true; + } else if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src, &pa, &aper)) { + /* + * r375 read-after-write authority: the landing table + * records where every write of this VA landed (writes + * store it, fresh pins repair it). A reader MUST read + * exactly that page or the engine's output is invisible + * (the sum=0 class). The walk/pin sources can answer + * differently - e.g. the same numeric pa in a different + * aperture - so the table wins for device VAs; only a + * VA with no recorded landing falls through to the + * resolver below. + * + * r376 pin-agreement: unless a FRESH pin disagrees - + * then the pin is newer (the guest re-pinned the VA) + * and wins, and the entry is repaired. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-src va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + src, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(src, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } + src = pa; + src_aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + src_prov = 1; + src_virt = true; } else if (svc_op_translate(vgpu, src, &pa, &aper)) { u64 lpa = 0; u32 lap = 0; @@ -5330,6 +6738,16 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, src_aperture = aper; src_prov = svc.op_tr_src; src_virt = true; + /* + * r375b: do NOT record a resolver answer into the + * landing table here. The resolver can return a stale + * PTE page (the m169 class) and the table is the WRITE + * authority - a poisoned entry makes every later write + * of this VA land on the stale page (r375 boot: slab + * GPF in kmem_cache_alloc at guest t=77s, before any + * client). The table is fed by writes and fresh pins + * only. + */ } } @@ -5341,8 +6759,33 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, */ if ((total >= SVC_COPY_AUDIT_MIN || (dst_virt && dst_orig >= SVC_VAS_FLOOR)) && - bigcopy_log < gsp_sema_bigcopy_log_max) { + (bigcopy_log < gsp_sema_bigcopy_log_max || total == 65536u)) { u8 sb[16] = {0}, db[16] = {0}; + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + bool trace64 = (total == 65536u); + const char *dir = + (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; + + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + dir = (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; bigcopy_log++; bigcopy_this = true; @@ -5366,10 +6809,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else { svc_read(vgpu, dst, db, sizeof(db)); } - pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n", + pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu vbase=%px\n", dst_orig, dst, dst_aperture, dst_prov, dst_shadow, src_orig, src, src_aperture, src_prov, src_shadow, - total); + total, (void *)vgpu->bars[1].vaddr); + /* + * r348 diagnostic: for a high-VA SYS source name the exact + * host PA this read will touch (gpa_to_hpa) - the direct + * comparison against the svc-fwd-hpa writeback line. + */ + if (src_orig >= SVC_VAS_FLOOR && + src_aperture == CE_APERTURE_SYS) { + u64 shpa = 0; + u32 soff = 0; + + if (!svc_fwd_operand_hpa(vgpu, SVC_APERTURE_SYS, + src, 0, SEMA_PAGE_SIZE, + &shpa, &soff) && !soff) + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> hpa=0x%llx\n", + src_orig, src, shpa); + else + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> UNMAPPED\n", + src_orig, src); + } pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n", sb, db); /* @@ -5379,7 +6841,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * (the engine-authored page as ints). The expected content is * ints 0,1,2,3,... for the probe A buffer. */ - if (total == 65536u && svc_atrace_log < 128u) { + if (trace64 && svc_atrace_log < 128u) { u32 i0, i1, i2, i3, j0 = 0, j1 = 0, j2 = 0, j3 = 0; svc_atrace_log++; @@ -5391,9 +6853,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, memcpy(&j1, db + 4, 4); memcpy(&j2, db + 8, 4); memcpy(&j3, db + 12, 4); - pr_info("vgsp: A-TRACE pt=%s dst_va=0x%llx dst_pa=0x%llx dst_prov=%u | src_va=0x%llx src_pa=0x%llx src_prov=%u | src_int4=%d %d %d %d dst_int4_before=%d %d %d %d\n", - dst_shadow || dst_aperture == CE_APERTURE_VID ? - "P1" : "P4", + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_prov=%u | src_va=0x%llx src_pa=0x%llx src_prov=%u | src_int4=%d %d %d %d dst_int4_before=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P1" : "P4", dir, dst_orig, dst, dst_prov, src_orig, src, src_prov, i0, i1, i2, i3, j0, j1, j2, j3); } @@ -5589,7 +7050,22 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, chunk = min_t(size_t, chunk, 0x1000 - ((dst_orig + o) & 0xfff)); - if (src_shadow) { + if (snap) { + /* + * r314: consume the decode-time staging snapshot. + * The capture machinery existed since cb50282b but + * the execution never read it (lookup at svc_copy's + * top, free on success, NO consumer) - every copy + * read the live staging, which the driver had + * refilled once the push's fence fired (the + * deterministic 786335-bad vector_add output, + * identical across boots). Reading from the + * snapshot also needs no live source translation, + * so the copies execute on their first allowed + * pass instead of storming 60+ rc=-2 retries. + */ + memcpy(bounce, snap + o, chunk); + } else if (src_shadow) { if (svc_vas_read(src_orig + o, bounce, chunk)) SVC_COPY_TRUNC_RET(-EIO); } else if (src_virt) { @@ -5598,7 +7074,34 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 lpa = 0; u32 lap = 0; - if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { + /* + * r375: the landing table is the read-after-write + * authority for device VAs (see the top-level src + * resolve). Per page, the table wins whenever it has + * an entry; the resolver below only answers a VA the + * table has never seen. r376: a fresh disagreeing pin + * is newer than the table and wins (repaired). + */ + if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap)) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src_orig + o, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)lap)) { + svc_ledger_pin_repairs++; + svc_ledger_store(src_orig + o, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + lpa = ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + lap = (u32)iap; + } + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { /* * m180 unification: a per-page resolution miss * must not fail the whole copy when the unified @@ -5650,6 +7153,21 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, SVC_COPY_TRUNC_RET(-EIO); } + /* + * r367 TABLE-COPY witness: a page-table page/entry copy is + * the guest's authoritative table content; dump the carried + * bytes so a table that later walks as ZERO can be traced to + * the copy that built it (or proven the source was already + * zero). Bounded; table-shaped sizes only. + */ + if (!o && (total == 4096 || total == 16) && + svc_tabcopy_log < 64) { + svc_tabcopy_log++; + pr_info("sema-tabcopy: dst_orig=0x%llx src_orig=0x%llx total=%zu dphys=%u dvirt=%d svirt=%d r_dst=0x%llx r_src=0x%llx first16=%16phN\n", + dst_orig, src_orig, total, dst_phys, dst_virt, + src_virt, dst, src, bounce); + } + if (dst_shadow) { if (svc_vas_write(dst_orig + o, bounce, chunk)) SVC_COPY_TRUNC_RET(-ENOMEM); @@ -5662,6 +7180,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 aper = 0; u64 lpa = 0; u32 lap = 0; + u32 wprov = 0; if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) { /* @@ -5674,26 +7193,76 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, pa = lpa; aper = (lap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF1; } else if (dst_orig >= SVC_VAS_FLOOR && !svc_ledger_lookup(dst_orig + o, &lpa, &lap) && ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || (lap == SVC_APERTURE_FB) != (aper == CE_APERTURE_VID))) { - /* - * m180 unification: keep ONE authoritative - * page per VA. A new write goes to the page - * the previous content landed on, and the - * fill/D2H readers (which prefer the table) - * stay consistent with it. - */ svc_ledger_diverges++; - if (svc_ledger_diverges <= 32) - pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", - dst_orig + o, lpa, pa); - pa = lpa; - aper = (lap == SVC_APERTURE_FB) ? - CE_APERTURE_VID : CE_APERTURE_SYS; + if (svc.op_tr_src == 1) { + /* + * r336 PIN-WINS (the r335 vector_add + * guest-panic class): a pin-backed + * translation (guest pagemap truth, + * op_tr_src == 1) that disagrees + * with the ledger means the guest + * remapped or reused this VA. The + * ledger page is where an earlier + * write landed; the guest has since + * freed or moved it, so writing it + * can land on memory the guest + * kernel now owns (observed: RM + * object fields overwritten with + * client c-data, slab freelist + * corruption, guest panic). Trust + * the pin for the write and re-home + * the ledger (the store after the + * write records the fresh page); + * readers then converge on the same + * page too. + */ + svc_ledger_pinwins++; + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx (pin wins)\n", + dst_orig + o, lpa, pa); + wprov = svc.op_tr_src; + } else { + /* + * m180 unification: keep ONE + * authoritative page per VA. A new + * write goes to the page the + * previous content landed on, and + * the fill/D2H readers (which + * prefer the table) stay consistent + * with it. Non-pin translations + * (2..5) can carry stale PTEs + * (m169), so the write history + * stays the authority there. + */ + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", + dst_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF0; + } + } else { + wprov = svc.op_tr_src; + } + /* + * D2H-MAP (diagnostic): per-page destination + * resolution of large host-dest copies - the direct + * witness for which guest pages a readback lands on. + * Bounded; one line per 4 KB page of copies >= 1 MB. + */ + if (total >= (1u << 20) && !(o & (SEMA_PAGE_SIZE - 1)) && + svc_d2hmap_log < 4096u) { + svc_d2hmap_log++; + pr_info("sema-d2hmap: va=0x%llx o=%zu pa=0x%llx ap=%u prov=%u\n", + dst_orig + o, o, pa, aper, wprov); } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; @@ -5772,7 +7341,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * the copy - for H2D this is the landed upload (P2), for D2H * the host-side readback result (P4R). */ - if (total == 65536u && svc_atrace_log < 128u) { + if (trace64 && svc_atrace_log < 128u) { u32 i0, i1, i2, i3; svc_atrace_log++; @@ -5780,9 +7349,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, memcpy(&i1, da + 4, 4); memcpy(&i2, da + 8, 4); memcpy(&i3, da + 12, 4); - pr_info("vgsp: A-TRACE pt=%s dst_va=0x%llx dst_pa=0x%llx dst_int4_after=%d %d %d %d\n", - dst_shadow || dst_aperture == CE_APERTURE_VID ? - "P2" : "P4R", + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_int4_after=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P2" : "P4R", dir, dst_orig, dst, i0, i1, i2, i3); } } @@ -5826,6 +7394,27 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, pr_info("vgsp: COPYAUDIT path=ce-soft-copy va=0x%llx len=0x%llx done=0x%llx\n", dst_orig, (u64)total, (u64)o); } + /* + * m180 parity with svc_fill: a large virtual-destination copy is a + * write/read pair with the launch rec-build readers - record its + * landing pages so the GR fill follows the page the copy actually + * wrote. Without this, a rec table built before the pins landed + * kept the stale pre-pin pages while the copy landed on the pinned + * page (streams2 r254: the b1 rec range carried the a2 pages and + * the vadd read a2's pattern as its second input). + */ + if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && + total >= (64u << 10)) { + u32 cap = (dst_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + size_t po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(dst_orig + po, dst + po, cap); + } + if (snap) + svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (done_out) *done_out = (u64)o; /* o == total on the success path */ return 0; @@ -5970,13 +7559,145 @@ static void svc_handle_map_add(void); /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ + +/* + * r34x: evaluate a C56F SEM_EXECUTE acquire against the semaphore word. + * The class defines the opcodes (clc56f.h NVC56F_SEM_EXECUTE_OPERATION: + * 0 ACQUIRE, 1 RELEASE, 2 ACQ_STRICT_GEQ, 3 ACQ_CIRC_GEQ, 4 ACQ_AND, + * 5 ACQ_NOR, 6 REDUCTION) and the CUDA driver documents the wait + * conditions each op must implement (cuda_device.h CUIsemaphoreAquire- + * Flags_enum: GEQ default = circular >=, EQUAL = ==, AND = (v & p) != 0, + * NOR = ~(v | p) != 0, STRICT_GEQ = >= with no wrap; ampere_semaphore.c + * maps those flags onto the opcodes: EQUAL->0, STRICT_GEQ->2, GEQ->3, + * AND->4, NOR->5). Note op 0 is EQUALITY, not a >=: only op 2/3 are + * ordering compares. 32-bit payloads: every observed CUDA acquire is + * PAYLOAD_SIZE 32BIT (SEM_PAYLOAD_HI == 0); a 64-bit acquire is captured + * and judged on the low word only (its high word is not modelled). + */ +static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) +{ + switch (op) { + case 0: /* ACQUIRE: wait for v == p */ + return cur == need; + case 2: /* ACQ_STRICT_GEQ: v >= p, no wrap */ + return cur >= need; + case 3: /* ACQ_CIRC_GEQ: v in [p, p + 2^31 - 1] (wrap allowed) */ + return (s32)(cur - need) >= 0; + case 4: /* ACQ_AND: (v & p) != 0 */ + return (cur & need) != 0; + case 5: /* ACQ_NOR: ~(v | p) != 0 */ + return ~(cur | need) != 0; + default: + return true; + } +} + +/* r34x: the per-channel first-hold clock for the bounded wait. */ +static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) +{ + struct svc_wait_hold *free_slot = NULL; + int i; + + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h) + return e; + if (!e->active && !free_slot) + free_slot = e; + } + if (!create || !free_slot) + return NULL; + memset(free_slot, 0, sizeof(*free_slot)); + free_slot->active = true; + free_slot->h = h; + free_slot->since = jiffies; + return free_slot; +} + +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (10000 ms, + * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the + * latch (used by svc_run_ops) precedes that define. + */ +#define SVC_OPFAIL_WINDOW_MS 10000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, + u32 entry_idx, u64 pass_hash) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -5988,6 +7709,140 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * output was a_stale + b_fresh). */ svc_handle_map_add(); + /* + * r336/r34x: honor every host-semaphore acquire (C56F SEM_EXECUTE + * op 0/2/3/4/5) captured for THIS ring entry before any of its work + * runs, so a copy cannot execute ahead of the dependency its push + * waits on (the streams2 readback-before-scale class). Decision + * ladder per wait: + * 1. saved value satisfies the op's condition -> pass; + * 2. the entry's own release, when its method offset precedes the + * acquire's (release-then-acquire, channel.c crossEngineSync), + * counts as the candidate value - the release method has + * already executed by the time the acquire is evaluated; + * an acquire that precedes its own trailing tracking release + * does NOT count it (nothing has installed that value yet); + * 3. drain the pending-translation release queue (releases OUR + * machinery is merely late about) and re-read once; + * 4. still unsatisfied -> hold the entry (release withheld through + * the op-pending machinery) for at most gsp_sema_wait_hold_ms + * per episode, then fail open loudly ('give-up': the value + * cannot be installed by firmware alone - it sits behind an + * upstream opdef/operand hold - so the old pass-through + * behaviour is safer than parking the channel forever). + * Unreadable semaphore: fail-open (the pre-r336 behaviour). + * The pass/hold/give-up decisions are logged as 'sema-wait' rows + * (op, payload, observed value, decision). + */ + if (svc_wait_n && gsp_sema_wait_gate) { + struct svc_wait_hold *gh; + bool hold = false; + u32 wi; + /* + * Kill switch: a wait that never satisfies (a counter our + * emulation does not advance the way the driver expects) + * would otherwise park entries until the sweep expires each + * one. After 2048 holds in a boot the gate fails open + * permanently (the pre-r336 behaviour) and says so once. + */ + if (svc_wait_held > 2048) { + if (svc_wait_log < 4096) { + svc_wait_log++; + pr_info("sema-wait: gate disabled after %llu holds (fail-open)\n", + svc_wait_held); + } + svc_wait_pass++; + goto svc_wait_done; + } + gh = svc_wait_hold_get(ch_h, true); + for (wi = 0; wi < svc_wait_n; wi++) { + const struct svc_wait_op *w = &svc_waits[wi]; + u32 cur = 0, waited_ms = 0; + bool ok = true, same_rel = false, pend = false; + + if (gh) + waited_ms = jiffies_to_msecs(jiffies - gh->since); + if (svc_sem_read(vgpu, w->va, &cur)) + continue; /* unreadable: fail-open */ + ok = svc_wait_satisfied(cur, w->op, w->payload); + if (!ok && svc_wait_rel_have && + svc_wait_rel_va == w->va && + svc_wait_rel_off < w->off) { + same_rel = true; + ok = svc_wait_satisfied(svc_wait_rel_payload, + w->op, w->payload); + } + if (!ok) { + svc_apply_pending(vgpu); + if (!svc_sem_read(vgpu, w->va, &cur)) + ok = svc_wait_satisfied(cur, w->op, + w->payload); + } + if (ok) { + if (svc_wait_pass_log < 256) { + svc_wait_pass_log++; + pr_info("sema-wait: pass ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u waited_ms=%u -> ops run this pass\n", + ch_h, w->va, w->op, w->payload, + cur, same_rel, waited_ms); + } + continue; + } + pend = svc_wait_pend_release(w->va); + if (waited_ms >= gsp_sema_wait_hold_ms) { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; + } + hold = true; + if (svc_wait_log < 512) { + svc_wait_log++; + pr_info("sema-wait: hold ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u pend=%u waited_ms=%u -> entry held (acquire not satisfied)\n", + ch_h, w->va, w->op, w->payload, cur, + same_rel, pend, waited_ms); + } + break; /* one unsatisfied wait holds the entry */ + } + if (hold) { + svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; + svc.op_tr_pending = 1; + return; + } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } + svc_wait_pass++; + if (gh) + gh->active = false; + } +svc_wait_done: for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -6000,8 +7855,21 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -6014,11 +7882,13 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture, - &ce_done); + &ce_done, m->rows, m->dpitch, + m->spitch, ch_h, entry_idx, + pass_hash); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture, - &ce_done); + &ce_done, m->rows, m->dpitch); /* * CEAUDIT (2026-10-18 r220): per-op audit of the CE * data path - the software executor of every guest @@ -6075,9 +7945,11 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + } else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; @@ -6107,7 +7979,9 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || - (m->type == 1 && m->dst >= SVC_VAS_FLOOR))) + (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -6118,7 +7992,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (gsp_sema_dbg && ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || - (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64))) + (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 2048))) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, @@ -6136,7 +8010,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * read-back signature: always keep room for it past the * general trace budget so the DtoH's src/dst/rc is on record. */ - if ((svc.op_trace_log < 160 && + if ((svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && (time_before(jiffies, svc.op_trace_until) || m->dst >= SVC_VAS_FLOOR || (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) || @@ -6457,7 +8331,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, wrc = svc_sem_write(vgpu, h, sem_va, payload); if (!wrc) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u cur=%u -> pool rescue\n", @@ -6531,6 +8405,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, return; } nv = inc ? cur + payload : payload; + /* + * r378: the slot PA was learned from the release VA (or registered + * from the client's pagemap) earlier; re-validate it against the + * current pin before the 4-byte release lands, so a slot the guest + * has since freed/re-homed is never written through the stale page. + */ + if (sem_va && !gsp_sema_write_guard(sem_va, pa, SVC_APERTURE_SYS, + "counter-rel")) + return; if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; @@ -6542,7 +8425,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -6625,7 +8508,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -6964,6 +8847,13 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } + /* + * r378: keep the VA the slot was learned from so a + * late write to release_pa (the stale-recover zero) + * can re-validate the target through + * gsp_sema_write_guard before it lands. + */ + svc.chan[i].release_va = sem_va; break; } } @@ -6996,7 +8886,7 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } @@ -7024,7 +8914,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, svc.pushes_pending++; if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } return false; @@ -7051,7 +8941,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -7069,14 +8969,24 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -7199,6 +9109,84 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) static u32 svc_gr_qmd_seen; static u64 svc_gr_nocap_log; +/* + * r383: has a launch with this push VA ever been captured? Scans the live + * candidate ring, the recent-capture history and the r401 session-wide + * (push_va, hash) seen table. Used by the replayed-QMD dispatch below: a + * replay whose push was never captured is a launch that would otherwise be + * silently dropped (the streams2 second-vadd class); a replay of a + * captured/offered push is the exactly-once no-op. The hash is required + * for the seen table so a later, DIFFERENT push at the same VA (a loop + * iteration: same entry words, new content) is never misclassified as + * seen - the caller only consults this on a true replay anyway. + */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + svc_gr_cand_ring[i].push_va == push_va) + return true; + for (i = 0; i < SVC_GR_CAP_HIST; i++) + if (svc_gr_cap_hist[i] == push_va) + return true; + return svc_gr_seen_lookup(push_va, hash); +} + +/* + * r391f: once-only identity for GRCE inline writes. A push's + * LOAD_INLINE_DATA runs author bytes into the vidmem shadow; a re-decode of + * the same push (replay/rescue/rehold) must not re-run them - the target + * page may have been superseded in between (victory_reduce: the client's + * cudaMemset inline write re-zeroed the accumulator after the launch's + * copy-out, the sum=0 class). Keyed by (push_va, push content hash, dst, + * bytes): the release payload inside every push makes the hash unique per + * push, so a new client's identical-looking write never collides. Session + * scoped (cleared by svc_session_flush). Service-thread only. + * + * Sizing (r398): the ring must outlive the gap between a push's first + * execution and its next re-decode. The observed failing case had 76 + * inline writes between the two, which evicted a 64-slot table and let + * the memset re-run; 8192 slots (256 KB) gives a ~100x margin over the + * busiest observed burst. + */ +#define SVC_INLINE_ONCE_SLOTS 8192 +static struct { + u64 push_va; + u64 hash; + u64 dst; + u32 bytes; +} svc_inline_once[SVC_INLINE_ONCE_SLOTS]; +static u32 svc_inline_once_next; + +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_INLINE_ONCE_SLOTS; i++) + if (svc_inline_once[i].push_va == push_va && + svc_inline_once[i].hash == hash && + svc_inline_once[i].dst == dst && + svc_inline_once[i].bytes == bytes) + return true; + return false; +} + +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i = svc_inline_once_next++ % SVC_INLINE_ONCE_SLOTS; + + svc_inline_once[i].push_va = push_va; + svc_inline_once[i].hash = hash; + svc_inline_once[i].dst = dst; + svc_inline_once[i].bytes = bytes; +} + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -7281,13 +9269,52 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, pr_info("sema-gr: inline recs n=%u dup_dst=%u\n", n, dup); } - if (gsp_phys_gr_forward_enabled() && trusted && !replay) - svc_gr_cand_capture(ch, st, recs, n, push, push_len, - st->push_va); - else if (!replay || !gsp_phys_gr_forward_enabled()) + if (gsp_phys_gr_forward_enabled()) { + u64 push_hash = (push && push_len) ? + svc_push_hash(push, push_len) : 0; + + if (trusted && !replay) { + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (trusted && replay && + !svc_gr_cand_push_seen(st->push_va, + push_hash)) { + /* + * r383 (streams2 second-vadd class): a + * complete QMD first seen on a replay pass + * fell through BOTH arms below - the + * forward path skips replays and the + * bridge arm needs !replay - so the + * launch was silently dropped (zero + * engine writes, client read stale/zero + * outputs). The push VA was never + * captured, so capture it now (trusted + * content only, as the first-sight path); + * a replay of an already captured/offered + * launch stays the exactly-once no-op. + */ + pr_info("sema-gr: REPLAY-CAPTURE h=0x%x push=0x%llx (complete QMD first seen on a replay pass; no capture on record)\n", + ch->h_object, st->push_va); + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (replay && svc_gr_nocap_log < 64) { + svc_gr_nocap_log++; + pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n", + ch->h_object, st->push_va, + svc_gr_cand_push_seen(st->push_va, + push_hash), + trusted); + } else if (!replay) { + gsp_cuda_bridge_launch(vgpu, ch->h_object, + st->qmd, + st->qmd_dev_va, + st->push_va, recs, n); + } + } else if (!replay) { gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); + } st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -7296,6 +9323,89 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } +static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ + +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + */ +unsigned gsp_sema_push_raw; +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_n; + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + if (len < 512 || len > 8192) + return; + if (push_va == svc_pushraw_last_va) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -7303,6 +9413,35 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u64 *ops_done) { struct svc_push_state st; + + /* r391f: the inline-write once-only key needs the push's identity. */ + svc_pass_hash = svc_push_hash(buf, len); + + /* + * r316: pending releases record the channel they were decoded for; + * the verified pool resolver is channel-keyed (see svc_pending_add + * and the matmul r322 starved/hung note). + */ + svc.cur_ch = ch->h_object; + /* + * r34x: the acquire-wait capture is PER RING ENTRY. It used to be + * reset only at SET_OBJECT, so a decode pass over a later entry (or + * a set of entries, since a held entry is re-decoded after the + * sweep walks ahead) leaked its acquire into an earlier entry's + * release-site gate: r340 held a D2H copy entry that carries no + * acquire at all on a later launch's 'wait for the next tracking + * value' (need=85 while the held entry releases 84) - a circular + * wait only our own withheld release could ever satisfy, so every + * hold expired via the 10 s fallback (wait=4542 etc.). Reset here, + * per entry; SET_OBJECT still resets at the push boundary, and the + * re-decode re-captures the same waits, so held entries re-evaluate + * identically. + */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service @@ -7331,6 +9470,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -7356,14 +9502,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_defer_armed = false; svc.pages_decoded++; /* Diagnostic: name every push decoded in the post-completion window. */ - if (svc.op_trace_log < 160 && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && time_before(jiffies, svc.op_trace_until)) { svc.op_trace_log++; pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, trusted); } - if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 96) { + /* + * fail4: de-windowed decode trace - every push the service looks at, + * bounded per session. The windowed/identified-channel diagnostics + * above can miss a whole client's push stream (unidentified channel, + * no bridge completion to arm the window); this record cannot. + */ + /* + * fail4 r302: with sema_dbg on the boot-time driver init spends the + * 512-line budget long before the run under diagnosis, leaving the + * interesting push stream (a later client's copies) invisible. The + * debug flag raises the bound so one boot covers init + run. + */ + if (svc.pushtrace_all < (gsp_sema_dbg ? 32768 : 512)) { + svc.pushtrace_all++; + pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n", + ch->h_object, ch->engine_type, push_va, page_pa, len, + allow_ops, trusted, + len >= 4 ? *(const u32 *)(buf + 0) : 0, + len >= 8 ? *(const u32 *)(buf + 4) : 0); + } + if (gsp_sema_dbg && svc_dbg_streams < 96) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, @@ -7411,28 +9577,195 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * fail4: name every big method-data run and every run whose + * address is the LOAD_INLINE_DATA method (any op/subch - the + * encoding of the 64 KB H2D upload is what we are pinning). + */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.bigrun_log < 128) { + svc.bigrun_log++; + pr_info("sema-bigrun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, st.have_out, st.offset_out, st.line_length, + st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0); + } + if (gsp_sema_dbg && len >= 8192 && + (addr == GRCE_M_LOAD_INLINE_DATA || addr == 0x1b8) && + svc.lid_hdr_log < 64) { + svc.lid_hdr_log++; + pr_info("sema-lid-hdr: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, trusted, allow_ops); + } + + /* * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries * the DMA source bytes in the push itself. Run the write to * the destination the preceding methods selected. */ if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && count && off + 4 + count * 4 <= len) { - if ((trusted || allow_ops) && !gsp_sema_skip_ops && - st.have_out && st.line_length) { - u32 bytes = st.line_length * - (st.gr_line_count ? st.gr_line_count : 1); - u32 have = count * 4; + /* + * fail4 slot rule: in the UVM upload pushes each + * LOAD_INLINE_DATA run fills a whole 32 KB slot - its + * data runs from the header to the next run header + * (or to the next 0x8000 boundary when this is the + * last run), regardless of the count field. A compact + * push (no room for a following slot) keeps the + * count*4 reading (the small parameter banks). The + * old count*4 reading truncated the big runs to half + * their slot and left the tail to be misparsed as + * methods (the split-upload bug). + */ + bool slot_run = (off + 0x8000 < len); + u32 slot_end = off + 0x8000; + u32 slot_bytes = 0; + /* + * fail4: the bridge's argument-block search needs the + * run's START as its destination; the cursor advance + * below must not leak into the rec (an end-of-run + * dst_va makes the QMD param patch miss and the + * engine never consumes the launch). + */ + u64 run_dst = st.offset_out; + + if (slot_run) { + u32 p; + + for (p = off + 4 + count * 4; + p + 4 <= slot_end && p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 3 && + ((h2 >> 13) & 0x7) == 1 && + ((h2 & 0x1fff) << 2) == + GRCE_M_LOAD_INLINE_DATA && + ((h2 >> 16) & 0xfff)) { + slot_end = p; + break; + } + } + slot_bytes = slot_end - (off + 4); + if (slot_bytes > 0x8000 - 4) + slot_bytes = 0x8000 - 4; + } + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the inline-write + * gate `trusted || allow_ops` re-ran the write on + * every re-decode (a replay is trusted by + * construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an inline + * write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H readback, + * zeroing the accumulator the launch had just + * written - the client then read zeros). A coarse + * !replay gate is wrong (a push whose first pass + * was untrusted must still author its bytes on the + * later trusted pass); key the write by + * (push_va, push content hash, dst, bytes) - a new + * push's release payload makes the hash unique - + * and skip only a genuine duplicate. The table is + * session-scoped (a new client starts clean). + */ + { + u32 bytes = slot_run ? slot_bytes : + st.line_length * + (st.gr_line_count ? + st.gr_line_count : 1); + u32 have = slot_run ? slot_bytes : count * 4; if (bytes > have) bytes = have; - svc_grce_inline_write(vgpu, st.offset_out, - buf + off + 4, bytes, - ch->h_object); - } else if (gsp_sema_dbg && svc.grce_log < 16) { - svc.grce_log++; - pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", - trusted, allow_ops, st.have_out, - st.line_length, ch->h_object); + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the + * inline-write gate `trusted || allow_ops` + * re-ran the write on every re-decode (a replay + * is trusted by construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an + * inline write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H + * readback, zeroing the accumulator the launch + * had just written - the client then read + * zeros). A coarse !replay gate is wrong (a push + * whose first pass was untrusted must still + * author its bytes on the later trusted pass); + * key the write by (push_va, push content hash, + * dst, bytes) - the release payload inside every + * push makes the hash unique per push - and skip + * only a genuine duplicate. Session-scoped: a new + * client starts clean. + */ + if ((trusted || allow_ops) && !gsp_sema_skip_ops && + st.have_out && st.line_length && bytes && + !svc_inline_once_seen(push_va, svc_pass_hash, + st.offset_out, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + st.offset_out, + bytes); + svc_grce_inline_write(vgpu, st.offset_out, + buf + off + 4, + bytes, + ch->h_object); + /* + * fail4: name the header that follows + * each big LID run - the split 64 KB + * upload's other payload carriers sit + * exactly there. + */ + if (gsp_sema_dbg && len >= 4096 && + svc.next_hdr_log < 24) { + u32 noff = off + 4 + count * 4; + u32 nh = 0, nw = 0; + + if (noff + 4 <= len) + memcpy(&nh, buf + noff, 4); + if (noff + 8 <= len) + memcpy(&nw, buf + noff + 4, + 4); + svc.next_hdr_log++; + pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n", + ch->h_object, push_va, + off, count, nh, nw, + st.offset_out); + } + } else if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", + trusted, allow_ops, st.have_out, + st.line_length, ch->h_object); + } + /* + * fail4 (r526): hardware semantics - a + * LOAD_INLINE_DATA run advances the destination + * cursor by the bytes it carries whether or not + * the once-only gate let the write run. Without + * this, the consecutive runs of a split upload + * all land at the first destination (the old + * RAWA bug), and worse: on a re-decode pass where + * the gate suppresses the FIRST run, the cursor + * stays at the first run's destination and the + * SECOND run executes there - its data (the + * buffer's bytes 8..16) overwriting offset 0 + * (the observed tgemm XDIFF: X[0..1] <- + * X[2..3], deterministic, every pass). The gate + * suppresses the write, not the method stream's + * positional semantics. + */ + if (bytes) + st.offset_out += bytes; } /* * Record the destination and the payload bytes for @@ -7442,21 +9775,43 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * kernel arguments in the QMD parameter bank, and the * destination VA may not be readable later. */ - if (st.have_out && st.offset_out) { + if (st.have_out && run_dst) { struct svc_grce_rec *rec = &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; - rec->dst_va = st.offset_out; - rec->len = min_t(u32, count * 4, + rec->dst_va = run_dst; + rec->len = min_t(u32, + slot_run ? slot_bytes : count * 4, GSP_BRIDGE_INLINE_BYTES); memcpy(rec->data, buf + off + 4, rec->len); svc_grce_dst_ring_next++; } - off += 4 + count * 4; + /* + * fail4 slot rule: hop to the next slot's run header + * instead of walking the slot's raw tail as methods. + */ + off += slot_run ? (slot_end - off) : 4 + count * 4; continue; } + /* + * fail4: name the moderate op=1 method runs on big pushes - + * the split 64 KB upload's payload carriers beyond the LID + * class land here (their addr selects the write port). + */ + if (gsp_sema_dbg && len >= 4096 && op == 1 && + count >= 16 && count <= 1023 && svc.mystery_log < 48) { + svc.mystery_log++; + pr_info("sema-mystery: h=0x%x push=0x%llx off=0x%x hdr=0x%08x cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x %08x\n", + ch->h_object, push_va, off, hdr, count, subch, + addr, st.have_out, st.offset_out, + st.line_length, st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0, + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); + } + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -7521,6 +9876,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * held INC-shaped words queued a bogus LAUNCH_DMA). * No release is decoded from these encodings. */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.skiprun_log < 64) { + svc.skiprun_log++; + pr_info("sema-skiprun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, + count, subch, addr, trusted, allow_ops); + } off += 4 + count * 4; continue; } @@ -7587,6 +9949,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); + /* r336/r34x: new push, new acquire set + release mirror */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; off += 4 + count * 4; continue; } @@ -7605,6 +9973,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -7612,7 +9984,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -7657,7 +10030,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; - if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm() && + !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { @@ -7702,6 +10076,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, va, payload, op); } } + } else if (op != HOST_SEM_OP_RELEASE && + va >= SEMA_VA_LO && va < SEMA_VA_HI) { + /* + * r336/r34x: C56F SEM_EXECUTE acquire ops + * (0 = ACQUIRE/equal, 2 = strict GEQ, + * 3 = circular GEQ, 4 = AND, 5 = NOR). + * Core CUDA starts its compute-channel + * pushes with one of these to wait for a + * progress counter (observed: + * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on + * the D2H readback push - ACQ_CIRC_GEQ on + * the channel counter 0x20400ffd0). + * svc_run_ops honors every captured acquire + * of this entry before running its work, so + * a copy cannot execute ahead of the + * dependency it waits on (the streams2 + * readback-before-scale class). Capturing + * into the per-entry list (instead of a + * single last-wait global) is what stops a + * later entry's acquire from gating an + * earlier entry (the r340 over-hold). + * PAYLOAD_SIZE 64BIT (execute bit 24) is + * captured on the low word only; every + * observed CUDA acquire is 32-bit. + */ + if (svc_wait_n < SVC_WAIT_MAX) { + struct svc_wait_op *w = + &svc_waits[svc_wait_n++]; + + w->va = va; + w->payload = payload; + w->op = (u8)op; + w->off = off; + } else { + svc_wait_ovf++; + } } off += 4 + count * 4; continue; @@ -7751,6 +10161,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = st.rs_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -7771,7 +10185,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -7886,7 +10301,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * operand state, so one run covers both the * C5B5 and the GRCE copy shapes. */ - if (ch->h_object && gsp_sema_dbg && + if (gsp_sema_dbg && svc.xfer_raw_log < 48) { u32 dl = len < 192 ? len : 192; @@ -7928,11 +10343,61 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: + /* + * r343 FILL instrument: a fill-shaped LAUNCH_DMA + * (REMAP_ENABLE + destination PITCH, OFFSET_OUT and + * no OFFSET_IN) needs this constant to build its op. + * The 8MB-memset class (memcpy_roundtrip all-bad, + * victory_memset2d) shows fill-shaped launches with + * have_remap=0 that decode nothing and are consumed + * silently. Log every decode (bounded) so one run + * captures the real run shape even when the count + * guard below cannot use it. + */ + if (svc_remapconst_log < 256) { + svc_remapconst_log++; + pr_info("sema-remapconst: h=0x%x push=0x%llx off=0x%x count=%u v0=0x%x v1=0x%x v2=0x%x trusted=%d allow=%u\n", + ch->h_object, push_va, off, count, + v0, count >= 2 ? v1 : 0u, + count >= 3 ? v2 : 0u, trusted, allow_ops); + } if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; + st.have_remap_a = 1; + st.have_remap_b = 1; + st.have_remap_c = 1; + } else if (count >= 1) { + /* + * r365: the driver also pushes the fill constants + * as SEPARATE count=1 methods (the CE + * page-table fills carry only SET_REMAP_CONST_A + * with A=0). Capture the individual parts but do + * NOT set have_remap: r365b proved that letting + * these pass the fill guard executes below-floor + * window fills (0x2000c0xxx, 0x204020000) whose + * svc_fill resolution landed in GUEST RAM and + * zeroed slab pages (guest kernel GPF in + * kmem_cache_alloc -> panic at ~75 s). The + * window fills only run behind gsp_sema_winfill + * with the correct resolution proven first. + */ + st.remap_a = v0; + st.have_remap_a = 1; + } + break; + case CE_M_SET_REMAP_CONST_B: + if (count >= 1) { + st.remap_b = v0; + st.have_remap_b = 1; + } + break; + case CE_M_SET_REMAP_COMPONENTS: + if (count >= 1) { + st.remap_comp = v0; + st.have_remap_c = 1; } break; case CE_M_SET_SRC_PHYS_MODE: @@ -7945,12 +10410,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; + break; + /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ + case CE_M_LINE_COUNT: + st.line_count = v0; + break; + case CE_M_PITCH_IN: + st.pitch_in = v0 & 0xffffff; + st.have_pitch_in = 1; + break; + case CE_M_PITCH_OUT: + st.pitch_out = v0 & 0xffffff; + st.have_pitch_out = 1; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); + u32 n_ops_before = n_ops; - if (gsp_sema_dbg && ch->h_object && + if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", @@ -7967,7 +10462,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* Diagnostic: full operand state in the post-completion * window (or for high device VAs), bounded. */ - if (svc.op_trace_log < 160 && ch->h_object && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && ch->h_object && (time_before(jiffies, svc.op_trace_until) || st.offset_out >= SVC_VAS_FLOOR || st.offset_in >= SVC_VAS_FLOOR)) { @@ -7991,7 +10486,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * bounded number of such launches so the operand * encoding (or its absence) is on record. */ - if (transfer != 0 && ch->h_object && gsp_sema_dbg && + if (transfer != 0 && gsp_sema_dbg && svc.xfer_raw_log < 48 && (!st.have_out || !st.line_length)) { u32 dl = len < 192 ? len : 192; @@ -8006,25 +10501,182 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (allow_ops && transfer != 0 && st.line_length && st.have_out) { - if (DMA_REMAP_ENABLE(dma) && st.have_remap && + /* + * r361: a fill-shaped transfer whose + * destination is PHYSICAL guest RAM (the + * driver zeroes freshly-allocated PTE / + * page-directory pages with exactly this + * push shape) needs no translation: the + * destination already IS the GPA. The old + * `offset_out >= SVC_VAS_FLOOR` guard + * compared the GPA against the VA floor + * and blocked every one of these, so the + * PD0 pages the GMMU walk later reads + * stayed stale and the first CUDA client + * of a fresh boot dropped operand pages + * (GRFILL-DROP). SEMA_PA_MAX bounds the + * range so a ramfill can only touch guest + * RAM; virtual engine-window fills + * (0x2040xxxxx) stay excluded. + */ + bool ramfill = DMA_DST_TYPE_PHYSICAL(dma) && + st.have_dst_mode && + st.dst_phys_mode == CE_APERTURE_SYS && + st.offset_out + (u64)st.line_length <= + SEMA_PA_MAX; + /* r365: vidmem-window fill (see gsp_sema_winfill). */ + bool winfill = gsp_sema_winfill && + st.offset_out >= SVC_VIDMEM_VA_LO && + st.offset_out < SVC_VIDMEM_VA_BASE + + svc_fb_shadow_size(vgpu); + + if (DMA_REMAP_ENABLE(dma) && + (st.have_remap || + (st.offset_out >= SVC_VAS_FLOOR) || + ramfill || winfill) && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { - u32 csz = (st.remap_comp >> 16) & 0x3; - u32 ndst = (st.remap_comp >> 24) & 0x3; + u32 csz = st.have_remap ? + ((st.remap_comp >> 16) & 0x3) : 0; + u32 ndst = st.have_remap ? + ((st.remap_comp >> 24) & 0x3) : 0; + u32 dst_x = st.have_remap ? + (st.remap_comp & 0x7) : 4; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; - m->value = ndst >= 1 ? - (((u64)st.remap_b << 32) | - st.remap_a) : st.remap_b; - m->elem = (csz + 1) * (ndst + 1); - m->count = st.line_length; + /* + * r367: the driver's own memset push + * (channel_utils.c) proves the + * encoding: LINE_LENGTH_IN is the + * fill SIZE IN BYTES, and + * SET_REMAP_COMPONENTS selects + * COMPONENT_SIZE ([17:16], 0..3 -> + * 1/2/3/4 bytes) and + * NUM_DST_COMPONENTS ([25:24], n+1 + * components). elem = + * (csz+1)*(ndst+1) is the pattern + * width; count must therefore be + * line_length/elem - the old + * count=line_length wrote + * elem x too much (r365: a 383620- + * byte memset wrote 1534480 bytes, + * clobbering the bytes after it). + * The value comes from the mapped + * constants: X<-CONST_A (4) and + * Y<-CONST_B (5); with two + * components the pattern is + * B:A 64-bit. + */ + if (st.have_remap) + m->value = ndst >= 1 ? + (((u64)st.remap_b << 32) | + st.remap_a) : + (dst_x == 5 ? st.remap_b : + st.remap_a); + else + /* + * r357: a fill-shaped transfer + * (REMAP_ENABLE + DST PITCH, + * no source) to a CLIENT DEVICE + * VA whose value constant did + * not decode is a ZERO fill: + * the driver's page-table + * clears and cudaMemset(0) + * issue exactly this shape with + * no SET_REMAP_CONST method. + * Building no op left the + * guest's GPU page tables + * stale (the GMMU walk then + * missed freshly-mapped + * operand pages and the first + * CUDA client of every fresh + * boot ran with dropped + * operands). The >= floor + * guard keeps engine-window + * fills (0x2040xxxxx init + * clears) OUT of the executor: + * executing those clobbered + * emulated engine state and + * hung client init (r356). + */ + m->value = 0; + m->elem = st.have_remap ? + (csz + 1) * (ndst + 1) : 1; + if (m->elem > 1 && + st.line_length % m->elem) { + /* not pattern-divisible: exact + * byte count wins */ + m->elem = 1; + m->count = st.line_length; + } else { + m->count = st.line_length / + m->elem; + } + /* + * r402: a 2D fill (cudaMemset2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes at + * PITCH_OUT stride. A pitch WIDER + * than the row selects row mode; a + * flat fill carries pitch == row + * length (or no pitch method) and + * keeps the historical single-run + * path. + * + * r420c: LINE_COUNT is + * authoritative for the flat case + * too - cudaMemset2D with pitch == + * width (victory_memset2d, + * pitch=4096/w=4096) is CONTIGUOUS + * rows, so the single run must + * cover rows*line bytes. The + * historical path filled one row + * and the readback lost the rest + * (bad ~= 12.5M of 12.58M). + */ + if (st.line_count > 1) { + u32 fpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + + if (fpitch > st.line_length) { + m->rows = + st.line_count; + m->dpitch = fpitch; + } else if (st.line_length % + m->elem) { + m->elem = 1; + m->count = + st.line_length * + st.line_count; + } else { + m->count = + (st.line_length / + m->elem) * + st.line_count; + } + } + if (m->rows > 1 && svc_pitch_log < 32) { + svc_pitch_log++; + pr_info("sema-2d: fill rows=%u dpitch=%u line=%u dst=0x%llx h=0x%x push=0x%llx\n", + m->rows, m->dpitch, + st.line_length, m->dst, + ch->h_object, push_va); + } m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; + if (ramfill && svc.fill_ram_log < 24) { + svc.fill_ram_log++; + pr_info("vgsp-sema: RAMFILL-EXEC push=0x%llx dst=0x%llx line=%u ap=%u value=0x%llx (physical guest-RAM fill built)\n", + push_va, st.offset_out, + st.line_length, + m->aperture, m->value); + } } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { @@ -8041,6 +10693,80 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; + /* + * r402: a 2D copy (cudaMemcpy2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes with + * PITCH_IN/PITCH_OUT strides. + * Row mode when a pitch is + * wider than the row. + * + * r420c: pitch == row length + * (or no pitch method) makes the + * rows CONTIGUOUS, so the single + * flat run must cover + * rows*line bytes - the + * historical path copied one row + * (victory_memset2d's pitched + * D2H/H2D at pitch 4096 = width + * 4096 lost all but the first + * row). + */ + if (st.line_count > 1) { + u32 dpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + u32 spitch = st.have_pitch_in ? + (st.pitch_in ? + st.pitch_in : + dpitch) : + dpitch; + + if (dpitch == st.line_length && + spitch == st.line_length) { + m->bytes = + st.line_length * + st.line_count; + } else { + m->rows = + st.line_count; + m->dpitch = dpitch; + m->spitch = spitch; + } + } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -8069,6 +10795,317 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * r365 FILL-RAW: the fill class that still decodes no + * op (window VA, no remap const decoded) - dump the + * raw push and decode its method stream so the true + * value/component encoding is on record. Bounded. + */ + if (allow_ops && transfer != 0 && + DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + st.have_out && !st.have_in && !st.have_remap && + svc_fill_raw_log < 48) { + u32 dl = len < 160 ? len : 160; + u32 mo = 0, mn = 0; + + svc_fill_raw_log++; + pr_info("sema-fill-raw: h=0x%x push=0x%llx out=0x%llx line=%u dma=0x%08x dmode=%u have_dstmode=%u trusted=%d allow=%d\n", + ch->h_object, push_va, st.offset_out, + st.line_length, dma, st.dst_phys_mode, + st.have_dst_mode, trusted, allow_ops); + pr_info("sema-fill-raw: raw len=%u %*phN\n", + len, dl, buf); + /* decode the first method pairs */ + while (mo + 8 <= len && mn < 14) { + u32 h2, c2, a2; + + memcpy(&h2, buf + mo, 4); + c2 = (h2 >> 16) & 0xfff; + a2 = (h2 & 0x1fff) << 2; + if ((h2 >> 29) != 1 || !c2 || + mo + 4 + c2 * 4 > len) + break; + if (c2 >= 2) + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x v1=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4), + *(const u32 *)(buf + mo + 8)); + else + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4)); + mo += 4 + c2 * 4; + mn++; + } + } + + /* + * r343 FILL diagnostic (LOG-ONLY, no hold): a + * fill-shaped transfer (REMAP_ENABLE + destination + * PITCH, OFFSET_OUT and no OFFSET_IN) whose value + * constant did not decode currently decodes no op + * and is consumed silently - the guest's cudaMemset + * then reports success with zero bytes written + * (r341 memcpy_roundtrip: 8MB memset readback + * all-bad; victory_memset2d). An op_tr_pending hold + * here WEDGED client init (r343: first cudaMalloc + * retried forever - driver init issues fill-shaped + * CE pushes, the hold re-armed every decode pass + * and never expired). So: log loudly only; the + * sema-remapconst rows above capture the real + * decode shape to build the proper fill decoder. + */ + if (allow_ops && transfer != 0 && + st.line_length >= 1024 && st.have_out && + !st.have_in && DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + !st.have_remap && n_ops == n_ops_before && + svc_fill_noremap < 8) { + svc_fill_noremap++; + pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx phys=%u dstmode=%u: fill-shaped launch, value source not decoded (log-only diagnostic)\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_out, + DMA_DST_TYPE_PHYSICAL(dma), + st.have_dst_mode ? st.dst_phys_mode : 99); + } + + /* + * fail4: name a transfer-shaped launch that decoded NO + * op because the pass was denied (allow_ops=0) - the + * op-chain above (and its dma-skip dump) is skipped + * wholesale in that case, so this class was invisible. + */ + if (gsp_sema_dbg && transfer != 0 && !allow_ops && + svc.dmatrace_log < (gsp_sema_dbg ? 4096 : 64)) { + svc.dmatrace_log++; + pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d replay=%d eng=%u pend_would=%d\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_in, + st.offset_out, st.have_in, st.have_out, + st.have_remap, trusted, replay, + ch->engine_type, + (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && + st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR && + entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS && + ch->slot_pend_tries[entry_idx] < 4) ? 1 : 0); + } + /* + * fail4 r302: a denied device-destination copy that + * WOULD have built a type=1 op (the exact branch shape + * above: no remap, both offsets, device dst) must not + * be consumed silently. This is the sw-copy client + * mode: the GR-classified GRCE channel's ops run only + * once the pin/trust stream (injected) reaches the + * entry, and the first pass over a chunk push lands + * before that - the copy then never ran and the guest + * computed on stale VRAM (vector_add r302: chunk + * copies cc..d1 skipped; only the tail entries were + * re-decoded by a rewind and ran). Mark the pass + * op-pending so the pump holds the entry through the + * m163/opdef machinery and the retry runs the copy as + * soon as the trust (or the operand translation) + * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps + * the pre-existing skip as the last resort. + */ + if (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR) { + /* + * r314 (CONT.57 lead 2): capture the staging + * source on the DENIED pass too. This is the + * pipeline-order moment: the push was just + * decoded, the driver has not seen this + * push's fence (it applies later in this same + * pass at the earliest), so the staging still + * holds this chunk's bytes. The old capture + * gate lived in the op-build block, which + * only runs on ALLOWED passes - by then the + * fence had long fired and the staging was + * refilled (r302-r313: captures 0-3/8, the + * late executions read refilled staging, the + * deterministic 786335-bad output). Same + * translation-probe guard as the build-time + * gate: only an SYS-resolving source is + * host staging. + */ + if (st.line_length <= SVC_COPY_SNAP_MAX) { + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, + st.offset_in, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + st.offset_out, + st.offset_in, + st.line_length, + ch->h_object, + entry_idx, + svc_pass_hash); + } + /* + * r308: the `!replay` gate above kept this + * dead on the sw-copy channel: after the first + * consumed pass every later pass classifies as + * a replay, so the denied chunk copies were + * never retried (vector_add: 6 of 8 chunks + * skipped). A replay pass that marks the slot + * pending un-completes it (slot_ops_complete + * is written from svc.op_tr_pending below), so + * the next pass is treated fresh and the copy + * runs once trust/translation arrives. The + * per-slot counter caps the re-holds so a slot + * whose trust never arrives cannot wedge the + * channel in 10 s hold cycles. + */ + if (entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS && + ch->slot_pend_tries[entry_idx] < 4) { + ch->slot_pend_tries[entry_idx]++; + svc.op_tr_pending = 1; + if (gsp_sema_dbg && + svc.pend_dbg_log < 512) { + svc.pend_dbg_log++; + pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + replay, trusted, + ch->slot_pend_tries[entry_idx]); + } + } + } + /* + * fail4: name every op appended to this pass (copy or + * fill) with its resolved operand fields, so the + * executor's population is on record even when the + * 64 KB copy witness below never fires for it. + */ + if (n_ops != n_ops_before) { + struct svc_memset_op *lm = &ops[n_ops - 1]; + + /* + * fail4 CONT.53: H2D-shaped copies (virtual + * device destination, virtual host source) + * read the driver's ping-pong staging. The + * driver paces restaging on the copy's + * completion fence; our execution may run + * later than the push's processing position, + * so capture the source NOW (bounded pool) + * and let svc_copy execute from the capture - + * the hardware's in-order staging read. + */ + /* + * H2D shape ONLY: device destination (VID + * aperture), host source (SYS). A D2H readback + * (host dst) captured here would freeze + * pre-kernel zeros (observed in the r300 + * captures) and break readbacks. + */ + if (lm->type == 1 && !lm->phys && + lm->dst >= SVC_VAS_FLOOR && + lm->aperture == CE_APERTURE_VID && + !lm->src_phys && + lm->src >= SVC_VAS_FLOOR && + lm->bytes <= SVC_COPY_SNAP_MAX) { + /* + * r311: the src_aperture equality + * gate never matched the sw-copy + * client (the driver sends no + * SET_SRC_PHYS_MODE, so src_ap + * defaults to VID=0 while the + * source is host staging; r302-r310 + * logged zero captures and the + * late executions read refilled + * staging). The translation is the + * truth: capture only when the + * first source page resolves + * through the SYS aperture. + */ + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, lm->src, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + lm->dst, + lm->src, + lm->bytes, + ch->h_object, + entry_idx, + svc_pass_hash); + if (gsp_sema_dbg && + svc.snap_gate_log < 256) { + svc.snap_gate_log++; + pr_info("sema-snapgate: dst=0x%llx src=0x%llx trc=%d sap=%u\n", + lm->dst, lm->src, + svc_op_translate(vgpu, lm->src, &spa, &sap), + sap); + } + } + + /* + * fail4 CE linear auto-advance, BOTH sides: the + * driver re-sets only ONE offset per op and the + * copy engine advances the other by the transfer + * size. H2D ops show an explicit advancing DST + * with a FIXED SRC base (every 1 MB chunk read + * the source base again: vector_add's device a + * held h_a[0..1MB) x4 -> 75% mismatch; the + * readbacks are the mirror image). Advance both + * registers; an explicit re-set simply overrides. + */ + { + u64 adv = (lm->type == 1) ? lm->bytes : + (u64)lm->elem * lm->count; + + st.offset_out += adv; + if (lm->type == 1) + st.offset_in += adv; + } + if (svc.opadd_log < 2048) { + svc.opadd_log++; + pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n", + ch->h_object, lm->type, + lm->dst, lm->src, lm->bytes, + lm->elem, lm->count, + lm->aperture, lm->src_aperture, + lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in); + } + } + + /* * Both release encodings write the payload word first * (NvGpuSemaphore.data[0] for the four-word format, the * single word for the one-word format), and the client's @@ -8094,6 +11131,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; + svc_wait_rel_va = sem_va; + svc_wait_rel_payload = st.sem_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; if (st.have_sem && st.sem_payload && @@ -8191,6 +11232,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->entry_want = st.sem_payload; have_cur = true; pending = 1; + /* + * fail4: apply this seed ABSOLUTE. + * The generic applier still sees the + * increment flag (inc), so leaving + * the residue in cur makes it write + * cur+payload (observed 134+1=135) + * - a completed value far past the + * guest's queued count, which + * trips the guest UVM assert + * (uvm_channel.c:204 completed <= + * queued) and kills the device. + * cur=0 makes inc-apply land the + * payload itself, exactly the + * re-seed the client polls. + */ + cur = 0; svc.pool_residue++; if (svc.pool_residue_log < 32) { svc.pool_residue_log++; @@ -8307,7 +11364,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -8403,7 +11461,44 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); - if (svc_rel_apply_launch(vgpu, ch, + /* + * fail4 CONT.53: never clear a + * push's fence before its ops + * read their source. The pass's + * ops run at the pass end; the + * driver's ping-pong staging + * restages as soon as the fence + * clears, and a later op read + * then returns chunk+N bytes + * (vector_add +2 MB). Queue the + * release through the pending + * pass instead. + */ + if (n_ops > 0 || + svc.op_tr_pending) { + /* + * r316 (CONT.60): + * op_tr_pending covers + * the denied pass - the + * pass built no op + * (allow=0) but a copy is + * still owed; applying + * the fence here (the old + * n_ops==0 path) let the + * driver refill the + * ping-pong staging before + * the copy ever ran - the + * cold-run capture gap + * (vector_add run 1 + * bad=157696/193536 vs + * bad=0 warm). + */ + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, @@ -8437,7 +11532,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_rel_direct_note(ch->h_object, sem_va, st.sem_payload); - if (svc_rel_apply_launch(vgpu, ch, sem_va, + /* + * fail4 CONT.53: same ops-order + * guard as above - a pass with + * pending copy ops must run + * them before its fence clears. + */ + if (n_ops > 0) { + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, + sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { @@ -8457,7 +11565,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } - if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) { + if (gsp_sema_dbg && ch->h_object && + svc.entry_log < 16384) { svc.entry_log++; { u8 tail[24]; @@ -8523,6 +11632,12 @@ struct svc_pending_release { u64 va; u32 payload; u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ + u32 h; /* r316: channel the release was decoded for; the + * verified pool resolver is channel-keyed and h=0 + * never matches its per-channel entries (matmul + * r322: pool-rel cand resolved at decode (rrc=0, + * h=channel) while svc_apply_pending with h=0 + * starved forever and hung the client). */ /* * Resolution attempts that found no translation for this VA. * Telemetry for ISSUE-0001: a pending release that keeps failing to @@ -8599,6 +11714,7 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) svc_pending[i].payload = payload; svc_pending[i].op = op; svc_pending[i].tries = 0; + svc_pending[i].h = svc.cur_ch; } /* @@ -8756,6 +11872,26 @@ static void svc_session_flush(void) svc.counter_noslot_dbg = 0; svc.sem256_log = 0; svc.xfer_raw_log = 0; + svc.pushtrace_all = 0; + svc.opadd_log = 0; + svc.dmatrace_log = 0; + svc.copyent_log = 0; + svc_wait_log = 0; + svc_wait_pass_log = 0; + svc_remapconst_log = 0; + svc_pitch_log = 0; + svc_fill_raw_log = 0; + svc_lowfill_log = 0; + svc_tabcopy_log = 0; + svc_fill_noremap = 0; + svc_wait_n = 0; + svc_wait_rel_have = false; + memset(svc_wait_holds, 0, sizeof(svc_wait_holds)); + svc.bigrun_log = 0; + svc.lid_hdr_log = 0; + svc.skiprun_log = 0; + svc.mystery_log = 0; + svc.next_hdr_log = 0; /* * The push decode budget (`sema-push` lines) is per-session too: the * first client of a boot otherwise spends it and the next session's @@ -8792,6 +11928,8 @@ static void svc_session_flush(void) memset(svc_retry_q, 0, sizeof(svc_retry_q)); memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); memset(svc_fwd_rel, 0, sizeof(svc_fwd_rel)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; memset(svc_grce_dst_ring, 0, sizeof(svc_grce_dst_ring)); for (i = 0; i < SVC_VAS_SLOTS; i++) { if (svc_vas[i].data) @@ -8833,6 +11971,15 @@ static void svc_session_flush(void) svc_hold_fb_log = 0; svc_cand_ovw_log = 0; svc_inline_recs_log = 0; + /* + * r401: the exactly-once capture memory is session-scoped. A new + * client's pushes restart their release counters, so their content + * (and hence hash) can collide with the previous client's - keeping + * the table would wrongly classify a fresh launch of the new + * session as already captured (LAUNCH-SKIP, silent drop). + */ + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + svc_gr_seen_stale = 0; /* m176: new session generation - channels and scrubbers re-stamp on * registration and never inherit per-session learned state. */ svc_sess_gen++; @@ -8887,6 +12034,25 @@ void gsp_sema_service_guest_reset(void) } /* + * r317: queue a pass's recorded forward releases through the pending + * machinery instead of applying them at a hold break / fallback while an + * op of that pass is still owed. Applying the fence early let the driver + * refill the ping-pong staging before the copy read it (the cold-run + * capture gap: vector_add run 1 of every cold boot, last ~2 chunks). The + * pending store applies each release once the word resolves (the copy has + * run by then, since the op execution is what clears the pending state on + * the retry). + */ +static void svc_fwd_rel_queue_pending(struct svc_fwd_rel *rel, u8 n) +{ + u8 i; + + for (i = 0; i < n; i++) + svc_pending_add(rel[i].va, rel[i].payload, + rel[i].inc_claimed ? 1u : 0u); +} + +/* * Loud, bounded telemetry for a starved deferred release (ISSUE-0001). * Called from the two "no translation yet, keep" paths of * svc_apply_pending() with an incremented try count; logs at 4/64/1024 @@ -8917,8 +12083,11 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) * resolver: the learned map can answer with a stale * page whose value makes this release look already * delivered and drops it (see svc_sem_write). + * r316: resolve with the channel the release was + * decoded for (h=0 never matches the per-channel + * pool entries - matmul r322 starved/hung). */ - if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap, + if (svc_pool_pa_resolve(vgpu, p->h, p->va, &ppa, &pap, NULL)) { /* not translatable yet: keep (ISSUE-0001) */ p->tries++; @@ -8972,7 +12141,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); @@ -9145,7 +12314,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); applied++; if (payload != cur + 1) { svc.releases_jumped++; @@ -9176,7 +12345,6 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ -#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* @@ -10146,6 +13314,119 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + u64 dist; + bool pending; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + } + } return -ENOENT; } @@ -10243,7 +13524,7 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, /* Keep the completion visible to clients that start waiting * after the event fired. */ - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } @@ -10906,6 +14187,14 @@ unsigned gsp_phys_gr_hold_max_ms = 3000; unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ /* + * r346: trace GMMU walks for unresolved high-VA operands (the arg-miss + * diagnostic re-walks each published vaspace for a failing base). When + * nonzero, those walks print the per-level entries (L0..L2, PD0 halves) + * so a failing VA shows exactly where the walk dies. + */ +unsigned gsp_sema_walk_trace; + +/* * F1-b: GR release-word authorship gate (default 1). While a forwarded GR * launch is in flight, the RAM-content sweep must not author the very QMD * completion release the backend poll is waiting for: a sweep write could @@ -10944,6 +14233,30 @@ bool gsp_sema_pool_hold_fresh = 1; * 0 restores the historical immediate-write behavior. */ bool gsp_sema_copy_dst_pin_guard = 1; +/* + * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured + * for a ring entry before running its work. DEFAULT ON since r408: the + * r408 streams2d forensics reproduced the class the gate was built for + * with the gate off - the c2 readback (C56F ACQ_CIRC_GEQ pay=4 on the + * scale channel's progress counter 0x20400ffd0) ran at 106617.279 with + * cur=2, i.e. BEFORE the c2-add launch was even submitted (106617.411), + * read the not-yet-produced buffer (zeros) and the client failed with + * 'c2 all zeros'. With the gate on the same readback holds (sema-wait: + * hold ... cur=2 waited_ms=353) until the counter reaches 4 (pass ... + * cur=4 at 106812.259) and the ordering is correct. The semantics are + * the class contract's: op 0 = equal, 2 = strict GEQ, 3 = circular GEQ, + * 4 = AND, 5 = NOR; the wait set is captured PER ENTRY (the old + * single-last-wait global leaked a later entry's acquire into an earlier + * entry - the r340 over-hold); the entry's own release counts only when + * its method order precedes the acquire; pending releases our machinery + * is merely late about are drained first; and every hold is bounded at + * gsp_sema_wait_hold_ms (default 4000 ms, r401) then fails open loudly + * ('give-up') so a value that can only be installed behind an upstream + * opdef hold cannot park a channel. Diagnostics: 'sema-wait: + * pass/hold/give-up' rows carry op, payload, observed value and decision. + * sema_wait_gate=0 restores the pre-r336 pass-through for an A/B. + */ +bool gsp_sema_wait_gate = 1; /* * m169b: the guard above must never deadlock the boot's first client. The @@ -11073,6 +14386,24 @@ bool gsp_sema_rel_recover; bool gsp_sema_dbg; /* + * r365: execute fill-shaped CE transfers whose destination is a VIDMEM + * WINDOW VA (>= SVC_VIDMEM_VA_LO, below the 1 TiB VA floor). The guest's + * page-table writes arrive exactly this way (dma=0x586, REMAP+PITCH, a + * count=1 SET_REMAP_CONST_A, line=4, out=0x2000c0xxx) and previously + * decoded no op, so the shadow never saw them and the first client of a + * fresh boot dropped operand pages (GRFILL-DROP -> launch held/fail-open + * with 8 dropped pages -> wrong results or a client wedge). + * + * The op is built with the VA unchanged (phys=0): svc_fill's virtual path + * resolves it through the guest's own published page tables + * (svc_op_translate) and only falls back to the shadow for floor-range + * VAs - never through the window convention (r362 proved a forced + * `addr - SVC_VIDMEM_VA_BASE` writes the wrong page and corrupts results: + * reduce sum=0 + mail hang). Default off for A/B. + */ +bool gsp_sema_winfill; + +/* * Skip executing a push's collected work (CE memsets/copies into the vidmem * shadow) while still applying its semaphore release. The client only ever * observes the release; the shadow writes exist so the emulation can walk @@ -11131,6 +14462,139 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + int i; + unsigned long flags; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -11407,6 +14871,124 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -11414,8 +14996,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -12326,6 +15907,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -12349,9 +15949,20 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n", ch->h_object, pa, val, cur, ch->counter_last); - if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &val, sizeof(val))) - return; + { + int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, + &val, sizeof(val)); + if (wrc) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: pub-adv-wrc h=0x%x pa=0x%llx val=%u wrc=%d (write did not land)\n", + ch->h_object, pa, val, wrc); + } + return; + } + } ch->counter_last = val; + ch->counter_gen = svc_sess_gen; if (gsp_sema_dbg && svc.counter_pub_dbg < 48) { svc.counter_pub_dbg++; pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n", @@ -12449,7 +16060,7 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } } @@ -12503,6 +16114,20 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { + static u32 consume_log; + + /* + * fail4 r303: name every consumed slot (all channels). Paired with + * the sema-entry decode rows this shows exactly which entries were + * consumed without ops and when (the vector_add chunk-drop chase: + * a consumed slot whose decode row carried a device-dst copy but + * whose op never ran is the silent skip under diagnosis). + */ + if (gsp_sema_dbg && consume_log < 16384) { + consume_log++; + pr_info("sema-consume: h=0x%x idx=%u put=%u\n", + ch->h_object, idx, put); + } (void)put; /* * STALL FIX (entry=82/pay=84 class, r186): the entry leaving the @@ -12782,6 +16407,24 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, return -EFAULT; *hpa = (u64)page_to_phys(pg); *out_off = page_off; + /* + * r372 two-path identity check: the copy-out writes through + * __va(hpa) while every reader reads base+pa - if those are + * not the same memory the engine's output silently vanishes + * (the sum=0 class). Prove it empirically, bounded. + */ + if (svc_fbhpa_log < 16) { + u64 at_base = 0, at_va = 0; + + svc_fbhpa_log++; + memcpy(&at_base, base + page, sizeof(at_base)); + memcpy(&at_va, (u8 *)__va(*hpa), sizeof(at_va)); + pr_info("sema-fbhpa: pa=0x%llx base=%px pg=%px hpa=0x%llx shadow=0x%llx at_base=%016llx at___va=%016llx %s\n", + page, base, pg, *hpa, + (unsigned long long)svc_fb_shadow_size(vgpu), + at_base, at_va, + at_base == at_va ? "SAME" : "DIFFER"); + } return 0; } if (kind != SVC_APERTURE_SYS) @@ -12901,6 +16544,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The + * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold, + * two holds per launch), so the pause can only last a few minutes; the cap is + * a defence-in-depth bound so a stuck quiet-hold flag can never defer the + * client's completion indefinitely. + */ +#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS 600000u +/* + * r420: same defence for the per-launch consumption bound pause in + * svc_gr_hold_bound_expired(). + */ +#define SVC_GR_QUIET_PAUSE_MAX_MS 600000u struct svc_fwd_hold { bool active; @@ -12936,9 +16592,9 @@ struct svc_fwd_hold { /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible - * semaphore page. On the success edge the hold re-reads the guest - * word and, when it carries the payload, skips that one record's - * software apply. Cleared with the hold. + * semaphore page. On the success edge the hold re-reads the guest word + * and, when it carries the payload, skips that one record's software + * apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; @@ -12951,6 +16607,18 @@ struct svc_fwd_hold { struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; + /* + * r420: WEDGE-QUIET deferral state. While the backend deliberately + * holds the in-flight launch in its bounded quiet window + * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward + * (paused_until) so the hold cannot author the launch's release word + * under the hold - the client would then verify data the post-hold + * copy-out has not written yet (the r419 mid-boot escape). The + * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from + * `since` and is announced once per hold. + */ + unsigned long paused_until; + bool pause_logged; }; static struct svc_fwd_hold svc_fwd_hold; @@ -13176,6 +16844,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } @@ -13207,7 +16898,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, if (e->n) { svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc.opdef_applied++; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: apply h=0x%x entry=%u n=%u (ops complete)\n", ch->h_object, entry_idx, e->n); @@ -13217,6 +16908,37 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, } /* + * r34x: is a release to this VA currently withheld by our own machinery? + * True when the pending-translation ring holds one (translation late) or + * an opdef slot has one recorded (operand hold). The acquire gate calls + * this to classify an unsatisfied wait in its diagnostics: pend=1 names + * the 'value sits behind our own withhold' case - the bounded 100 ms + * give-up is exactly for it (waiting longer cannot install the value + * until the upstream entry's ops complete). Never forces an opdef release + * here: those releases are withheld because their ops have NOT completed, + * and delivering early re-opens the work-before-fence hole they close. + */ +static bool svc_wait_pend_release(u64 va) +{ + u32 i; + + for (i = 0; i < svc_pending_n; i++) + if (svc_pending[i].va == va) + return true; + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active) + continue; + for (k = 0; k < e->n; k++) + if (e->rel[k].va == va) + return true; + } + return false; +} + +/* * STALL FIX (r186 entry=82/pay=84): called from svc_replay_note() on every * entry-consume path. If a release set was withheld for this entry while an * operand was pending and the entry is now leaving the decode path (so the @@ -13284,7 +17006,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (!svc_opdef_contains(e, r)) { if (r->payload && e->n < SVC_FWD_REL_MAX) { e->rel[e->n++] = *r; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: hold h=0x%x entry=%u kind=%u va=0x%llx pay=%u (operand pending)\n", ch->h_object, entry_idx, @@ -13326,7 +17048,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, * flushes the slot and consumes the entry. */ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, - u32 entry_idx) + u32 entry_idx, u64 launch_va) { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); @@ -13335,11 +17057,46 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return false; } + /* + * r361: absolute per-launch bound keyed to the LAUNCH PUSH VA + * (the GRFIX lesson, r213). The slot's `since` restarts whenever + * a pin-batch rewind replays the entry at a different GPFIFO + * index, so a slot-keyed-only bound can be re-armed forever - + * observed as opdef=3301/32/0 (held climbing, expired 0) with the + * guest wedged. The push VA never changes across replays, so the + * first hold of a launch arms an ABSOLUTE deadline nothing moves. + */ + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + /* + * r368: an ACTIVE bound is never re-armed. The pump walks up to + * SVC_OPDEF_SLOTS held entries, each with its OWN push VA; the r361 + * arm-on-VA-change rule restarted the deadline on every entry + * change, so with several held launches the deadline moved forever + * (streams2 wedge: opdef=955/13/0, held climbing, expired 0). The + * first hold arms the absolute deadline; entries held while it is + * active ride it and flush together when it fires. A fresh bound is + * armed once the previous one has passed. + */ + if (!ch->opdef_hold_dead_va || + time_after_eq(jiffies, ch->opdef_hold_deadline)) { + ch->opdef_hold_dead_va = launch_va; + ch->opdef_hold_deadline = jiffies + + msecs_to_jiffies(SVC_OPDEF_HOLD_MS); + ch->opdef_hold_exp_logged = false; + } if (time_before(jiffies, - e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) { + e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS)) && + time_before(jiffies, ch->opdef_hold_deadline)) { svc.opdef_held++; return true; } + if (!ch->opdef_hold_exp_logged) { + ch->opdef_hold_exp_logged = true; + pr_err_ratelimited("vgsp-sema: OPDEF-EXPIRED h=0x%x idx=%u va=0x%llx (absolute per-launch bound %u ms; rewind replays no longer re-arm)\n", + ch->h_object, entry_idx, launch_va, + SVC_OPDEF_HOLD_MS); + } return false; } @@ -13416,9 +17173,22 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * The ring advanced past a slot's entry (a slot for another entry of the * same channel is being processed): its set belongs to a consumed entry. * Deliver late (forward-only, so it cannot double-apply) and drop it. + * + * r406: an entry is only "passed" if the ring actually CONSUMED it + * (slot_done_epoch). Without that proof the pin-batch rewind breaks the + * invariant: it re-opens lower indices (e.g. 83) while a held entry + * (e.g. 91) sits AHEAD in the same epoch, and the old unconditional drop + * treated the held entry as consumed and delivered its withheld releases + * early. Observed (r406 inplace_scale): the H2D copy's operand was + * unresolved (rc=-2), the entry held with its release withheld; a rewind + * pass at idx=83 fired drop-stale caller=4 for idx=91 and the release + * went out at 104955.751 - while the copy only executed at 104957.418. + * The client raced past its blocking memcpy, launched, the launch read + * the stale buffer (previous run's data), and the copy landed after the + * launch's copy-out: wrong results with 32/1032 "changed". */ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, - struct svc_channel *ch, u32 keep_idx) + struct svc_channel *ch, u32 keep_idx, u32 put) { int i; @@ -13428,21 +17198,8 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, if (!e->active || e->vgpu != vgpu || e->h_object != ch->h_object || e->entry_idx == keep_idx) continue; - /* - * STALLFREE r228 (entry=82 of 0x5c000059): a slot whose entry - * the ring has NOT consumed in this put epoch is NOT stale. A - * pin-batch rewind moves last_get backwards over an unconsumed - * entry and re-decodes it; clearing the slot here restarted the - * 10 s opdef deadline on every re-walk, so the hold re-armed - * forever (log: 'opdef-clear caller=4 entry=82' + 'opdef: hold - * entry=82' repeating while the 1 MiB op's operand never - * resolved, rc=-2; put=84 get=82 last=82 frozen for 10 s). The - * epoch bit is set by svc_replay_note on consume, so only an - * entry that was truly consumed has its withheld set - * delivered-late and dropped here. - */ - if (!test_bit(e->entry_idx, ch->slot_done_epoch)) - continue; + if (!svc_replay_seen(ch, e->entry_idx, put)) + continue; /* never consumed: not stale (rewind) */ if (e->n) svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc_opdef_clear(e, SVC_OPDEF_CLR_STALE); @@ -13481,31 +17238,6 @@ static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, } } -/* - * STALLFREE: drop every opdef slot bound to one channel handle whose - * first-pending stamp PREDATES `before` (the rebinding's reconcile instant). - * A context switch/teardown invalidates the handle's withheld sets - their - * waiters are gone and the releases must not ride the reused channel (the - * opdef-hold class). The `before` filter is the provable-safety rule: a slot - * the fresh binding creates after its registration (the reconcile runs at - * the top of the first pump pass, before that pass's walk) always has - * since >= before and is NEVER touched; only the dead context's slots are - * dropped. Counted as a LOSS when the slot still held undelivered releases. - */ -static void svc_opdef_drop_channel(struct nv_vgpu_device *vgpu, u32 h_object, - unsigned long before) -{ - int i; - - for (i = 0; i < SVC_OPDEF_SLOTS; i++) { - struct svc_opdef_ent *e = &svc_opdef[i]; - - if (e->active && e->vgpu == vgpu && e->h_object == h_object && - time_before(e->since, before)) - svc_opdef_drop_counted(e); - } -} - static void svc_opdef_reset_all(void) { int i; @@ -13712,9 +17444,38 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + /* + * r420: while the backend deliberately holds the in-flight + * launch in its bounded WEDGE-QUIET window, the GR deadline + * must not fire - expiring here authors the launch's release + * word (svc_fwd_hold_gr_release) and applies the recorded + * releases before the post-hold copy-out, so the client + * verifies stale data (the r419 soak's mid-boot + * "matmul fails after time"). Pause instead, bounded by + * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the + * pause extends the deadline a full GR window past the last + * quiet-hold pass, so the backend's retry/accept can finish + * before the fallback would author. Announced once per hold. + */ + if (gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) { + if (!svc_fwd_hold.pause_logged) { + svc_fwd_hold.pause_logged = true; + pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n", + svc_fwd_req.gr_push_va); + } + svc_fwd_hold.paused_until = jiffies + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS); + return false; + } + if (svc_fwd_hold.paused_until && + time_before(jiffies, svc_fwd_hold.paused_until)) + return false; return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -13748,6 +17509,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -13774,6 +17538,8 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_rel_pending = 0; } @@ -14401,6 +18167,16 @@ static void svc_fwd_gr_work(void) data[gs.n_data].hpa = dhpa; data[gs.n_data].reloc_va = dg->reloc_va; /* + * r348 diagnostic: the last 32 high-VA data pages (the output + * operands live there) with the exact (gpa, ap) -> hpa the + * writeback will use - comparable against the D2H read's + * resolution line (sema-bigcopy) for the same VA. + */ + if (dg->va >= SVC_VAS_FLOOR && + i + 32 >= svc_fwd_req.gr_n_data) + pr_info("svc-fwd-hpa: va=0x%llx gpa=0x%llx ap=%u hpa=0x%llx bi=%u\n", + dg->va, dg->pa, dg->ap, dhpa, i); + /* * GRVAUDIT (diagnostic only): audit the m180 landing * authority against the PA this submit will put on the wire * for every tracked data page - the data[]/n_data set the @@ -14446,7 +18222,7 @@ static void svc_fwd_gr_work(void) * permutation. Bounded: first 8 recs, 4 lines per launch, * boot budget 128. */ - if (svc_fwd_req.gr_n_data <= 64u && i < 8u && + if (svc_fwd_req.gr_n_data <= 1024u && i < 8u && svc_readdiv_log < 128u) { u64 opa = 0, lpa2 = 0; u32 oap = 0, lap2 = 0; @@ -14499,6 +18275,12 @@ static void svc_fwd_gr_work(void) gs.rel_hpa = rel_hpa; gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); gs.rel_payload = svc_fwd_req.gr_rel_payload; + /* + * r378: carry the release VA so the backend can re-validate the + * release-word write targets (zero-on-retry, restore-on-accept) + * through gsp_sema_gr_write_guard before they land. + */ + gs.rel_va = svc_fwd_req.gr_rel_va; gs.sem_hpa = 0; gs.sem_off = 0; gs.sem_payload = 0; @@ -14522,6 +18304,17 @@ static void svc_fwd_gr_work(void) ga_va, ga_used, ga_led); hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + /* + * r420b: the backend redirected the launch's QMD release0 to the + * module semaphore (rel_mod=1 or no targetable guest release page), so + * the ENGINE never writes the guest's own QMD release word. Treat the + * word as unobserved so the bounded outcome check (and the hold's + * edges) author it in software on every outcome; without this a + * rel_mod=1 launch with no deferred release records is never completed + * on the guest word and the client hangs (the r414 saxpy stall). + */ + if (cp.qmd_rel_redirected) + svc_fwd_req.gr_rel_unobserved = true; if (!hw_rc) { svc.fwd_gr_ok++; } else { @@ -14800,6 +18593,8 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) u64 pa = 0; u32 ap = 0; u32 i; + /* r372 diagnostic: which branch resolved this launch page. */ + const char *src = "?"; if (!page) return; @@ -14839,6 +18634,44 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) u64 lpa = 0; u32 lap = 0; + /* + * r372 pin-wins: a live injected pin is the guest's newest + * statement about this VA. If it disagrees with the landing + * table, the pin wins and the table is repaired (see the + * PIN-REPAIR note in svc_handle_map_add): the old landing was + * a freed page, and resolving the launch there wrote engine + * output onto memory the guest kernel had reused. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u64 lpa2 = 0; + u32 lap2 = 0; + + if (svc_ledger_lookup(page, &lpa2, &lap2) || + ((lpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap2 != (u32)iap) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS note va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + page, lpa2, lap2, ipa, iap); + svc_ledger_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + svc_fwd_req.gr_pages[i].pa = + ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)iap; + svc_pin_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + iap); + src = "ipin"; + goto note_done; + } + } if (!svc_ledger_lookup(page, &lpa, &lap)) { u64 rpa = 0; u32 rap = 0; @@ -14867,13 +18700,45 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) */ svc_pin_store(page, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), (int)lap); - return; + pa = lpa; + ap = lap; + src = "ledger"; + goto note_done; } } if (svc_resolve_va(vgpu, page, &pa, &ap)) return; svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + /* + * r370: store the landing table for EVERY launch page, not just the + * data-fill spans. An indirect/bank operand (the kernel's output + * buffer, qmd_word NONE) was resolved HERE, the engine wrote it via + * this hpa, the copy-out wrote it back to this hpa - and the client's + * D2H readback then resolved the same VA through the ledger/pin to a + * DIFFERENT page (an older landing), read stale zeros and the kernel + * looked wrong (victory_reduce sum=0 with the correct sum proven in + * the bounce page by OUT-PAGE-WIT). Recording this resolution makes + * the readback read exactly the page the engine wrote. + */ + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "walk"; +note_done: + /* + * r372 diagnostic: the exact (pa, ap) and branch the launch page + * resolved through - the value the engine's hpa and the copy-out use. + * A D2H readback that lands elsewhere (the sum=0 class) names its + * disagreement against these lines. + */ + if (svc_gr_pagenote_log < 512) { + svc_gr_pagenote_log++; + pr_info("sema-gr: page-note va=0x%llx pa=0x%llx ap=%u src=%s qmd=%u\n", + page, svc_fwd_req.gr_pages[i].pa, + svc_fwd_req.gr_pages[i].ap, src, qmd_word); + } } /* Does any captured inline write land in this page? */ @@ -14935,7 +18800,12 @@ static bool svc_gr_page_inline_covers(u64 page) * reach the engine. The legacy value is the pre-B-2 window below the mail * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ -#define SVC_GR_DATA_BOUNCE_MAX GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES +/* + * fail4: each launch may relocate up to half of the grown region - the + * other half is the alternate base of a concurrently in-flight launch + * (see the per-capture base flip in svc_gr_data_fill). + */ +#define SVC_GR_DATA_BOUNCE_MAX (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / 2u) #define SVC_GR_DATA_BOUNCE_MAX_OLD 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ @@ -15154,6 +19024,127 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out) } /* + * r378 write guard, GR backend half (see gsp_sema_service.h). The GR submit + * resolves every operand page once and the copy-out/release writes run + * seconds later (the bounded poll, the retire wait, the outcome retries); a + * guest buffer freed and re-homed inside that window would receive a full + * page of another buffer's content through the stale HPA - the guest + * boot-corruption class (slab freelist clobbered by a foreign page write). + * + * Authority order mirrors the fill's own choice (r372/r376 pin-wins): a + * FRESH injected pin is the guest's newest statement, so the guard compares + * the write target against the pin's page first; only a pin-less VA falls + * back to the trusted resolver chain. The landing table is deliberately NOT + * the authority here - our own launches feed it, so it would confirm the + * target by construction. A miss on every source keeps the historical + * behaviour (the recorded landing is the write authority), so the guard can + * only remove provably stale writes. Bounded diagnostic budget. + */ +static u32 svc_wguard_log; /* r378 WGUARD-SKIP budget (256/boot) */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 ipa = 0, cur_hpa = 0; + int iap = SVC_APERTURE_SYS; + u32 off = 0; + int ret; + + if (!va || !hpa) + return true; + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + + ret = svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + 0, SEMA_PAGE_SIZE, &cur_hpa, &off); + if (ret || off) + return true; /* pin page unresolvable: keep */ + } else { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va, &r)) + return true; /* no current answer: keep */ + cur_hpa = r.hpa; + } + if ((cur_hpa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (hpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx now=0x%llx (stale write target)\n", + path, va, hpa, cur_hpa); + } + return false; +} + +/* + * r378 write guard, service half (see gsp_sema_service.h). A fresh injected + * pin is the guest's newest statement about a VA; a pin that disagrees with + * the recorded write target means the page was freed/re-homed (the r372/r376 + * pin-wins rule applied at the write instead of only at the resolution). A + * missing pin keeps the historical behaviour - the recorded landing is then + * the write authority - so no green path loses its write. + */ +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path) +{ + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!va || !pa || va < SVC_VAS_FLOOR) + return true; + if (svc_ipin_lookup(va, &ipa, &iap)) + return true; /* no fresh pin: nothing newer to obey */ + if ((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + svc_ap_of(iap) == svc_ap_of((int)ap)) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx/%u pin=0x%llx/%d (stale write target)\n", + path, va, pa, ap, ipa, iap); + } + return false; +} + +/* + * r391 diagnostic (sum=0 class): the landing-table resolution of a GR data VA + * plus the first 8 bytes read through it - the exact page the D2H reader's + * ledger-first branch will use. Printed by the GR copy-out next to the write + * target, so a launch/readback disagreement is visible in one line. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8) +{ + u64 lpa = 0; + u32 lap = 0; + + if (svc_ledger_lookup(va, &lpa, &lap)) + return false; + *pa = lpa & ~(u64)(SEMA_PAGE_SIZE - 1); + *ap = lap; + *val8 = 0; + if (!svc_fwd_req.vgpu) + return true; + if (lap == SVC_APERTURE_FB) { + u8 *base = svc_fwd_req.vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(svc_fwd_req.vgpu); + + if (base && *pa + 8 <= size) + memcpy(val8, base + *pa, 8); + } else { + svc_read(svc_fwd_req.vgpu, *pa, val8, 8); + } + return true; +} + +/* r391 diagnostic: the shadow base this service resolves through (compare + * against the CE copy path's own vgpu->bars[1].vaddr in the same run). */ +u8 *gsp_sema_gr_shadow_base(void) +{ + return svc_fwd_req.vgpu ? svc_fwd_req.vgpu->bars[1].vaddr : NULL; +} + +/* * m144 (second-client launch-state divergence): collect device-VA-shaped * candidate pointers from the bytes the ENGINE will actually execute. * @@ -15258,13 +19249,68 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va = 1; /* anonymous launch: still absolutely bound */ if (!cap_ms) cap_ms = 3000; - if (ch->gr_hold_dead_va != launch_va) { + /* + * r368: mirror of the opdef fix - an ACTIVE bound is never re-armed + * by a different launch VA. The pump can hold several launches whose + * push VAs differ; arming on every VA change moved the deadline + * forever (the same identity bug class as the opdef=955/13/0 + * streams2 wedge). The first hold arms; others ride it; a fresh + * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). + */ + if (!ch->gr_hold_dead_va || + time_after_eq(jiffies, ch->gr_hold_deadline)) { ch->gr_hold_dead_va = launch_va; ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; + } + /* + * r420: pause the per-launch consumption bound while the backend + * deliberately holds the in-flight launch in its bounded WEDGE-QUIET + * window. At the bound the pump falls through to the historical + * consume, which authors the capture's QMD completion release in + * software (GR-STALL-hold) - but the backend is still going to + * submit/accept this very launch after the hold and copy its data + * out; authoring now releases the client onto stale data (the r419 + * mid-boot escape). The quiet hold is a bounded msleep (the + * backend's own caps), and the deferral here is bounded by + * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as + * defence-in-depth. Rate-limited announcement. + */ + if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, ch->gr_hold_dead_since + + msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) { + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n", + launch_va); + return false; + } + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + return false; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -15282,8 +19328,32 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * window below the mail page for A/B trialing. */ bool region_on = gsp_phys_gr_data_region_value() != 0; - u64 reloc_base = region_on ? SVC_GR_DATA_RELOC_BASE : - SVC_GR_DATA_RELOC_BASE_OLD; + /* + * fail4: two concurrent forwarded launches (streams2's vadd pair) + * both relocated their operands from the same base, so the second + * launch's copy-in overwrote the first's bounce windows and each + * engine read the other's inputs (c1 came out as a2+b2). Alternate + * the per-launch base across the grown region's two halves (32 MB + * each, above any single launch's span), keyed to the capture push + * so a retried launch keeps its half, so concurrent launches + * cannot collide. + */ + static u64 svc_gr_reloc_half_key; + static u8 svc_gr_reloc_half; + u64 reloc_off = 0; + + if (region_on) { + if (svc_gr_cand.push_va != svc_gr_reloc_half_key) { + svc_gr_reloc_half_key = svc_gr_cand.push_va; + svc_gr_reloc_half ^= 1; + } + if (svc_gr_reloc_half) + reloc_off = (u64)(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / + 2u) * SEMA_PAGE_SIZE; + } + u64 reloc_base = region_on ? + SVC_GR_DATA_RELOC_BASE + reloc_off : + SVC_GR_DATA_RELOC_BASE_OLD; u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR : SVC_GR_DATA_PAGES_PER_PTR_OLD; u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX : @@ -15639,6 +19709,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, wins[w], ((u64)p + 1) << 12, (u64)p << 12); + /* + * r377 pin-demand: name every + * unresolved operand page as a + * stalled VA. The userspace pin + * stream watches dmesg for this + * exact line and pushes the VA + * to the in-guest scanner's + * demand file, so the page gets + * pinned and the bounded launch + * hold (gsp_phys_gr_fill_hold) + * resumes on real data instead + * of waiting out its bound (the + * r345 wedge: held forever + * because nobody demanded the + * page). + */ + svc_log_stalled_va(va); if (dropped_pages < 4) { drop_va[dropped_pages] = va; drop_kind[dropped_pages] = @@ -15662,8 +19749,34 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (va >= SVC_VAS_FLOOR) { u64 lpa = 0; u32 lap = 0; + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; - if (!svc_ledger_lookup(va, &lpa, &lap) && + /* + * r372 pin-wins: when a live pin exists it is the + * newest truth; the landing table may name a page + * the guest has since freed (VA reuse across + * clients). Keep the pin and repair the table. + * Only a pin-less VA still follows the table. + */ + if (!svc_ipin_lookup(va, &ipa, &iap)) { + pa = ipa; + ap = (u32)iap; + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)iap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, ipa, iap); + svc_ledger_store(va, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + } else if (!svc_ledger_lookup(va, &lpa, &lap) && ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || (u32)lap != (u32)ap)) { @@ -15699,12 +19812,27 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; /* + * r349: record the launch's own resolution as the + * ledger's landing authority for this VA. The read + * path's m171 read-after-write rule follows the + * ledger when it disagrees with the live translation, + * and the engine writeback below writes exactly this + * (pa, ap). Without the store the ledger kept a stale + * page for engine-written outputs (victory_reduce: + * the launch wrote FB 0x7c00000 while the ledger + * said SYS 0x7c00000; the D2H read followed the + * ledger and returned zeros while the correct sum + * sat on the written page). + */ + if (va >= SVC_VAS_FLOOR) + svc_ledger_store(va, rec->pa, rec->ap); + /* * m142: bounded per-page attribution log. A wrong data * page is invisible in the aggregate counters; these * lines let a post-mortem compare the chosen PA against * the guest's own mapping. */ - if (svc_gr_data_log < 16) { + if (svc_gr_data_log < 192) { svc_gr_data_log++; pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", va, rec->pa, rec->ap); @@ -15759,7 +19887,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, - false); + gsp_sema_walk_trace != 0); o += snprintf(vb + o, sizeof(vb) - o, " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", vi, svc.vas[vi].pdb, @@ -16047,8 +20175,23 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * The candidate is per-channel: a capture left unconsumed by an * earlier pass (e.g. the entry was held before its request) must never * be queued against a different channel's handle by that channel's - * own end-of-pass request call. + * own end-of-pass request call. fail4: with several reserved ring + * slots, select THIS channel's oldest unconsumed candidate first. */ + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (svc_gr_cand_ring[rp].valid && + !svc_gr_cand_ring[rp].offered && + svc_gr_cand_ring[rp].h_object == ch->h_object) { + svc_gr_cand_pos = rp; + break; + } + } + } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; @@ -16161,6 +20304,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mch->gr_hold_dead_since)); } svc_gr_data_stalled = false; /* proceed: forward */ + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GRFILL-PROCEED launch=0x%llx (bound expired; forwarding best-effort)\n", + lva); + } } } if (svc_gr_data_stalled) { @@ -16548,7 +20696,7 @@ static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h, static void svc_entry_outcome(const char *what, const struct svc_channel *ch, u64 idx, u64 va, u32 len) { - if (!gsp_sema_dbg || !ch->engine_type || svc.gpfifo_entry_log >= 512) + if (!gsp_sema_dbg || svc.gpfifo_entry_log >= 16384) return; svc.gpfifo_entry_log++; pr_info("sema-entry: %s h=0x%x idx=%llu va=0x%llx len=%u want=%u exp=%u unread=%d\n", @@ -16638,121 +20786,52 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the * window): * - * 1. stale epoch (host caught up, last_get == put): the ring still - * carries a session's residue (put > 1). The host resets the stale - * publish state - zero the host-owned completion word and - * counter_last (m181's resync; the residual put is NEVER published, - * see the r207 far-jump note in svc_counter_publish) - and, when the - * whole consumed range delivered nothing (counter_last == 0 and - * submissions >= 8), additionally rewinds last_get to slot 0 - * (reason=stale-dead-range) so the fresh session's own entries, which - * start at slot 0, are walked. The forced-replay marker - * (wd_nocap_put) makes that re-walk decode releases only (idempotent, - * forward-only) and never re-capture/re-forward a dead launch. - * 2. stale replay: last_get pinned behind put while put == get (the + * 1. stale-epoch: a GR channel that still carries a residual put + * (put > 1 at this pass, publish withheld) and whose ring the host + * believes is fully consumed (last_get == put) - the fresh session + * never proved its switch with a put drop. The host applies the + * m181 recovery on a TIMEOUT instead: arm the counter publish and + * void the withheld publish state (no last_get rewind - a timeout is + * not proof the ring is the dead session's; the proven m181 rewind + * stays reserved for its put-drop signature). This is the recovery + * the killed-client rerun needs. + * + * 2. stale-replay: last_get pinned behind put while put == get (the * producer has nothing pending) AND the loop consumed entries during - * the window (submissions moved). The cursor is clamped to put once - * per put epoch (bookkeeping only). The pin-batch rewind is NEVER - * touched: r235 proved disabling it kills the next client's - * context-init (its pins land after the first decode; the rewind is - * the retry path). A clamp that a rewind undoes is harmless. + * the window (submissions moved) - i.e. the rewind is re-opening an + * already-consumed window faster than the loop closes it. The window + * is dropped (last_get = put) and the pin-batch rewind is put on a + * cooldown (wd_rewind_off_until) so the futile retry cannot pin the + * pump; the rewind budget returns later and a translation that lands + * can still retry (bounded duty cycle). * - * Nothing here can fire on a green channel: a green channel either moves - * put or consumes its first entries within milliseconds (submissions > 0), - * so the no-progress window never forms; the stale-replay leg - * additionally requires entries consumed during a stalled window, which - * only the rewind re-opening of a consumed window produces. + * Nothing here can fire on a green channel: a fresh channel has put <= 1 + * (signature 1) or consumes its first entries within milliseconds + * (submissions > 0); signature 2 additionally requires BOTH a stalled + * (last_get, put) pair for 3 s with no pending producer work AND consumed + * entries during that window, which only the rewind re-opening of a + * consumed window produces. */ #define SVC_GR_STALE_WEDGE_MS 3000u -/* - * STALLFREE r235: the replay/clamp leg needs a MUCH longer stalled window - * than the epoch legs. A fresh client's context-init legitimately shows a - * 3 s position oscillation (rewind replays a just-consumed entry while its - * pins land); treating that as a livelock killed the second client (r235: - * 8 channels classified at waited_ms=3491). Only a position stalled for - * ~10 s with consumption happening is a livelock worth clamping. - */ -#define SVC_GR_STALE_REPLAY_MS 10000u -/* - * STALLFREE r229/r230: freshness window after a genuinely new channel - * binding. Within it, a GR entry whose page is unmapped or whose decode - * yields nothing is NOT silently consumed: the pump requests an immediate - * rebuild/pin pass and leaves the entry pending, so the first launch of a - * fresh client executes on its FIRST attempt instead of being dropped - * before the page translations land (r229/r231: attempt 1 stalls, attempt - * 2 works because the rebuild caught up). Stale-free cost: at most this - * window of extra latency on first contact. - */ -#define SVC_GR_NEWBIND_MS 3000u - static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put, u32 get) { - bool catchup, residual, dead_reset, replay_cand, stalled_cand; - bool fire_replay, fire_reset, fire_stall; + bool residual, replay; const char *reason = ""; unsigned long waited; if (ch->engine_type || !ch->gpfifo_entries) return; - /* - * New producer work (put advanced) clears the per-put-epoch latches - * and the replay cooldown: the next stale region gets its own - * recovery. - */ - if (ch->wd_put != put) - ch->wd_epoch_done = false; - /* The forced-replay re-walk is done once the walk catches up. */ - if (ch->wd_nocap_put && ch->last_get == put) - ch->wd_nocap_put = 0; - + /* New work (put advanced) clears the replay cooldown immediately. */ + if (ch->wd_rewind_off_until && ch->wd_put != put) + ch->wd_rewind_off_until = 0; if (ch->wd_since && (ch->wd_put != put || ch->wd_last_get != ch->last_get)) ch->wd_since = 0; /* progress (or a new position): re-arm */ - - catchup = (ch->last_get == put); - replay_cand = (!catchup && put == get && !ch->wd_epoch_done); - /* - * Stale epoch: the host caught up (last_get == put) but the ring - * still carries a session's residue: put > 1 and the epoch reset - * latch is unset. Two additive recoveries: - * - residual: the host published its own value for this channel at - * some point (counter_last != 0), so reset the stale publish - * state (zero the word, m181 semantics). - * - dead-reset: nothing was ever published for this channel - * (counter_last == 0) and the consumed range delivered nothing - * while being non-trivial (submissions >= 8): the whole walked - * range was the dead session's ring. Rewind last_get to 0 so the - * fresh session's own entries (which start at slot 0) get walked; - * the re-walk forces replay so a dead launch is never re-captured - * or re-forwarded. - */ - residual = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last != 0); - dead_reset = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last == 0 && - ch->submissions >= 8); - /* - * STALLFREE r228: stalled PENDING entry. The producer has submitted - * work the host has not consumed (put > get) and the entry loop has - * made no progress - position AND submissions static across the - * window - while no bounded owner is active for the stuck entry: no - * unreadable-push grace (defer_since), no no-capture window, no - * opdef operand slot, no m150 hold window. That combination is a - * genuinely unexplained stall (r228: the opdef deadline kept being - * reset by rewind replays, fixed in svc_opdef_drop_stale; this leg - * is the safety net for any future variant that slips the bounded - * owners). A green channel always advances submissions (consume) or - * sets one of the bounded owners on a pending entry, so the leg - * cannot fire there. - */ - stalled_cand = (!catchup && put != get && !ch->defer_since && - !ch->gr_nocap_since && - !(ch->gr_hold_idx == (u32)ch->last_get && - time_before(jiffies, ch->gr_hold_until)) && - !svc_opdef_find(vgpu, ch->h_object, - (u32)ch->last_get)); - if (!residual && !dead_reset && !replay_cand && !stalled_cand) { + residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); + replay = (ch->last_get != put && put == get && + ch->wd_since && ch->submissions != ch->wd_sub); + if (!residual && !replay) { ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; @@ -16769,98 +20848,86 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) return; waited = jiffies_to_msecs(jiffies - ch->wd_since); - fire_reset = dead_reset; - /* - * The replay leg only fires with proof of a rewind livelock: entries - * were consumed during the window while the position did not move - * (the rewind re-opened an already-consumed window). Without that - * proof this is a producer-pending channel on an unreadable entry, - * which the existing grace paths own. - */ - fire_replay = replay_cand && ch->submissions != ch->wd_sub && - waited >= SVC_GR_STALE_REPLAY_MS; - /* - * The stalled-entry leg fires only with proof of no progress at all: - * nothing was consumed during the window (submissions static), so - * the entry is genuinely parked with no bounded owner. - */ - fire_stall = stalled_cand && ch->submissions == ch->wd_sub; - if (!residual && !fire_reset && !fire_replay && !fire_stall) { - /* - * Keep an accumulating replay window alive (its evidence needs - * SVC_GR_STALE_REPLAY_MS); anything else restarts. - */ - if (!replay_cand) - ch->wd_since = 0; - return; - } - if (residual || fire_reset) { + if (residual) { /* - * Reset the stale publish state. The residual put is NEVER - * published: writing a far-forward value into the fresh - * client's completion word is the r207/mM2 far-jump wedge - * (UVM_GPU_SEMAPHORE_MAX_JUMP), which is exactly what - * gr_pub_armed exists to prevent. Only a word the host itself - * published this session is zeroed (m181 semantics) so a - * channel the host never wrote (counter_last == 0) is left - * untouched; the arm stays reserved for the proven put-drop - * (m181) and the pristine put <= 1 discovery. + * m181 recovery on timeout: the fresh session never dropped + * the residual put, so prove the epoch switch here. Arm the + * publish (the withheld completion counter was the plausible + * ctx-init wait - r225's client polled service-interrupts 1/s + * with no host edge) and void any publish state the stale + * pass advanced, exactly like m181; the guest word itself is + * only zeroed when the host authored it. No last_get rewind: + * a timeout is not proof that the ring is the dead session's + * (re-walking it could re-capture a stale launch), so the + * proven m181 rewind stays reserved for its put-drop + * signature; the client's own first put write will trigger it. */ + ch->gr_pub_armed = true; if (ch->counter_last) { u64 upa = ch->release_pa ? ch->release_pa : ch->counter_pa; u32 zero = 0; - if (upa) - (void)svc_write_ap(vgpu, upa, SVC_APERTURE_SYS, - &zero, sizeof(zero)); + /* + * r311: zero only a word authored by a DEAD + * generation. Zeroing a word this live session + * authored makes the client's UVM observe a + * downward jump (its tracker then never completes) + * while the publish re-advances it - the r311 + * vector_add wedge (pub-adv/zero cycles, client + * polling cur=0 forever). + */ + if (ch->counter_gen == svc_sess_gen) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero-skip h=0x%x upa=0x%llx last=%u (authored by live gen)\n", + ch->h_object, upa, + ch->counter_last); + } + ch->counter_last = 0; + } else if (upa) { + int zrc; + + /* + * r378: the zero is authored from a PA + * learned earlier (release_va -> release_pa); + * re-validate it before the 4-byte write + * lands so a freed/re-homed slot is never + * zeroed through the stale page. + */ + if (!gsp_sema_write_guard(ch->release_va, + upa, + SVC_APERTURE_SYS, + "stale-recover-zero")) + zrc = -ESTALE; + else + zrc = svc_write_ap(vgpu, upa, + SVC_APERTURE_SYS, + &zero, sizeof(zero)); + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n", + ch->h_object, upa, + ch->counter_last, zrc); + } + } ch->counter_last = 0; } - } - if (fire_reset || fire_stall) { - /* - * The canonical unstuck: rewind to slot 0 so the fresh - * session's own entries (which start there) get walked, with - * forced replay (wd_nocap_put) so the re-walk decodes releases - * only and never re-captures a launch; once per put epoch. - */ - ch->wd_epoch_done = true; - ch->last_get = 0; - ch->wd_nocap_put = put; - } - if (fire_replay) { - /* - * STALLFREE r235: clamp the stale software cursor to the - * producer position ONCE per put epoch. The pin-batch rewind - * is deliberately NOT touched: disabling it killed every - * second client (r235-2: 8 channels got a 24 s rewind - * cooldown at ctx-init while their pins were landing, so - * their init pushes were never retried). The clamp is - * bookkeeping only and rewind retries stay available. - */ + reason = "stale-epoch"; + } else { + /* Drop the rewind-reopened consumed window; cool the rewind. */ ch->last_get = put; - ch->wd_epoch_done = true; + ch->wd_rewind_off_until = jiffies + + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + reason = "stale-replay"; } - if (residual || fire_reset || fire_replay) - ch->wd_epoch_done = true; ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; svc.gr_stale_wedge++; - if (fire_replay) - reason = "stale-replay"; - else if (fire_reset) - reason = "stale-dead-range"; - else if (fire_stall) - reason = "stale-entry"; - else - reason = "stale-epoch"; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d clamp=%d entry=%d cpa=0x%llx rpa=0x%llx\n", + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", ch->h_object, ch->gpfifo_va, (u32)ch->last_get, - put, get, ch->submissions, waited, reason, - (fire_reset || fire_stall) ? 1 : 0, - fire_replay ? 1 : 0, fire_stall ? 1 : 0, - ch->counter_pa, ch->release_pa); + put, get, ch->submissions, waited, reason); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -16878,8 +20945,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) + local[n++] = svc.chan[i]; + } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + !svc.chan[i].engine_type) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -17238,12 +21325,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + memset(ch->slot_pend_tries, 0, + sizeof(ch->slot_pend_tries)); /* GRFIX FIX1b: a recycled slot in the next put epoch * gets a fresh absolute-bound record; without this a * wrapped slot would inherit a stale expired launch * record. */ ch->gr_hold_dead_va = 0; ch->gr_hold_exp_logged = false; + /* r361: same for the opdef absolute bound. */ + ch->opdef_hold_dead_va = 0; + ch->opdef_hold_exp_logged = false; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -17257,47 +21349,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_stale_wedge_check(vgpu, ch, put, get); /* - * STALLFREE r229: first-use reconciliation after a genuinely - * new binding (see svc_chan_registration). The rebinding's - * ring cursor starts at 0 (fresh sequence); drop the dead - * context's opdef slots under this handle (their waiters are - * gone and the sets must not ride the reused channel, the - * r228 opdef-hold class), and when the ring already carries - * content (put > 1) log the reconcile so the transition is - * visible. Captures stay enabled: the fresh client's own - * entries may already be at slot 0.., and suppressing them - * would lose exactly the first launch this must protect; a - * stale launch re-walk is bounded by the submit budget and - * the m181 put-drop recovery (same precedent). - */ - if (ch->wd_chanswitch) { - ch->wd_chanswitch = false; - svc_opdef_drop_channel(vgpu, ch->h_object, jiffies); - ch->wd_newbind_until = jiffies + - msecs_to_jiffies(SVC_GR_NEWBIND_MS); - /* - * Bounded proof (r234 asks): the window was armed for - * THIS binding; if a later stall shows no such line, - * its binding never reached the pump (pre-registration - * class) and this mechanism cannot apply. - */ - if (svc.gr_newbind_log < 16) { - svc.gr_newbind_log++; - pr_info("vgsp-gr: GR-STALL-newbind h=0x%x put=%u get=%u action=window reason=stale-entry-newbind\n", - ch->h_object, put, get); - } - if (put > 1) { - svc.gr_stale_wedge++; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=0 action=recover reason=stale-entry-chanswitch reset=0 rewind=0 entry=0 cpa=0x%llx rpa=0x%llx\n", - ch->h_object, ch->gpfifo_va, - (u32)ch->last_get, put, get, - ch->submissions, - ch->counter_pa, - ch->release_pa); - } - } - - /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry @@ -17378,7 +21429,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * (forward-only, so it cannot double-apply) and drop * it before this entry is examined. */ - svc_opdef_drop_stale(vgpu, ch, (u32)idx); + svc_opdef_drop_stale(vgpu, ch, (u32)idx, put); /* * A pin-batch rewind replays recently consumed @@ -17448,10 +21499,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -17459,22 +21517,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * leaves the per-op done bitmap to decide: * work that already ran must not run again, * the pending op must get its retry. + * + * r354: the content witness decides (the hash + * override sits below, after the push bytes + * are read): entry words alone cannot tell a + * true rewind replay (identical bytes - must + * not run twice) from a REWRITTEN push that + * reuses the same push-buffer VA and length + * with new methods (the driver's repeated + * page-table/fill pushes at the same ring + * slots - suppressing those left the guest's + * GPU page tables stale, the GMMU walk + * missing operand pages, and the first CUDA + * client of every fresh boot failing with + * dropped operands). */ entry_replay = same && ch->slot_ops_complete[idx]; - /* - * STALLFREE r227: while a stale-epoch recovery - * re-walks a dead session's consumed range - * (wd_nocap_put set by the watchdog), force - * replay for the dead slots: their releases - * are re-decoded/applied forward-only - * (idempotent) but a dead launch is never - * re-captured or re-forwarded. Cleared by the - * watchdog when the walk catches up. - */ - if (!entry_replay && ch->wd_nocap_put && - (u32)idx < ch->wd_nocap_put) - entry_replay = true; if (same) entry_done = ch->slot_ops_done[idx]; } @@ -17563,7 +21622,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -17754,6 +21826,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ bool cap_trust = injected || rehold; + /* + * r354: the content witness for + * the once-only op gate. Entry + * words alone cannot tell a true + * rewind replay (identical bytes) + * from a rewritten push that + * reuses the same push VA and + * length with new methods (the + * driver's repeated fill / + * page-table pushes at the same + * ring slots). Suppressing the + * rewritten push left the GPU + * page tables stale and the first + * client's operands unresolved. + * slot_ops_hash has been stored + * since r62 for this comparison. + */ + if (entry_replay && + ch->slot_ops_hash[idx] != + svc_push_hash(pb, len)) { + entry_replay = false; + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-sema: rewritten push runs (idx=%llu va=0x%llx len=%u)\n", + (unsigned long long)idx, va, len); + } + } + if (rehold && !injected && + svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", + (unsigned long long)idx, va); + } if (!cap_trust && !entry_replay && !ch->engine_type && gsp_phys_gr_forward_enabled() && @@ -17781,41 +21886,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (rehold) ch->gr_rehold_idx = ~0u; /* - * STALLFREE r230: a fresh - * binding's entry that decoded - * NOTHING (no release, no QMD, - * no ops) is the r229 attempt-1 - * class: the page content read - * before the client's real - * writes/translations landed, so - * consuming it would drop the - * launch. Hold it for the short - * new-binding window and ask for - * an immediate rebuild; after the - * window it consumes as before, so - * the cost is bounded. - */ - if (!ch->engine_type && - !entry_replay && - svc_rel_count == rel_n_iter && - svc_gr_qmd_seen == 0 && - ch->wd_newbind_until && - time_before(jiffies, - ch->wd_newbind_until)) { - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - kfree(pb); - break; - } - /* * Record what ran for this entry * identity before any hold path can * break out: a held entry is decoded @@ -17862,8 +21932,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (svc.op_tr_pending && svc_opdef_hold(vgpu, ch, - (u32)idx)) { + (u32)idx, + va)) { svc.gpfifo_nomap++; + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-pend: hold h=0x%x idx=%u va=0x%llx\n", + ch->h_object, (u32)idx, va); + } /* * P6b wave 1.5: * releases recorded @@ -17887,7 +21963,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -17899,6 +21975,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -17990,7 +22094,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, - ch->gr_hold_until))) { + ch->gr_hold_until) || + /* + * r420: the backend's + * bounded WEDGE-QUIET + * hold outlives this + * one-shot window; keep + * re-holding (never + * consume-with-sw-release) + * while it is active, so + * the launch is still + * forwarded and copied out + * after the hold instead of + * being released onto stale + * data. + */ + gsp_phys_gr_quiet_hold_active())) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; @@ -18119,15 +22238,49 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; - pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=proceed (data-stall hold bound expired; entry consumed, QMD release authored)\n", + /* + * r351: the bound firing used to + * fall through to the historical + * consume (entry consumed, QMD + * release authored) WITHOUT ever + * forwarding the launch - the + * client's kernel never ran and + * the client hung (victory_reduce + * with gsp_phys_gr_fill_hold=1: + * bound fired every 3 s, zero GR + * LAUNCH lines, 110 s+ hang). + * Instead, force ONE more fresh + * capture (rehold) with the stall + * verdict cleared: the request's + * own bound is expired too, so it + * proceeds and forwards the launch + * best-effort with whatever + * resolved. If that retry cannot + * run, the next pass (flag set) + * falls through to the historical + * consume as before. + */ + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=retry-forward (bound expired; one best-effort forward pass)\n", svc_gr_launch_va(ch), jiffies_to_msecs(jiffies - ch->gr_hold_dead_since)); + svc_gr_data_stalled = false; + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(200); + kfree(pb); + break; } /* * P6b wave 1.5: when @@ -18241,10 +22394,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && + (svc_gr_qmd_seen > 0 || + all_zero) && !entry_replay && !(svc_gr_cand.valid && svc_gr_cand.h_object == @@ -18263,6 +22487,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies; } if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( @@ -18296,6 +22521,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len); } + } /* * A CE push that should carry this * channel's next tracking release but @@ -18872,30 +23098,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->defer_since) ch->defer_since = jiffies; - /* - * STALLFREE r230: a fresh - * binding's unmapped entry is - * the host-readiness race - - * request the immediate rebuild - * so the pages resolve inside - * this grace instead of the - * 10 s cadence (r229/r231: - * attempt 1 lost exactly - * here, attempt 2 won after - * the rebuild caught up). - */ - if (!ch->engine_type && - ch->wd_newbind_until && - time_before(jiffies, - ch->wd_newbind_until)) { - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - } svc.gpfifo_nomap++; kfree(pb); break; @@ -19029,6 +23231,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -19055,6 +23265,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so @@ -19102,6 +23316,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; + /* r361: the opdef absolute-bound record is + * snapshot state too (the GRFIX lesson) - + * without the write-back the deadline would + * be zeroed every pass and the bound could + * never fire. */ + svc.chan[k].opdef_hold_dead_va = + ch->opdef_hold_dead_va; + svc.chan[k].opdef_hold_deadline = + ch->opdef_hold_deadline; + svc.chan[k].opdef_hold_exp_logged = + ch->opdef_hold_exp_logged; /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever @@ -19121,10 +23350,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].wd_put = ch->wd_put; svc.chan[k].wd_sub = ch->wd_sub; svc.chan[k].wd_since = ch->wd_since; - svc.chan[k].wd_epoch_done = ch->wd_epoch_done; - svc.chan[k].wd_nocap_put = ch->wd_nocap_put; - svc.chan[k].wd_chanswitch = ch->wd_chanswitch; - svc.chan[k].wd_newbind_until = ch->wd_newbind_until; + /* + * r312: gr_pub_armed / counter_gen live only on + * the snapshot copy if not persisted. Losing the + * armed bit re-arms the residual watchdog every + * pass; its recovery then keeps zeroing the + * completion word and re-publishing - the r311/ + * r312 vector_add wedge loop (66 stale-wedge + * rows with the client polling forever). + */ + svc.chan[k].gr_pub_armed = ch->gr_pub_armed; + svc.chan[k].counter_gen = ch->counter_gen; + svc.chan[k].wd_rewind_off_until = + ch->wd_rewind_off_until; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -19150,6 +23388,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(svc.chan[k].slot_ops_ident, ch->slot_ops_ident, sizeof(ch->slot_ops_ident)); + memcpy(svc.chan[k].slot_pend_tries, + ch->slot_pend_tries, + sizeof(ch->slot_pend_tries)); memcpy(svc.chan[k].slot_ops_done, ch->slot_ops_done, sizeof(ch->slot_ops_done)); @@ -19552,6 +23793,39 @@ static void svc_handle_map_add(void) } svc_ipin_store(va, req.pa + off, ap); svc_lat_store(va, req.pa + off, ap); + /* + * r372 (stale-ledger corruption class): a fresh pin is the + * guest's newest statement of where this VA's content + * lives. The landing table (ledger) otherwise only moves + * on host writes, so a VA re-pinned to a NEW page (the + * guest freed the old buffer and reallocated it - CUDA + * reuses VA ranges across processes) kept resolving to + * the OLD, freed page: every later write through that + * ledger entry landed on memory the guest kernel had + * since reused (proven class: 8 zero bytes into an + * nv_free_system_pages page array -> guest oops at + * client teardown). Repair the entry only when it + * disagrees with the pin (a no-op otherwise, so the + * table is not flooded with pin pages). + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + ((req.pa + off) & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)ap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-REPAIR va=0x%llx old=0x%llx ap=%u -> new=0x%llx ap=%u\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, + req.pa + off, ap); + svc_ledger_store(va, req.pa + off, ap); + } + } } /* * m149: bounded per-session evidence that the CE push-buffer @@ -19594,6 +23868,20 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* + * STALLFREE r225: a channel under the stale-replay + * watchdog's rewind cooldown is NOT rewound. Without + * this the rewind re-opened the same consumed window + * every batch while the producer had nothing pending, + * pinning last_get and re-walking stale entries + * forever (r225: slots 93..100 of 0x5c000059 for + * 42+ s). The cooldown expires on its own (bounded + * duty cycle) and is cleared by new work, so a + * translation landing later can still retry. + */ + if (ch->wd_rewind_off_until && + time_before(jiffies, ch->wd_rewind_off_until)) + continue; + /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only @@ -19997,27 +24285,7 @@ static int svc_thread(void *arg) * svc_rebuild() publishes (svc.last_build). */ sess = svc_guest_get(vgpu, true); - /* - * STALLFREE r230: a pending rebuild request from the - * pump (a fresh binding's entry hit an unmapped page) - * forces the rebuild now instead of waiting for the - * 10 s cadence, so the first attempt's pages resolve. - */ - if (svc.rebuild_now) { - svc.rebuild_now = false; - /* - * Bounded proof (r234 asks): a pump-requested - * rebuild was actually consumed by the service - * thread here (the only consumer). - */ - if (svc.gr_rebuild_log < 16) { - svc.gr_rebuild_log++; - pr_info("vgsp-gr: GR-STALL-rebuild consumed=1 by=svc_thread action=rebuild reason=stale-entry-newbind\n"); - } - svc_rebuild(vgpu); - if (sess) - sess->last_build = svc.last_build; - } else if (svc_sess_flush || !svc.direct_base || + if (svc_sess_flush || !svc.direct_base || !sess || time_after(jiffies, (unsigned long)(sess->last_build + @@ -20545,6 +24813,6 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, * after the write still receives the completion event. */ if (!rc) - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } |