From da9a755bd8841fca0f01199c99aeaa495a108da7 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 16 Oct 2026 16:02:00 -0400 Subject: vgsp: m160 reliability wave: fresh-ack outcome retries (zero release words, fresh qmdwin bank + rebuilt PB, restore-on-no-write), event-table slot reclaim (free-slot reuse + loud full log), GR capture trust for fresh plausibility-verified ring entries --- kernel/gsp_phys_booter.c | 113 ++++++++++++++++++++++++++++++++++++++++++++-- kernel/gsp_rm_emulation.c | 101 ++++++++++++++++++++++++++--------------- kernel/gsp_sema_service.c | 33 +++++++++++++- 3 files changed, 207 insertions(+), 40 deletions(-) diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0f7e91c..009eb4f 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17186,6 +17186,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; + u32 qw_bank = 0; char qmdwin_note[32] = ""; bool rearmed = false; u32 attempts = 0; @@ -17818,7 +17819,7 @@ gr_page_unmapped: NULL; if (boot_stage.rmcall_gr_qmdwin_va && qw) { - u32 qw_bank = (epoch & 1u) % + qw_bank = (epoch & 1u) % GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; qmdwin_on = true; @@ -18643,6 +18644,69 @@ gr_page_unmapped: break; } } + /* + * m160 (fresh-ack + fresh-deposit retry): the m153 + * retry re-published the same PB and then polled + * the release words WITHOUT clearing them - the + * previous attempt's ack (the identical payload) + * satisfied the poll instantly, so every retry + * burned only the retirement wait and concluded + * "engine wrote 0" without ever having waited for + * the engine to consume the re-published entry + * (r162: 3 dud retries, ~870 ms each, then + * accepted). Clear the guest release words (and + * the module semaphore when the QMD won't reach a + * guest page) so the poll below waits for a FRESH + * ack, and move the inline-QMD deposit to the + * other qmdwin bank (fresh epoch) and rebuild the + * method block for it, so the engine cannot have + * a cached fetch line or a stale deposit for the + * retry's entry. The mechanical sequence is + * exactly the one r162's ok=7 saga used when it + * recovered the no-execute launch (fresh ring + * entry + resched + fresh deposit). + */ + if (qmd_rel_hpa) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off), 0); + if (ch_rel_hpa) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off), 0); + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + if (qmdwin_on && + boot_stage.rmcall_gr_qmdwin.pages) { + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmd_dst = boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * + GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank); + if (qmd_dst_mem) { + memcpy(qmd_dst_mem, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + put_unaligned_le32( + GSP_PHYS_GR_QMD_SENTINEL, + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + } + words = gsp_phys_rmcall_build_gr_launch( + methods, req, qmd, qmd_dst, + ch_sem_va, want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), + methods, words * 4); + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + PB rebuilt\n", + attempts, qw_bank, qmd_dst); + } else { + pr_info("vgsp-gr: outcome retry %u: release words zeroed (qmdwin off)\n", + attempts); + } gsp_phys_gr_fill_bounce(req, bounce_csum_in); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( @@ -18661,8 +18725,10 @@ gr_page_unmapped: { unsigned long dl = jiffies + msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_RETRY_MS); + GSP_PHYS_RMCALL_GR_POLL_MS); + unsigned long ack_start = jiffies; u32 rw0 = 0, rw1 = 0; + bool acked = false; for (;;) { if (qmd_rel_hpa) @@ -18683,18 +18749,59 @@ gr_page_unmapped: *(u32 *)(sem_va + 4)); if ((!qmd_rel || rw0 == want_qmd) && - rw1 == want_ch) + rw1 == want_ch) { + acked = true; break; + } if (time_after_eq(jiffies, dl)) break; msleep(GSP_PHYS_RMCALL_POLL_MS); } + pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", + attempts, + acked ? "OK" : "TIMEOUT", + rw0, want_qmd, rw1, + jiffies_to_msecs(jiffies - + ack_start)); } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", attempts, wrote); } + /* + * m160: if every fresh-ack retry still produced no + * engine writes, the release words may be left zeroed + * from the last retry. Restore the expected payloads so + * the guest client observes its completion and proceeds + * to its own result check (a genuinely lost launch then + * surfaces as the client's own FAIL, never as a hang). + */ + if (wrote == 0) { + bool restored = false; + + if (qmd_rel_hpa) { + u8 *rp = (u8 *)__va(qmd_rel_hpa) + + qmd_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_qmd) { + WRITE_ONCE(*(u32 *)rp, want_qmd); + restored = true; + } + } + if (ch_rel_hpa) { + u8 *rp = (u8 *)__va(ch_rel_hpa) + + ch_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_ch) { + WRITE_ONCE(*(u32 *)rp, want_ch); + restored = true; + } + } + if (restored) + pr_info("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", + want_qmd, want_ch); + } } } /* diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 2ffc8ac..0afb013 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2444,51 +2444,80 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, * notifyIndex, data}). The driver allocates one per channel and waits * for the GPU to post the event before it considers submitted work * complete - remember them so completed work can be signalled back. + * + * The table is append+retire: FREE retires records (valid=false) but + * never shrinks events_n, so long-lived boots with many client + * create/destroy cycles would otherwise exhaust the 128 slots and + * SILENTLY drop new registrations - after which the completion + * wake-ups (post_channel_event / post_all_events) fail with EINVAL + * and a later client waits forever for a completion that can no + * longer be notified (observed r162: `completion hint rc=-22` + * repeating from the first post-teardown client onward). Reclaim the + * first retired slot for a new registration, and log loudly when the + * table is genuinely full of live entries. */ - if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) { + if (class_id == 0x79 && params_size >= 24) { u32 h_src = 0, notify_index = 0, hclass = 0; u64 data64 = 0; int hc_off = -1, i; + u32 slot = rm->events_n; - /* - * Locate hClass (0x79) inside the params instead of trusting a - * fixed offset: NV0005_ALLOC_PARAMETERS is - * {hParentClient, hSrcResource, hClass, notifyIndex, data}, - * with hSrcResource immediately before hClass. - */ - for (i = 0; i + 4 <= (int)params_size; i += 4) { - u32 v; - - memcpy(&v, params + i, 4); - if (v == 0x79) { - hc_off = i; + for (i = 0; i < (int)rm->events_n; i++) { + if (!rm->events[i].valid) { + slot = (u32)i; break; } } - if (hc_off >= 4) { - memcpy(&h_src, params + hc_off - 4, 4); - memcpy(&hclass, params + hc_off, 4); - if (hc_off + 8 <= (int)params_size) - memcpy(¬ify_index, params + hc_off + 4, 4); - if (hc_off + 16 <= (int)params_size) - memcpy(&data64, params + hc_off + 8, 8); - } else { - memcpy(&h_src, params + 4, 4); - memcpy(¬ify_index, params + 12, 4); - memcpy(&data64, params + 16, 8); + if (slot >= GSPRM_MAX_EVENTS) + nv_vgpu_err(ctx->vgpu, + "vgsp-rm: event table full (%u live records); dropping registration hObject=0x%x - completion wake-ups will not reach this client", + rm->events_n, ho); + else { + /* + * Locate hClass (0x79) inside the params instead of + * trusting a fixed offset: NV0005_ALLOC_PARAMETERS is + * {hParentClient, hSrcResource, hClass, notifyIndex, + * data}, with hSrcResource immediately before hClass. + */ + for (i = 0; i + 4 <= (int)params_size; i += 4) { + u32 v; + + memcpy(&v, params + i, 4); + if (v == 0x79) { + hc_off = i; + break; + } + } + if (hc_off >= 4) { + memcpy(&h_src, params + hc_off - 4, 4); + memcpy(&hclass, params + hc_off, 4); + if (hc_off + 8 <= (int)params_size) + memcpy(¬ify_index, + params + hc_off + 4, 4); + if (hc_off + 16 <= (int)params_size) + memcpy(&data64, + params + hc_off + 8, 8); + } else { + memcpy(&h_src, params + 4, 4); + memcpy(¬ify_index, params + 12, 4); + memcpy(&data64, params + 16, 8); + } + rm->events[slot].h_src = h_src; + rm->events[slot].h_event = ho; + rm->events[slot].h_client = hc; + rm->events[slot].notify_index = notify_index; + rm->events[slot].data = (u32)data64; + rm->events[slot].valid = true; + if (slot == rm->events_n) + rm->events_n++; + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d, slot %u) params=%*phN", + ho, h_src, notify_index, hc_off, + slot, + (int)min(params_size, (u32)24), + params); } - rm->events[rm->events_n].h_src = h_src; - rm->events[rm->events_n].h_event = ho; - rm->events[rm->events_n].h_client = hc; - rm->events[rm->events_n].notify_index = notify_index; - rm->events[rm->events_n].data = (u32)data64; - rm->events[rm->events_n].valid = true; - rm->events_n++; - if (gsp_sema_dbg) - nv_vgpu_info(ctx->vgpu, - "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN", - ho, h_src, notify_index, hc_off, - (int)min(params_size, (u32)24), params); } /* Echo the fixed alloc header + as many param bytes as are contiguous diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2019a35..c9c36b3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -14414,6 +14414,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->engine_type && ch->gr_rehold_idx == (u32)idx; + /* + * m160: the capture gate used to + * require a host-injected pin. A + * fresh ring entry whose push read + * succeeded and whose bytes parse + * as a plausible method stream is + * the client's actual work - the + * same evidence the untrusted ops + * gate already accepts - and a GR + * capture built from it is exactly + * what the m156 rehold forces for + * deferred launches. Without this, + * a channel whose pages the pin + * stream had not reached yet (the + * streams2 second-channel case: + * entry consumed, no capture, the + * client read zeros) never gets + * its launch forwarded at all. + * Replays stay suppressed: a + * consumed entry must not run its + * work twice. + */ + bool cap_trust = injected || rehold; + + if (!cap_trust && !entry_replay && + !ch->engine_type && + gsp_phys_gr_forward_enabled() && + !unreadable && + svc_method_stream_plausible(pb, + len)) + cap_trust = true; svc_entry_outcome("decode", ch, idx, va, len); @@ -14426,7 +14457,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, idx, va, - injected || rehold, + cap_trust, entry_replay && !rehold, (entry_replay && !rehold) ? NULL : -- cgit v1.2.3