diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 373 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 47 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 8 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 325 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 11 | ||||
| -rw-r--r-- | kernel/main.c | 10 |
6 files changed, 720 insertions, 54 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d21c5ea..f0f4baa 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -39,6 +39,7 @@ #include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/jiffies.h> +#include <linux/ktime.h> #include <linux/timekeeping.h> #include <linux/byteorder/generic.h> #include <linux/pci_regs.h> @@ -1542,6 +1543,13 @@ enum gsp_phys_rmcall_buf { * past the existing hold budget. */ #define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +/* + * F2: the short re-poll after the first-timeout fast re-arm (extended + * re-arm + fresh ring entry). The engine ack for a recovered launch is + * observed within tens of ms (r198); 1500 ms covers the ctxsw path without + * burning another full window before the historical ladder continues. + */ +#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u #define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -17160,6 +17168,142 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, dma_wmb(); } +/* + * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's + * pointer pairs name the win0 shadow bank the previous attempt used, move the + * already-prepared param/program/bank pages into the alternate bank and + * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies + * come from the module shadow pages - after relocation, inline replay and + * pointer fix-ups - so the engine-visible content is byte-identical to the + * first attempt and only the VA changes; the engine therefore cannot serve + * the retry from a cached fetch line of the previous attempt's bank. With + * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here, + * and the engine keeps reading the guest VAs exactly as before. Returns true + * when at least one pair moved (the caller must rebuild the method block). + */ +static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, + const struct gsp_phys_gr_submit *req, + u32 old_bank_base, u32 new_bank_base, + u32 new_bank_limit) +{ + struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0; + u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)old_bank_base * GSP_PHYS_PAGE_SIZE; + u64 old_hi = old_lo + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS * + GSP_PHYS_PAGE_SIZE; + u32 next_slot = new_bank_base; + u32 i; + bool moved = false; + + if (!pool->pages) + return false; + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u32 w = pg->qmd_word; + u64 pair_va, new_va; + u32 off, old_slot; + u8 *src, *dst; + + if (w != 32u && w != 48u && + !(w >= 34u && w <= 46u && !(w & 1u))) + continue; + pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w]; + if (pair_va < old_lo || pair_va >= old_hi) + continue; + if (next_slot >= new_bank_limit) + break; + off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1)); + old_slot = (u32)((pair_va - + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + src = gsp_phys_chunk_page_va(pool, old_slot); + dst = gsp_phys_chunk_page_va(pool, next_slot); + if (!src || !dst) + break; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)next_slot * GSP_PHYS_PAGE_SIZE + off; + qmd[w] = (u32)new_va; + qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) | + ((u32)(new_va >> 32) & 0x1ffffu); + if (w == 48u) { + u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) | + ((u64)qmd[8] << 8); + + if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 np = new_va | + (pref & (GSP_PHYS_PAGE_SIZE - 1)); + + qmd[8] = (u32)(np >> 8); + qmd[31] = (qmd[31] & ~0x1ffu) | + ((u32)(np >> 40) & 0x1ffu); + } + } + next_slot++; + moved = true; + } + if (moved) + dma_wmb(); + return moved; +} + +/* + * F4-a: the first poll break, rendered from a noinline helper so its + * addresses/counter scratch never grow the (already warning-bound) submit + * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack + * (GP_GET advanced, break latency long enough for a real retirement) from a + * host-authored word (break within a few ms of the doorbell, GP_GET frozen). + */ +static noinline void +gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, + const struct gsp_phys_rmcall_ce_submit *sub) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + w0, want_qmd, w1, + (u32)ktime_to_us(ktime_get()) - since_us, + usr_gpget, sub->gpget, sub->gpput); +} + +/* + * F4-b: one-shot deposit readback at the first timeout (noinline for the same + * frame reason as above). Reads the qmdwin/shadow destination (or the guest + * QMD page) with the booter's existing unaligned host readback; w63 is the + * engine fetch sentinel. + */ +static noinline void +gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req, + u8 *qmd_dst_mem) +{ + u8 *qp = qmd_dst_mem ? qmd_dst_mem : + (req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL); + + if (!qp) + return; + pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n", + get_unaligned_le32(qp), + get_unaligned_le32(qp + 16), + get_unaligned_le32(qp + 48), + get_unaligned_le32(qp + 52), + get_unaligned_le32(qp + 72), + get_unaligned_le32(qp + 76), + get_unaligned_le32(qp + 92), + get_unaligned_le32(qp + 128), + get_unaligned_le32(qp + 132), + get_unaligned_le32(qp + 192), + get_unaligned_le32(qp + 196), + get_unaligned_le32(qp + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4)); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -17219,8 +17363,33 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, char qmdwin_note[32] = ""; bool rearmed = false; u32 attempts = 0; + /* + * F2/F4 state: the fast first-timeout re-arm runs once per launch; the + * deposit readback is one-shot per launch (the first timeout is the + * evidence window - the old r198 post-mortem only ran after attempt 3, + * which recovered launches never reached); poll_ms is the window in + * force for the current poll, so the timeout log reports it accurately. + * Static like the other submit scratch: the single-inflight ordered + * work item is the only caller, and statics keep the frame at its + * warning-bound size. + */ + static bool fast_rearmed; + static bool deposit_rb_done; + static u16 poll_ms; + static u32 poll_start_us; int ret = 0; + fast_rearmed = false; + deposit_rb_done = false; + /* + * F2: the shortened first window is part of the fast path; with + * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior + * (the historical 5 s first window and ladder). + */ + poll_ms = gsp_phys_gr_retry_fast_value() ? + (u16)gsp_phys_gr_first_poll_ms_value() : + (u16)GSP_PHYS_RMCALL_GR_POLL_MS; + if (out) memset(out, 0, sizeof(*out)); if (!req || !req->qmd || @@ -18184,14 +18353,15 @@ gr_page_unmapped: gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, - qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); + qmd_rel, want_ch, poll_ms, qmdwin_note); if (qmd_win_diag) pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); - deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); + poll_start_us = (u32)ktime_to_us(ktime_get()); + deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { /* * m72: SKED transition recorder - sample the CWD/HWW registers @@ -18241,17 +18411,81 @@ gr_page_unmapped: ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); - if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* F4-a: first poll break + USERD witnesses. */ + gsp_phys_gr_poll_break_log(w0, want_qmd, w1, + poll_start_us, &sub); break; + } if ((++grpoll_n % 100) == 0) pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n", grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1); if (time_after_eq(jiffies, deadline)) { gsp_phys_gr_timed_out++; pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", - GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1, + poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); /* + * F4-b: one-shot deposit readback at the FIRST timeout + * (the old post-mortem only ran after attempt 3, which + * a recovered launch never reached). Reading the + * qmdwin/shadow destination here proves whether THIS + * attempt's inline-QMD copy landed and what the engine + * would fetch (w63 is the fetch sentinel). Bound: one + * line per launch. + */ + if (!deposit_rb_done) { + deposit_rb_done = true; + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + } + /* + * F2: fast re-arm at the first timeout. The m155 + * extended re-arm (lane-enable replay + GPFIFO/TSG + * schedule + doorbell) plus a FRESH ring entry recovers + * the r198 class in ~11 s only because it runs after two + * full 5 s windows. Do it now and re-poll briefly; if it + * still fails, fall through to the historical ladder + * (the fast entry counts against the same attempt + * budget, so the total number of host publishes and the + * ring wrap exposure stay as before). + */ + if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) { + int fr; + + fast_rearmed = true; + attempts++; + mutex_lock(&gsp_phys_rm_lock); + fr = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!fr) + fr = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!fr) { + gsp_phys_gr_retries++; + poll_ms = + GSP_PHYS_RMCALL_GR_FAST_RETRY_MS; + pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n", + sub.slot, poll_ms, attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + /* + * The immediate m142 re-arm would + * only duplicate the resched that just + * ran; the ladder below still owns the + * warm-sem gate and the remaining + * fresh entries. + */ + rearmed = true; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n", + fr); + } + /* * m142: one bounded re-arm before declaring the launch * dead. Re-issue the stage-4 schedule controls and re-ring * the doorbell, then poll again; if the channel was merely @@ -18266,11 +18500,11 @@ gr_page_unmapped: rr = gsp_phys_rmcall_gr_resched_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); if (!rr) { + poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS; pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n", - GSP_PHYS_RMCALL_GR_POLL_MS); + poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_POLL_MS); + msecs_to_jiffies(poll_ms); grpoll_n = 0; continue; } @@ -18327,14 +18561,13 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); if (!pr_ret) { gsp_phys_gr_retries++; + poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS; pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, - sub.slot, - GSP_PHYS_RMCALL_GR_RETRY_MS); + sub.slot, poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_RETRY_MS); + msecs_to_jiffies(poll_ms); rearmed = false; grpoll_n = 0; continue; @@ -18737,37 +18970,86 @@ gr_page_unmapped: WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); wmb(); - if (qmdwin_on && - boot_stage.rmcall_gr_qmdwin.pages) { - epoch = ++boot_stage.rmcall_gr_submit_epoch; - if (!epoch) + /* + * F3 retry hygiene: recompute the win0 bank + * and slot limit for THIS attempt (they were + * fixed at the first submission's epoch) and + * re-copy the prepared param/program/bank + * shadow pages into the alternate bank before + * the re-publish. The qmdwin bank swap (below) + * is unchanged; when both are active they share + * the same fresh epoch. + */ + { + bool win0_on = + gsp_phys_rmcall_gr_shadow_win0_value() && + boot_stage.rmcall_gr_shadow_win0.pages; + bool win0_moved = false; + bool qmdwin_swap = qmdwin_on && + boot_stage.rmcall_gr_qmdwin.pages; + + if (qmdwin_swap || win0_on) { epoch = ++boot_stage.rmcall_gr_submit_epoch; - qw_bank = (epoch & 1u) % - GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; - qmd_dst = boot_stage.rmcall_gr_qmdwin_va + - (u64)qw_bank * - GSP_PHYS_PAGE_SIZE; - qmd_dst_mem = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_qmdwin, - qw_bank); - if (qmd_dst_mem) { - memcpy(qmd_dst_mem, qmd, - GSP_PHYS_GR_QMD_WORDS * 4); - put_unaligned_le32( - GSP_PHYS_GR_QMD_SENTINEL, - qmd_dst_mem + - GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + if (win0_on) + win0_moved = + gsp_phys_gr_win0_retry_rearm( + qmd, req, + win0_bank_base, + boot_stage.rmcall_gr_shadow_win0_used, + boot_stage.rmcall_gr_shadow_win0_used + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS); + win0_bank_base = + boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = + win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; } - words = gsp_phys_rmcall_build_gr_launch( - methods, req, qmd, qmd_dst, - ch_sem_va, want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), - methods, words * 4); - pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + PB rebuilt\n", - attempts, qw_bank, qmd_dst); - } else { - pr_info("vgsp-gr: outcome retry %u: release words zeroed (qmdwin off)\n", - attempts); + if (qmdwin_swap) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmd_dst = + boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * + GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank); + if (qmd_dst_mem) { + memcpy(qmd_dst_mem, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + put_unaligned_le32( + GSP_PHYS_GR_QMD_SENTINEL, + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + } + } + if (qmdwin_swap || win0_moved) { + words = gsp_phys_rmcall_build_gr_launch( + methods, req, qmd, + qmd_dst, ch_sem_va, + want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), + methods, words * 4); + } + if (qmdwin_swap) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", + attempts, qw_bank, + qmd_dst, win0_bank_base, + win0_moved ? + " (shadow pages re-copied)" : + ""); + else if (win0_moved) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n", + attempts, win0_bank_base); + else + pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", + attempts); } gsp_phys_gr_fill_bounce(req, bounce_csum_in); mutex_lock(&gsp_phys_rm_lock); @@ -18871,6 +19153,17 @@ gr_page_unmapped: if (wrote == 0) { bool restored = false; + /* + * C6 (GAP-2 A1): the no-write accept. Unconditional + * by design - it is the rare, load-bearing event + * (zero engine writes still accepted as success), + * so it must be visible even when no release word + * needed restoring. + */ + pr_info("vgsp-gr: NOWRITE h=0x%x push=0x%llx n_data=%u wrote=0 qmd_rel=0x%x\n", + req->h_object, req->push_va, req->n_data, + want_qmd); + if (qmd_rel_hpa) { u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index b1945ee..95bd627 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -974,6 +974,53 @@ int gsp_phys_rmcall_gr_nowrite_retry_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value); /* + * F2 fast re-arm (gsp_phys_gr_retry_fast; default 1). r198: the first + * forwarded-launch timeout costs 2x5 s before the fresh-entry ladder runs, + * yet the m155 extended re-arm (lane-enable replay + GPFIFO/TSG schedule + + * doorbell) plus one fresh ring entry recovers most of these launches. With + * this on, the first timeout immediately runs that re-arm + a fresh entry + * and re-polls with a short window; only if that still fails does the + * historical timeout ladder continue. Set to 0 for today's exact behavior. + */ +static int gsp_phys_gr_retry_fast_param = 1; +module_param_named(gsp_phys_gr_retry_fast, + gsp_phys_gr_retry_fast_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_retry_fast, + "On the first GR launch timeout, run the extended re-arm + a fresh ring entry immediately and re-poll briefly (default 1; 0 = legacy ladder timing)"); + +int gsp_phys_gr_retry_fast_value(void) +{ + return gsp_phys_gr_retry_fast_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_retry_fast_value); + +/* + * F2 first-poll window (gsp_phys_gr_first_poll_ms; default 2500, clamped to + * [1000, 30000]). With gsp_phys_gr_retry_fast=1 the FIRST poll window of a + * forwarded GR launch (before any re-arm) uses this instead of the fixed 5 s, + * so the fast re-arm path above starts earlier; with retry_fast=0 the launch + * keeps today's exact behavior and this knob is not read. The later ladder + * windows keep their own constants. + */ +static unsigned int gsp_phys_gr_first_poll_ms_param = 2500u; +module_param_named(gsp_phys_gr_first_poll_ms, + gsp_phys_gr_first_poll_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_first_poll_ms, + "First GR launch poll window in ms, clamped [1000,30000], read when gsp_phys_gr_retry_fast=1 (default 2500)"); + +unsigned int gsp_phys_gr_first_poll_ms_value(void) +{ + unsigned int v = gsp_phys_gr_first_poll_ms_param; + + if (v < 1000u) + v = 1000u; + if (v > 30000u) + v = 30000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index a89493b..c058469 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -375,6 +375,14 @@ int gsp_phys_rmcall_gr_skedrec_value(void); * wrong-place engine writes (default 1 = enabled). */ int gsp_phys_rmcall_gr_nowrite_retry_value(void); +/* F2: on the first GR launch timeout, run the extended re-arm + a fresh ring + * entry immediately and re-poll briefly (default 1; 0 = legacy ladder). */ +int gsp_phys_gr_retry_fast_value(void); + +/* F2: first GR launch poll window in ms, clamped [1000,30000], read while + * gsp_phys_gr_retry_fast=1 (default 2500). */ +unsigned int gsp_phys_gr_first_poll_ms_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 7b12ecc..01f94cf 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -805,6 +805,7 @@ struct sema_svc { * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ + u64 crel_entry_dbg; /* C1: rel-apply entry budget, 512/session */ u64 counter_noslot_dbg; /* "no slot registered" budget, * separate so it cannot hide the * written/skip lines */ @@ -831,6 +832,8 @@ struct sema_svc { u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ + u64 walk_dbg; /* C4: GR put/walk witness budget */ + u64 gr_rel_auth_log; /* F1: GR-REL-AUTHOR budget, 256/session */ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ u32 doorbell_token; /* last token written */ u32 doorbell_log; /* token log budget */ @@ -1380,6 +1383,17 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* + * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): + * small, reset per session where they are service-thread state, so a later + * client's strand is never blacked out by the first client's spend. + */ +static u32 svc_relx_log; /* C2: deferred-launch apply entry witness */ +static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ +static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ +static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ +static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -3415,6 +3429,13 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) static void svc_pending_add(u64 va, u32 payload, u32 op); +/* + * F1: GR release-word authorship checks. Defined next to the svc_fwd_req + * slot; forward-declared because the sweep writer runs earlier in the file. + */ +static bool svc_gr_rel_match(u64 va, u32 *payload); +static bool svc_gr_rel_inflight(u64 va, u32 *payload); + static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { u64 addr = 0, old_addr; @@ -4528,6 +4549,17 @@ static void svc_gr_cand_capture(struct svc_channel *ch, { u32 i, n; + /* + * C9 (GAP-2 A4): a capture over a still-valid candidate (never + * offered) destroys an unforwarded launch - the single-slot + * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined + * later in the file and this site may not reference it). + */ + if (svc_gr_cand.valid && svc_cand_ovw_log < 64) { + svc_cand_ovw_log++; + pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.push_va, push_va); + } memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; svc_gr_cand.h_object = ch->h_object; @@ -5668,6 +5700,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, bool skip; int i; u32 dbg = 0; + u64 rel_pa = 0; + u64 cnt_pa = 0; if (!h || !payload) { /* @@ -5705,12 +5739,23 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * FORENSICS-grv7-counter F1). Fall back to counter_pa * when the release slot could not be resolved. */ - pa = svc.chan[i].release_pa ? - svc.chan[i].release_pa : svc.chan[i].counter_pa; + rel_pa = svc.chan[i].release_pa; + cnt_pa = svc.chan[i].counter_pa; + pa = rel_pa ? rel_pa : cnt_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); + /* + * C1 (GAP-1 H3/H4): was the apply entered, and with which slot? + * Budget 512/session (reset in svc_session_flush); the per-branch + * prints below stay on their own counter_rel_dbg budget. + */ + if (gsp_sema_dbg && svc.crel_entry_dbg < 512) { + svc.crel_entry_dbg++; + pr_info("sema-diag: crel-entry h=0x%x va=0x%llx pay=%u inc=%d rel_pa=0x%llx cnt_pa=0x%llx\n", + h, sem_va, payload, inc ? 1 : 0, rel_pa, cnt_pa); + } if (!pa) { /* * Pool (UVM tracking) semaphores have no counter slot. The @@ -6314,6 +6359,22 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, u32 payload, u32 cur, bool pending, bool inc, bool inc_apply) { + /* + * P4 (GAP-1 H3): the exact-slot counter delivery runs before every + * path that can early-return, so a GR completion-counter release can + * never be skipped by the generic write's outcome. The generic + * write/learn tail below is skipped entirely for counter VAs: an + * increment release is not idempotent, so it must apply exactly once + * (the early return here is that exactly-once edge). + */ + if (ch->engine_type == 0 && svc_counter_va(sem_va)) { + if (pending || inc_apply) + svc.pushes_pending++; + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + return false; + } if (pending || inc_apply) { svc.pushes_pending++; if (inc) @@ -6325,6 +6386,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, * generic translation can be a stale content-scanned page; * those releases go through the injected-pin-backed exact slot * (svc_counter_learn / svc_counter_release_apply) below. + * Counter VAs are handled by the early block above; the check + * stays as belt-and-braces for the generic write. */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ @@ -6524,6 +6587,24 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(recs[n].data, rec->data, rec->len); n++; } + /* + * C10 (GAP-2 A5): duplicate destinations in the newest-first + * inline capture decide which record wins the backend's + * overlay; bounded and unconditional (gsp_sema_dbg is defined + * later in the file and this site may not reference it). + */ + if (svc_inline_recs_log < 64) { + u32 dup = 0, a2, b2; + + for (a2 = 0; a2 < n; a2++) + for (b2 = a2 + 1; b2 < n; b2++) + if (recs[a2].dst_va == + recs[b2].dst_va) + dup++; + svc_inline_recs_log++; + pr_info("sema-gr: inline recs n=%u dup_dst=%u\n", + n, dup); + } if (gsp_phys_gr_forward_enabled() && trusted && !replay) svc_gr_cand_capture(ch, st, recs, n, push, push_len, st->push_va); @@ -7978,6 +8059,9 @@ static void svc_session_flush(void) * because the counters were already exhausted by the first client). */ svc.counter_rel_dbg = 0; + svc.crel_entry_dbg = 0; + svc.walk_dbg = 0; + svc.gr_rel_auth_log = 0; svc.counter_noslot_dbg = 0; svc.sem256_log = 0; svc.xfer_raw_log = 0; @@ -8051,6 +8135,13 @@ static void svc_session_flush(void) /* m176: re-arm the opdef diagnostic budget (it was never per-session, * so a late client's strand could be silent). */ svc_opdef_log = 0; + /* One-cycle diag budgets (C2/C3/C7/C8/C9/C10) are per-session too. */ + svc_relx_log = 0; + svc_opdef_clr_log = 0; + svc_busy_defer_log = 0; + svc_hold_fb_log = 0; + svc_cand_ovw_log = 0; + svc_inline_recs_log = 0; /* m176: new session generation - channels and scrubbers re-stamp on * registration and never inherit per-session learned state. */ svc_sess_gen++; @@ -8322,6 +8413,25 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * } /* + * F1-b: while the forwarded GR launch this release belongs to + * is still in flight, the sweep must not author its QMD + * completion word - the backend poll would break on a + * host-authored ack (r198: qmd_rel satisfied ~6 ms after the + * doorbell, zero engine writes). Defer: nothing is written or + * consumed here; the release stays in the guest page and later + * sweep passes re-see it, while the hold/fallback path owns a + * failed hardware copy. + */ + if (gsp_sema_gr_swrel_gate && svc_gr_rel_inflight(va, NULL)) { + if (svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x DEFER\n", + va, payload, 0u); + } + continue; + } + + /* * A release at page offset 0x10 with payload 1 in front of a * channel's first push identifies the page as that channel's * push-buffer base (see svc_pb_add). @@ -8330,6 +8440,18 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); + /* + * F1-a: tag software authorship of the forwarded launch's + * release word (gate off, or the request already resolved and + * the forward progress rule still allows the write). + */ + if (svc_gr_rel_match(va, NULL) && + svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x\n", + va, payload, 0u); + } + if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; @@ -10066,6 +10188,18 @@ bool gsp_sema_counter_derive = true; unsigned gsp_sema_gr_grace_ms = 8000; /* + * F1-b: GR release-word authorship gate (default 1). While a forwarded GR + * launch is in flight, the RAM-content sweep must not author the very QMD + * completion release the backend poll is waiting for: a sweep write could + * satisfy the poll before the engine retires the QMD (false completion, + * silent corruption). With the gate on, such a write is deferred - the + * release stays in RAM and later sweep passes re-see it; when the hardware + * copy resolves, the hold/fallback path applies it. 0 restores the + * unconditional sweep apply for an A/B. + */ +bool gsp_sema_gr_swrel_gate = true; + +/* * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode * saw QMD words but built no launch candidate for its channel (a replay- * suppressed capture, a partial/split QMD, or a push page the pin stream has @@ -11363,6 +11497,15 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, ch->counter_last = cur; return; } + /* + * C5 (GAP-1 H4 secondary): the put-based publish is about to advance + * the same word the decoded releases target; a pre-advance here can + * make a later absolute release a silent skip. Witness every advance. + */ + if (gsp_sema_dbg) + pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n", + ch->h_object, ch->counter_pa, val, cur, + ch->counter_last); if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &val, sizeof(val))) return; @@ -11716,6 +11859,34 @@ static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; /* + * F1 helpers: does a software release write target the release word of the + * forwarded GR launch the request slot holds? The slot is written only by the + * service thread (this code also runs there) and is never reused while the + * hold is active, so no lock is needed - same convention as the existing + * svc_fwd_hold_gr_release()/svc_fwd_hold_expired() reads. svc_gr_rel_match() + * covers the just-resolved hold (done may already be true); the inflight + * variant additionally requires the SVC_FWD_INFLIGHT bit and an unresolved + * work item, i.e. the hardware copy the poll is waiting on is still running. + */ +static bool svc_gr_rel_match(u64 va, u32 *payload) +{ + if (!va || !svc_fwd_req.gr || svc_fwd_req.gr_rel_va != va) + return false; + if (payload) + *payload = svc_fwd_req.gr_rel_payload; + return true; +} + +static bool svc_gr_rel_inflight(u64 va, u32 *payload) +{ + if (!svc_gr_rel_match(va, payload)) + return false; + if (!test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + return false; + return !smp_load_acquire(&svc_fwd_req.done); +} + +/* * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve * the page-aligned GPA through the guest-RAM backing file; VID operands * resolve the shadow byte offset through the vmalloc backing of BAR1 (the @@ -11965,8 +12136,19 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, bool pending = r->op != 0; bool inc = DMA_SEM_REDUCTION_EN(r->dma) && DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC; + int rd = svc_sem_read(vgpu, r->va, &cur); - if (svc_sem_read(vgpu, r->va, &cur)) { + /* + * C2 (GAP-1 H2/H3): prove the opdef apply handed the record to the + * helper and what the software read of the release word returned. + */ + if (gsp_sema_dbg && svc_relx_log < 128) { + svc_relx_log++; + pr_info("sema-diag: relx kind=%u op=%u inc_claim=%u va=0x%llx pay=%u rd=%d\n", + (unsigned)r->kind, (unsigned)r->op, + (unsigned)r->inc_claimed, r->va, r->payload, rd); + } + if (rd) { svc_pending_add(r->va, r->payload, inc ? 1 : 0); svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending, inc, r->inc_claimed); @@ -12103,8 +12285,31 @@ static struct svc_opdef_ent *svc_opdef_alloc(struct nv_vgpu_device *vgpu, return NULL; } -static void svc_opdef_clear(struct svc_opdef_ent *e) +/* C3 caller tags for the opdef-clear witness (H2: which path cleared). */ +#define SVC_OPDEF_CLR_APPLY 0u +#define SVC_OPDEF_CLR_FINISH 1u +#define SVC_OPDEF_CLR_FLUSH 2u +#define SVC_OPDEF_CLR_DROP 3u +#define SVC_OPDEF_CLR_STALE 4u +#define SVC_OPDEF_CLR_COUNTED 5u +#define SVC_OPDEF_CLR_SWEEP 6u +#define SVC_OPDEF_CLR_STOP 7u + +static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) { + /* + * C3 (GAP-1 H2): every clear is witnessed with its caller and the + * slot contents, so a "silent" clear (n=0, or a kind the apply + * switch ignores) is distinguishable from a delivery. + */ + if (gsp_sema_dbg && e->active && svc_opdef_clr_log < 128) { + svc_opdef_clr_log++; + pr_info("sema-diag: opdef-clear caller=%u h=0x%x entry=%u n=%u kind0=%u va0=0x%llx pay0=%u\n", + caller, e->h_object, e->entry_idx, (unsigned)e->n, + e->n ? (unsigned)e->rel[0].kind : 0u, + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } memset(e, 0, sizeof(*e)); } @@ -12142,7 +12347,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, ch->h_object, entry_idx, e->n); } } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_APPLY); } /* @@ -12174,7 +12379,7 @@ static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx) ch->h_object, idx, e->n); } } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_FINISH); return; } } @@ -12286,7 +12491,7 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->h_object, entry_idx, e->n); } svc.opdef_expired++; - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_FLUSH); } /* @@ -12338,7 +12543,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, (u32)(e->n - delivered), e->n); } } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_DROP); } /* @@ -12359,7 +12564,7 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, continue; if (e->n) svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_STALE); } } @@ -12373,7 +12578,7 @@ static void svc_opdef_drop_counted(struct svc_opdef_ent *e) pr_info_ratelimited("sema-opdef: LOSS entry h=0x%x idx=%u dropped with %u undelivered release(s)\n", e->h_object, e->entry_idx, e->n); } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_COUNTED); } static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, @@ -12459,7 +12664,7 @@ static void svc_opdef_expire_sweep(void) pr_info_ratelimited("sema-opdef: SWEEP-LOSS h=0x%x entry=%u n=%u owner channel gone (releases dropped)\n", e->h_object, e->entry_idx, e->n); } - svc_opdef_clear(e); + svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP); } } @@ -12500,6 +12705,16 @@ static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va, return; if (!svc_sem_read(vgpu, va, &cur) && cur >= payload) return; + /* + * F1-a: this is the hold/fallback authorship of the forwarded launch's + * release word (src="hold"), budgeted with the sweep tag so one + * 256-line session window covers both authors. + */ + if (svc_gr_rel_match(va, NULL) && svc.gr_rel_auth_log < 256) { + svc.gr_rel_auth_log++; + pr_info("sema-diag: GR-REL-AUTHOR src=hold va=0x%llx pay=%u h=0x%x\n", + va, payload, h); + } if (!svc_sem_write(vgpu, h, va, payload)) { svc.fwd_gr_sw_rel++; pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n", @@ -12849,6 +13064,22 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write * guest memory). */ + /* + * C7 (GAP-2 A1): the false-completion edge - a hold whose + * hardware copy failed/timed out still software-applies the + * release set here. + */ + if (gsp_sema_dbg && svc_hold_fb_log < 64) { + svc_hold_fb_log++; + pr_info("sema-diag: hold-fallback h=0x%x gr=%d done=%d hw_rc=%d nrel=%u rel0=0x%llx/%u -> sw apply\n", + ch->h_object, ch->engine_type == 0 ? 1 : 0, + done ? 1 : 0, svc_fwd_req.hw_rc, + (unsigned)svc_fwd_hold.nrel, + svc_fwd_hold.nrel ? + svc_fwd_hold.rel[0].va : 0ull, + svc_fwd_hold.nrel ? + svc_fwd_hold.rel[0].payload : 0u); + } if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); @@ -13846,6 +14077,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * in every launch and starved the real operands under the old * window. Skip them as data candidates; they are * shadowed/rewritten through gr_pages/Tier 2, not this path. + * P6: the QMD program pair (word 48/49) is launch-internal in + * exactly the same way and is skipped with them. */ if (svc_fwd_req.gr_param_va && (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == @@ -13859,7 +14092,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) &svc_fwd_req.gr_pages[i]; if (pp->qmd_word >= 32u && - pp->qmd_word <= 47u && + pp->qmd_word <= 49u && (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) { launch_internal = true; @@ -14846,6 +15079,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int rrc; u32 put = 0, get = 0, guard = 0; u32 get_before; + u32 put_prev_wit; u64 applied_before; bool held = false; @@ -15066,6 +15300,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); } + put_prev_wit = ch->put_prev; ch->put_prev = put; /* @@ -15103,6 +15338,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) get_before = ch->last_get; applied_before = svc.releases_applied; + /* + * C4 (GAP-1 H1): the GR walk witness - does put advance, and + * does last_get reach it? Budget 512/session; fires on a put + * change or while last_get is in the stalled 80..100 band. + */ + if (gsp_sema_dbg && svc.walk_dbg < 512 && + (put != put_prev_wit || + ((u32)ch->last_get >= 80 && (u32)ch->last_get <= 100))) { + svc.walk_dbg++; + pr_info("sema-diag: walk h=0x%x put=%u get=%u last=%u prev=%u\n", + ch->h_object, put, get, (u32)ch->last_get, + put_prev_wit); + } + while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; @@ -15693,8 +15942,56 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) fwd_guest_queued = svc_fwd_request(vgpu, ch); - if (fwd_guest_queued) + if (fwd_guest_queued) { svc.fwd_gr_defer++; + } else if (ch->gr_hold_idx != (u32)idx || + time_before(jiffies, + ch->gr_hold_until)) { + u32 hold_ms = gsp_sema_gr_grace_ms ? + gsp_sema_gr_grace_ms : 8000; + + /* + * P2 (GAP-2 A3): the wait + * expired without queueing. + * Do NOT fall through to the + * software-release consume - + * the launch was never + * submitted, so authoring + * its release would be a + * false completion. Hold + * exactly like the + * data-stalled path + * (m150/m156); the one-shot + * window lets the historical + * consume resume after it + * expires. + */ + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + svc.fwd_gr_defer++; + if (svc_busy_defer_log < 64) { + svc_busy_defer_log++; + pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> hold %u ms\n", + ch->h_object, + (unsigned long long)idx, + hold_ms); + } + kfree(pb); + break; + } else if (svc_busy_defer_log < 64) { + /* + * C8 (GAP-2 A3): the bounded + * hold window is over; the + * historical consume with the + * software release resumes. + */ + svc_busy_defer_log++; + pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> consume w/ sw release\n", + ch->h_object, + (unsigned long long)idx); + } } /* * m150: the GR fill held this @@ -17832,7 +18129,7 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) /* Held-operand release slots of this guest cannot be delivered. */ for (i = 0; i < SVC_OPDEF_SLOTS; i++) if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu) - svc_opdef_clear(&svc_opdef[i]); + svc_opdef_clear(&svc_opdef[i], SVC_OPDEF_CLR_STOP); /* Free the per-guest session and scrubber records. */ spin_lock_irqsave(&svc_guest_lock, flags); diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index b8c4dfb..e79064c 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -285,4 +285,15 @@ extern bool gsp_sema_pool_hold_fresh; */ extern bool gsp_sema_copy_dst_pin_guard; +/* + * F1-b: authorship gate for the GR completion release the forwarded launch's + * poll waits on. When on (default), a RAM-sweep release whose VA equals the + * in-flight forward request's gr_rel_va is deferred (skipped, not applied) + * while the hardware copy is still in flight, so the poll can only break on + * an engine-authored word; the deferred release is re-seen by later sweep + * passes and the hold/fallback paths apply it if the hardware copy fails. + * 0 restores the unconditional sweep apply. + */ +extern bool gsp_sema_gr_swrel_gate; + #endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/main.c b/kernel/main.c index d2dc57a..60bab26 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2771,6 +2771,16 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644); MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)"); /* + * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a + * RAM-sweep release whose VA is that launch's QMD completion word, so the + * backend poll can only break on an engine-authored ack. Deferred releases + * are re-seen by later sweep passes; the hold/fallback path applies them when + * the hardware copy fails. 0 restores the unconditional sweep apply. + */ +module_param_named(gsp_sema_gr_swrel_gate, gsp_sema_gr_swrel_gate, bool, 0644); +MODULE_PARM_DESC(gsp_sema_gr_swrel_gate, "Defer a sweep-authored in-flight GR release word (default 1; 0 = unconditional sweep apply)"); + +/* * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode * saw QMD words but built no launch candidate (replay-suppressed capture, * partial/split QMD, or an unreadable push page). The entry stays unconsumed |