summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--kernel/gsp_phys_booter.c373
-rw-r--r--kernel/gsp_phys_gpu.c47
-rw-r--r--kernel/gsp_phys_gpu.h8
-rw-r--r--kernel/gsp_sema_service.c325
-rw-r--r--kernel/gsp_sema_service.h11
-rw-r--r--kernel/main.c10
-rw-r--r--tools/guest-diag/guest_pin_stream.py213
-rw-r--r--tools/guest-diag/pin_gr_stream.py91
-rwxr-xr-xtools/guest-diag/pin_supervisor.sh44
-rwxr-xr-xtools/guest-diag/semfix_uvmt.py70
10 files changed, 994 insertions, 198 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index d21c5ea..f0f4baa 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -39,6 +39,7 @@
#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/jiffies.h>
+#include <linux/ktime.h>
#include <linux/timekeeping.h>
#include <linux/byteorder/generic.h>
#include <linux/pci_regs.h>
@@ -1542,6 +1543,13 @@ enum gsp_phys_rmcall_buf {
* past the existing hold budget.
*/
#define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u
+/*
+ * F2: the short re-poll after the first-timeout fast re-arm (extended
+ * re-arm + fresh ring entry). The engine ack for a recovered launch is
+ * observed within tens of ms (r198); 1500 ms covers the ctxsw path without
+ * burning another full window before the historical ladder continues.
+ */
+#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u
#define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u
/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */
#define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u
@@ -17160,6 +17168,142 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req,
dma_wmb();
}
+/*
+ * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's
+ * pointer pairs name the win0 shadow bank the previous attempt used, move the
+ * already-prepared param/program/bank pages into the alternate bank and
+ * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies
+ * come from the module shadow pages - after relocation, inline replay and
+ * pointer fix-ups - so the engine-visible content is byte-identical to the
+ * first attempt and only the VA changes; the engine therefore cannot serve
+ * the retry from a cached fetch line of the previous attempt's bank. With
+ * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here,
+ * and the engine keeps reading the guest VAs exactly as before. Returns true
+ * when at least one pair moved (the caller must rebuild the method block).
+ */
+static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd,
+ const struct gsp_phys_gr_submit *req,
+ u32 old_bank_base, u32 new_bank_base,
+ u32 new_bank_limit)
+{
+ struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0;
+ u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+ (u64)old_bank_base * GSP_PHYS_PAGE_SIZE;
+ u64 old_hi = old_lo +
+ (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS *
+ GSP_PHYS_PAGE_SIZE;
+ u32 next_slot = new_bank_base;
+ u32 i;
+ bool moved = false;
+
+ if (!pool->pages)
+ return false;
+ for (i = 0; i < req->n_pages; i++) {
+ const struct gsp_phys_gr_page *pg = &req->pages[i];
+ u32 w = pg->qmd_word;
+ u64 pair_va, new_va;
+ u32 off, old_slot;
+ u8 *src, *dst;
+
+ if (w != 32u && w != 48u &&
+ !(w >= 34u && w <= 46u && !(w & 1u)))
+ continue;
+ pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w];
+ if (pair_va < old_lo || pair_va >= old_hi)
+ continue;
+ if (next_slot >= new_bank_limit)
+ break;
+ off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1));
+ old_slot = (u32)((pair_va -
+ GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) /
+ GSP_PHYS_PAGE_SIZE);
+ src = gsp_phys_chunk_page_va(pool, old_slot);
+ dst = gsp_phys_chunk_page_va(pool, next_slot);
+ if (!src || !dst)
+ break;
+ memcpy(dst, src, GSP_PHYS_PAGE_SIZE);
+ new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE +
+ (u64)next_slot * GSP_PHYS_PAGE_SIZE + off;
+ qmd[w] = (u32)new_va;
+ qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) |
+ ((u32)(new_va >> 32) & 0x1ffffu);
+ if (w == 48u) {
+ u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) |
+ ((u64)qmd[8] << 8);
+
+ if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) ==
+ (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) {
+ u64 np = new_va |
+ (pref & (GSP_PHYS_PAGE_SIZE - 1));
+
+ qmd[8] = (u32)(np >> 8);
+ qmd[31] = (qmd[31] & ~0x1ffu) |
+ ((u32)(np >> 40) & 0x1ffu);
+ }
+ }
+ next_slot++;
+ moved = true;
+ }
+ if (moved)
+ dma_wmb();
+ return moved;
+}
+
+/*
+ * F4-a: the first poll break, rendered from a noinline helper so its
+ * addresses/counter scratch never grow the (already warning-bound) submit
+ * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack
+ * (GP_GET advanced, break latency long enough for a real retirement) from a
+ * host-authored word (break within a few ms of the doorbell, GP_GET frozen).
+ */
+static noinline void
+gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us,
+ const struct gsp_phys_rmcall_ce_submit *sub)
+{
+ struct gsp_phys_boot_chunk *ud_c =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
+ u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL;
+ u32 usr_gpget = ud ?
+ get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() +
+ GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0;
+
+ pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n",
+ w0, want_qmd, w1,
+ (u32)ktime_to_us(ktime_get()) - since_us,
+ usr_gpget, sub->gpget, sub->gpput);
+}
+
+/*
+ * F4-b: one-shot deposit readback at the first timeout (noinline for the same
+ * frame reason as above). Reads the qmdwin/shadow destination (or the guest
+ * QMD page) with the booter's existing unaligned host readback; w63 is the
+ * engine fetch sentinel.
+ */
+static noinline void
+gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req,
+ u8 *qmd_dst_mem)
+{
+ u8 *qp = qmd_dst_mem ? qmd_dst_mem :
+ (req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL);
+
+ if (!qp)
+ return;
+ pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n",
+ get_unaligned_le32(qp),
+ get_unaligned_le32(qp + 16),
+ get_unaligned_le32(qp + 48),
+ get_unaligned_le32(qp + 52),
+ get_unaligned_le32(qp + 72),
+ get_unaligned_le32(qp + 76),
+ get_unaligned_le32(qp + 92),
+ get_unaligned_le32(qp + 128),
+ get_unaligned_le32(qp + 132),
+ get_unaligned_le32(qp + 192),
+ get_unaligned_le32(qp + 196),
+ get_unaligned_le32(qp +
+ GSP_PHYS_GR_QMD_SENTINEL_WORD * 4));
+}
+
int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
struct gsp_phys_backend_copy_result *out)
{
@@ -17219,8 +17363,33 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
char qmdwin_note[32] = "";
bool rearmed = false;
u32 attempts = 0;
+ /*
+ * F2/F4 state: the fast first-timeout re-arm runs once per launch; the
+ * deposit readback is one-shot per launch (the first timeout is the
+ * evidence window - the old r198 post-mortem only ran after attempt 3,
+ * which recovered launches never reached); poll_ms is the window in
+ * force for the current poll, so the timeout log reports it accurately.
+ * Static like the other submit scratch: the single-inflight ordered
+ * work item is the only caller, and statics keep the frame at its
+ * warning-bound size.
+ */
+ static bool fast_rearmed;
+ static bool deposit_rb_done;
+ static u16 poll_ms;
+ static u32 poll_start_us;
int ret = 0;
+ fast_rearmed = false;
+ deposit_rb_done = false;
+ /*
+ * F2: the shortened first window is part of the fast path; with
+ * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior
+ * (the historical 5 s first window and ladder).
+ */
+ poll_ms = gsp_phys_gr_retry_fast_value() ?
+ (u16)gsp_phys_gr_first_poll_ms_value() :
+ (u16)GSP_PHYS_RMCALL_GR_POLL_MS;
+
if (out)
memset(out, 0, sizeof(*out));
if (!req || !req->qmd ||
@@ -18184,14 +18353,15 @@ gr_page_unmapped:
gsp_phys_rmcall_gr_prelude_value(),
gsp_phys_rmcall_gr_qmdvirt_value(),
boot_stage.rmcall_token,
- qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note);
+ qmd_rel, want_ch, poll_ms, qmdwin_note);
if (qmd_win_diag)
pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n",
qmd_dst, qmd_win_pde, qmd_win_installed,
(unsigned long long)boot_stage.rmcall_guest_pt_win_mask,
qmd_win_ret);
- deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
+ poll_start_us = (u32)ktime_to_us(ktime_get());
+ deadline = jiffies + msecs_to_jiffies(poll_ms);
for (;;) {
/*
* m72: SKED transition recorder - sample the CWD/HWW registers
@@ -18241,17 +18411,81 @@ gr_page_unmapped:
ch_rel_off));
else
w1 = READ_ONCE(*(u32 *)(sem_va + 4));
- if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch)
+ if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) {
+ /* F4-a: first poll break + USERD witnesses. */
+ gsp_phys_gr_poll_break_log(w0, want_qmd, w1,
+ poll_start_us, &sub);
break;
+ }
if ((++grpoll_n % 100) == 0)
pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n",
grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1);
if (time_after_eq(jiffies, deadline)) {
gsp_phys_gr_timed_out++;
pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n",
- GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1,
+ poll_ms, w0, want_qmd, w1,
want_ch, req->h_object, req->push_va);
/*
+ * F4-b: one-shot deposit readback at the FIRST timeout
+ * (the old post-mortem only ran after attempt 3, which
+ * a recovered launch never reached). Reading the
+ * qmdwin/shadow destination here proves whether THIS
+ * attempt's inline-QMD copy landed and what the engine
+ * would fetch (w63 is the fetch sentinel). Bound: one
+ * line per launch.
+ */
+ if (!deposit_rb_done) {
+ deposit_rb_done = true;
+ gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem);
+ }
+ /*
+ * F2: fast re-arm at the first timeout. The m155
+ * extended re-arm (lane-enable replay + GPFIFO/TSG
+ * schedule + doorbell) plus a FRESH ring entry recovers
+ * the r198 class in ~11 s only because it runs after two
+ * full 5 s windows. Do it now and re-poll briefly; if it
+ * still fails, fall through to the historical ladder
+ * (the fast entry counts against the same attempt
+ * budget, so the total number of host publishes and the
+ * ring wrap exposure stay as before).
+ */
+ if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) {
+ int fr;
+
+ fast_rearmed = true;
+ attempts++;
+ mutex_lock(&gsp_phys_rm_lock);
+ fr = gsp_phys_rmcall_publish_locked(
+ bar0, boot_stage.rmcall_va_pb, words,
+ &sub);
+ if (!fr)
+ fr = gsp_phys_rmcall_gr_resched_locked(
+ bar0);
+ mutex_unlock(&gsp_phys_rm_lock);
+ if (!fr) {
+ gsp_phys_gr_retries++;
+ poll_ms =
+ GSP_PHYS_RMCALL_GR_FAST_RETRY_MS;
+ pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n",
+ sub.slot, poll_ms, attempts,
+ GSP_PHYS_RMCALL_GR_ATTEMPTS);
+ deadline = jiffies +
+ msecs_to_jiffies(poll_ms);
+ /*
+ * The immediate m142 re-arm would
+ * only duplicate the resched that just
+ * ran; the ladder below still owns the
+ * warm-sem gate and the remaining
+ * fresh entries.
+ */
+ rearmed = true;
+ grpoll_n = 0;
+ continue;
+ }
+ pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n",
+ fr);
+ }
+ /*
* m142: one bounded re-arm before declaring the launch
* dead. Re-issue the stage-4 schedule controls and re-ring
* the doorbell, then poll again; if the channel was merely
@@ -18266,11 +18500,11 @@ gr_page_unmapped:
rr = gsp_phys_rmcall_gr_resched_locked(bar0);
mutex_unlock(&gsp_phys_rm_lock);
if (!rr) {
+ poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS;
pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n",
- GSP_PHYS_RMCALL_GR_POLL_MS);
+ poll_ms);
deadline = jiffies +
- msecs_to_jiffies(
- GSP_PHYS_RMCALL_GR_POLL_MS);
+ msecs_to_jiffies(poll_ms);
grpoll_n = 0;
continue;
}
@@ -18327,14 +18561,13 @@ gr_page_unmapped:
mutex_unlock(&gsp_phys_rm_lock);
if (!pr_ret) {
gsp_phys_gr_retries++;
+ poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS;
pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n",
attempts,
GSP_PHYS_RMCALL_GR_ATTEMPTS,
- sub.slot,
- GSP_PHYS_RMCALL_GR_RETRY_MS);
+ sub.slot, poll_ms);
deadline = jiffies +
- msecs_to_jiffies(
- GSP_PHYS_RMCALL_GR_RETRY_MS);
+ msecs_to_jiffies(poll_ms);
rearmed = false;
grpoll_n = 0;
continue;
@@ -18737,37 +18970,86 @@ gr_page_unmapped:
WRITE_ONCE(*(u32 *)sem_va, 0);
WRITE_ONCE(*(u32 *)(sem_va + 4), 0);
wmb();
- if (qmdwin_on &&
- boot_stage.rmcall_gr_qmdwin.pages) {
- epoch = ++boot_stage.rmcall_gr_submit_epoch;
- if (!epoch)
+ /*
+ * F3 retry hygiene: recompute the win0 bank
+ * and slot limit for THIS attempt (they were
+ * fixed at the first submission's epoch) and
+ * re-copy the prepared param/program/bank
+ * shadow pages into the alternate bank before
+ * the re-publish. The qmdwin bank swap (below)
+ * is unchanged; when both are active they share
+ * the same fresh epoch.
+ */
+ {
+ bool win0_on =
+ gsp_phys_rmcall_gr_shadow_win0_value() &&
+ boot_stage.rmcall_gr_shadow_win0.pages;
+ bool win0_moved = false;
+ bool qmdwin_swap = qmdwin_on &&
+ boot_stage.rmcall_gr_qmdwin.pages;
+
+ if (qmdwin_swap || win0_on) {
epoch = ++boot_stage.rmcall_gr_submit_epoch;
- qw_bank = (epoch & 1u) %
- GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS;
- qmd_dst = boot_stage.rmcall_gr_qmdwin_va +
- (u64)qw_bank *
- GSP_PHYS_PAGE_SIZE;
- qmd_dst_mem = gsp_phys_chunk_page_va(
- &boot_stage.rmcall_gr_qmdwin,
- qw_bank);
- if (qmd_dst_mem) {
- memcpy(qmd_dst_mem, qmd,
- GSP_PHYS_GR_QMD_WORDS * 4);
- put_unaligned_le32(
- GSP_PHYS_GR_QMD_SENTINEL,
- qmd_dst_mem +
- GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
+ if (!epoch)
+ epoch = ++boot_stage.rmcall_gr_submit_epoch;
+ boot_stage.rmcall_gr_shadow_win0_used =
+ (epoch & 1u) *
+ GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
+ if (win0_on)
+ win0_moved =
+ gsp_phys_gr_win0_retry_rearm(
+ qmd, req,
+ win0_bank_base,
+ boot_stage.rmcall_gr_shadow_win0_used,
+ boot_stage.rmcall_gr_shadow_win0_used +
+ GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS);
+ win0_bank_base =
+ boot_stage.rmcall_gr_shadow_win0_used;
+ shadow_slot_limit =
+ win0_bank_base +
+ GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS;
}
- words = gsp_phys_rmcall_build_gr_launch(
- methods, req, qmd, qmd_dst,
- ch_sem_va, want_ch);
- memcpy(gsp_phys_chunk_page_va(pb, 0),
- methods, words * 4);
- pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + PB rebuilt\n",
- attempts, qw_bank, qmd_dst);
- } else {
- pr_info("vgsp-gr: outcome retry %u: release words zeroed (qmdwin off)\n",
- attempts);
+ if (qmdwin_swap) {
+ qw_bank = (epoch & 1u) %
+ GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS;
+ qmd_dst =
+ boot_stage.rmcall_gr_qmdwin_va +
+ (u64)qw_bank *
+ GSP_PHYS_PAGE_SIZE;
+ qmd_dst_mem =
+ gsp_phys_chunk_page_va(
+ &boot_stage.rmcall_gr_qmdwin,
+ qw_bank);
+ if (qmd_dst_mem) {
+ memcpy(qmd_dst_mem, qmd,
+ GSP_PHYS_GR_QMD_WORDS * 4);
+ put_unaligned_le32(
+ GSP_PHYS_GR_QMD_SENTINEL,
+ qmd_dst_mem +
+ GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
+ }
+ }
+ if (qmdwin_swap || win0_moved) {
+ words = gsp_phys_rmcall_build_gr_launch(
+ methods, req, qmd,
+ qmd_dst, ch_sem_va,
+ want_ch);
+ memcpy(gsp_phys_chunk_page_va(pb, 0),
+ methods, words * 4);
+ }
+ if (qmdwin_swap)
+ pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n",
+ attempts, qw_bank,
+ qmd_dst, win0_bank_base,
+ win0_moved ?
+ " (shadow pages re-copied)" :
+ "");
+ else if (win0_moved)
+ pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n",
+ attempts, win0_bank_base);
+ else
+ pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n",
+ attempts);
}
gsp_phys_gr_fill_bounce(req, bounce_csum_in);
mutex_lock(&gsp_phys_rm_lock);
@@ -18871,6 +19153,17 @@ gr_page_unmapped:
if (wrote == 0) {
bool restored = false;
+ /*
+ * C6 (GAP-2 A1): the no-write accept. Unconditional
+ * by design - it is the rare, load-bearing event
+ * (zero engine writes still accepted as success),
+ * so it must be visible even when no release word
+ * needed restoring.
+ */
+ pr_info("vgsp-gr: NOWRITE h=0x%x push=0x%llx n_data=%u wrote=0 qmd_rel=0x%x\n",
+ req->h_object, req->push_va, req->n_data,
+ want_qmd);
+
if (qmd_rel_hpa) {
u8 *rp = (u8 *)__va(qmd_rel_hpa) +
qmd_rel_off;
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index b1945ee..95bd627 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -974,6 +974,53 @@ int gsp_phys_rmcall_gr_nowrite_retry_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value);
/*
+ * F2 fast re-arm (gsp_phys_gr_retry_fast; default 1). r198: the first
+ * forwarded-launch timeout costs 2x5 s before the fresh-entry ladder runs,
+ * yet the m155 extended re-arm (lane-enable replay + GPFIFO/TSG schedule +
+ * doorbell) plus one fresh ring entry recovers most of these launches. With
+ * this on, the first timeout immediately runs that re-arm + a fresh entry
+ * and re-polls with a short window; only if that still fails does the
+ * historical timeout ladder continue. Set to 0 for today's exact behavior.
+ */
+static int gsp_phys_gr_retry_fast_param = 1;
+module_param_named(gsp_phys_gr_retry_fast,
+ gsp_phys_gr_retry_fast_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_retry_fast,
+ "On the first GR launch timeout, run the extended re-arm + a fresh ring entry immediately and re-poll briefly (default 1; 0 = legacy ladder timing)");
+
+int gsp_phys_gr_retry_fast_value(void)
+{
+ return gsp_phys_gr_retry_fast_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gr_retry_fast_value);
+
+/*
+ * F2 first-poll window (gsp_phys_gr_first_poll_ms; default 2500, clamped to
+ * [1000, 30000]). With gsp_phys_gr_retry_fast=1 the FIRST poll window of a
+ * forwarded GR launch (before any re-arm) uses this instead of the fixed 5 s,
+ * so the fast re-arm path above starts earlier; with retry_fast=0 the launch
+ * keeps today's exact behavior and this knob is not read. The later ladder
+ * windows keep their own constants.
+ */
+static unsigned int gsp_phys_gr_first_poll_ms_param = 2500u;
+module_param_named(gsp_phys_gr_first_poll_ms,
+ gsp_phys_gr_first_poll_ms_param, uint, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_first_poll_ms,
+ "First GR launch poll window in ms, clamped [1000,30000], read when gsp_phys_gr_retry_fast=1 (default 2500)");
+
+unsigned int gsp_phys_gr_first_poll_ms_value(void)
+{
+ unsigned int v = gsp_phys_gr_first_poll_ms_param;
+
+ if (v < 1000u)
+ v = 1000u;
+ if (v > 30000u)
+ v = 30000u;
+ return v;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value);
+
+/*
* m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When
* set, the forwarded GR launch copies the QMD's program/param pages into the
* window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index a89493b..c058469 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -375,6 +375,14 @@ int gsp_phys_rmcall_gr_skedrec_value(void);
* wrong-place engine writes (default 1 = enabled). */
int gsp_phys_rmcall_gr_nowrite_retry_value(void);
+/* F2: on the first GR launch timeout, run the extended re-arm + a fresh ring
+ * entry immediately and re-poll briefly (default 1; 0 = legacy ladder). */
+int gsp_phys_gr_retry_fast_value(void);
+
+/* F2: first GR launch poll window in ms, clamped [1000,30000], read while
+ * gsp_phys_gr_retry_fast=1 (default 2500). */
+unsigned int gsp_phys_gr_first_poll_ms_value(void);
+
/* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */
int gsp_phys_rmcall_gr_inst_sys_value(void);
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 7b12ecc..01f94cf 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -805,6 +805,7 @@ struct sema_svc {
* deliverable exact slot */
u64 counter_rel_applied; /* releases to registered counters */
u64 counter_rel_dbg; /* rel-apply diagnostic budget */
+ u64 crel_entry_dbg; /* C1: rel-apply entry budget, 512/session */
u64 counter_noslot_dbg; /* "no slot registered" budget,
* separate so it cannot hide the
* written/skip lines */
@@ -831,6 +832,8 @@ struct sema_svc {
u64 vidmem_writes; /* semaphores written in vidmem */
u64 gpfifo_log; /* channel-state log budget */
u64 gpfifo_entry_log; /* per-entry log budget */
+ u64 walk_dbg; /* C4: GR put/walk witness budget */
+ u64 gr_rel_auth_log; /* F1: GR-REL-AUTHOR budget, 256/session */
u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
u32 doorbell_token; /* last token written */
u32 doorbell_log; /* token log budget */
@@ -1380,6 +1383,17 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu,
static void svc_opdef_reset_all(void);
static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx);
static u32 svc_opdef_log; /* m176: forward decl for the session flush */
+/*
+ * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10):
+ * small, reset per session where they are service-thread state, so a later
+ * client's strand is never blacked out by the first client's spend.
+ */
+static u32 svc_relx_log; /* C2: deferred-launch apply entry witness */
+static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */
+static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */
+static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */
+static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */
+static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */
void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
{
@@ -3415,6 +3429,13 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
static void svc_pending_add(u64 va, u32 payload, u32 op);
+/*
+ * F1: GR release-word authorship checks. Defined next to the svc_fwd_req
+ * slot; forward-declared because the sweep writer runs earlier in the file.
+ */
+static bool svc_gr_rel_match(u64 va, u32 *payload);
+static bool svc_gr_rel_inflight(u64 va, u32 *payload);
+
static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val)
{
u64 addr = 0, old_addr;
@@ -4528,6 +4549,17 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
{
u32 i, n;
+ /*
+ * C9 (GAP-2 A4): a capture over a still-valid candidate (never
+ * offered) destroys an unforwarded launch - the single-slot
+ * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined
+ * later in the file and this site may not reference it).
+ */
+ if (svc_gr_cand.valid && svc_cand_ovw_log < 64) {
+ svc_cand_ovw_log++;
+ pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n",
+ svc_gr_cand.h_object, svc_gr_cand.push_va, push_va);
+ }
memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
svc_gr_cand.valid = true;
svc_gr_cand.h_object = ch->h_object;
@@ -5668,6 +5700,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
bool skip;
int i;
u32 dbg = 0;
+ u64 rel_pa = 0;
+ u64 cnt_pa = 0;
if (!h || !payload) {
/*
@@ -5705,12 +5739,23 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
* FORENSICS-grv7-counter F1). Fall back to counter_pa
* when the release slot could not be resolved.
*/
- pa = svc.chan[i].release_pa ?
- svc.chan[i].release_pa : svc.chan[i].counter_pa;
+ rel_pa = svc.chan[i].release_pa;
+ cnt_pa = svc.chan[i].counter_pa;
+ pa = rel_pa ? rel_pa : cnt_pa;
break;
}
}
spin_unlock_irqrestore(&svc.chan_lock, flags);
+ /*
+ * C1 (GAP-1 H3/H4): was the apply entered, and with which slot?
+ * Budget 512/session (reset in svc_session_flush); the per-branch
+ * prints below stay on their own counter_rel_dbg budget.
+ */
+ if (gsp_sema_dbg && svc.crel_entry_dbg < 512) {
+ svc.crel_entry_dbg++;
+ pr_info("sema-diag: crel-entry h=0x%x va=0x%llx pay=%u inc=%d rel_pa=0x%llx cnt_pa=0x%llx\n",
+ h, sem_va, payload, inc ? 1 : 0, rel_pa, cnt_pa);
+ }
if (!pa) {
/*
* Pool (UVM tracking) semaphores have no counter slot. The
@@ -6314,6 +6359,22 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
u32 payload, u32 cur, bool pending,
bool inc, bool inc_apply)
{
+ /*
+ * P4 (GAP-1 H3): the exact-slot counter delivery runs before every
+ * path that can early-return, so a GR completion-counter release can
+ * never be skipped by the generic write's outcome. The generic
+ * write/learn tail below is skipped entirely for counter VAs: an
+ * increment release is not idempotent, so it must apply exactly once
+ * (the early return here is that exactly-once edge).
+ */
+ if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
+ if (pending || inc_apply)
+ svc.pushes_pending++;
+ svc_counter_learn(vgpu, ch->h_object, sem_va);
+ svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+ inc && inc_apply);
+ return false;
+ }
if (pending || inc_apply) {
svc.pushes_pending++;
if (inc)
@@ -6325,6 +6386,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
* generic translation can be a stale content-scanned page;
* those releases go through the injected-pin-backed exact slot
* (svc_counter_learn / svc_counter_release_apply) below.
+ * Counter VAs are handled by the early block above; the check
+ * stays as belt-and-braces for the generic write.
*/
if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
/* handled by the exact-slot path */
@@ -6524,6 +6587,24 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
memcpy(recs[n].data, rec->data, rec->len);
n++;
}
+ /*
+ * C10 (GAP-2 A5): duplicate destinations in the newest-first
+ * inline capture decide which record wins the backend's
+ * overlay; bounded and unconditional (gsp_sema_dbg is defined
+ * later in the file and this site may not reference it).
+ */
+ if (svc_inline_recs_log < 64) {
+ u32 dup = 0, a2, b2;
+
+ for (a2 = 0; a2 < n; a2++)
+ for (b2 = a2 + 1; b2 < n; b2++)
+ if (recs[a2].dst_va ==
+ recs[b2].dst_va)
+ dup++;
+ svc_inline_recs_log++;
+ pr_info("sema-gr: inline recs n=%u dup_dst=%u\n",
+ n, dup);
+ }
if (gsp_phys_gr_forward_enabled() && trusted && !replay)
svc_gr_cand_capture(ch, st, recs, n, push, push_len,
st->push_va);
@@ -7978,6 +8059,9 @@ static void svc_session_flush(void)
* because the counters were already exhausted by the first client).
*/
svc.counter_rel_dbg = 0;
+ svc.crel_entry_dbg = 0;
+ svc.walk_dbg = 0;
+ svc.gr_rel_auth_log = 0;
svc.counter_noslot_dbg = 0;
svc.sem256_log = 0;
svc.xfer_raw_log = 0;
@@ -8051,6 +8135,13 @@ static void svc_session_flush(void)
/* m176: re-arm the opdef diagnostic budget (it was never per-session,
* so a late client's strand could be silent). */
svc_opdef_log = 0;
+ /* One-cycle diag budgets (C2/C3/C7/C8/C9/C10) are per-session too. */
+ svc_relx_log = 0;
+ svc_opdef_clr_log = 0;
+ svc_busy_defer_log = 0;
+ svc_hold_fb_log = 0;
+ svc_cand_ovw_log = 0;
+ svc_inline_recs_log = 0;
/* m176: new session generation - channels and scrubbers re-stamp on
* registration and never inherit per-session learned state. */
svc_sess_gen++;
@@ -8322,6 +8413,25 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *
}
/*
+ * F1-b: while the forwarded GR launch this release belongs to
+ * is still in flight, the sweep must not author its QMD
+ * completion word - the backend poll would break on a
+ * host-authored ack (r198: qmd_rel satisfied ~6 ms after the
+ * doorbell, zero engine writes). Defer: nothing is written or
+ * consumed here; the release stays in the guest page and later
+ * sweep passes re-see it, while the hold/fallback path owns a
+ * failed hardware copy.
+ */
+ if (gsp_sema_gr_swrel_gate && svc_gr_rel_inflight(va, NULL)) {
+ if (svc.gr_rel_auth_log < 256) {
+ svc.gr_rel_auth_log++;
+ pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x DEFER\n",
+ va, payload, 0u);
+ }
+ continue;
+ }
+
+ /*
* A release at page offset 0x10 with payload 1 in front of a
* channel's first push identifies the page as that channel's
* push-buffer base (see svc_pb_add).
@@ -8330,6 +8440,18 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *
(lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
svc_pb_add(pa_page, lo & 0xfff);
+ /*
+ * F1-a: tag software authorship of the forwarded launch's
+ * release word (gate off, or the request already resolved and
+ * the forward progress rule still allows the write).
+ */
+ if (svc_gr_rel_match(va, NULL) &&
+ svc.gr_rel_auth_log < 256) {
+ svc.gr_rel_auth_log++;
+ pr_info("sema-diag: GR-REL-AUTHOR src=sweep va=0x%llx pay=%u h=0x%x\n",
+ va, payload, 0u);
+ }
+
if (svc_sem_write(vgpu, 0, va, payload) == 0) {
svc.releases_applied++;
svc.post_hint_until = jiffies + 10 * HZ;
@@ -10066,6 +10188,18 @@ bool gsp_sema_counter_derive = true;
unsigned gsp_sema_gr_grace_ms = 8000;
/*
+ * F1-b: GR release-word authorship gate (default 1). While a forwarded GR
+ * launch is in flight, the RAM-content sweep must not author the very QMD
+ * completion release the backend poll is waiting for: a sweep write could
+ * satisfy the poll before the engine retires the QMD (false completion,
+ * silent corruption). With the gate on, such a write is deferred - the
+ * release stays in RAM and later sweep passes re-see it; when the hardware
+ * copy resolves, the hold/fallback path applies it. 0 restores the
+ * unconditional sweep apply for an A/B.
+ */
+bool gsp_sema_gr_swrel_gate = true;
+
+/*
* m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode
* saw QMD words but built no launch candidate for its channel (a replay-
* suppressed capture, a partial/split QMD, or a push page the pin stream has
@@ -11363,6 +11497,15 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
ch->counter_last = cur;
return;
}
+ /*
+ * C5 (GAP-1 H4 secondary): the put-based publish is about to advance
+ * the same word the decoded releases target; a pre-advance here can
+ * make a later absolute release a silent skip. Witness every advance.
+ */
+ if (gsp_sema_dbg)
+ pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n",
+ ch->h_object, ch->counter_pa, val, cur,
+ ch->counter_last);
if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS,
&val, sizeof(val)))
return;
@@ -11716,6 +11859,34 @@ static DEFINE_MUTEX(svc_fwd_lock);
static bool svc_fwd_paused;
/*
+ * F1 helpers: does a software release write target the release word of the
+ * forwarded GR launch the request slot holds? The slot is written only by the
+ * service thread (this code also runs there) and is never reused while the
+ * hold is active, so no lock is needed - same convention as the existing
+ * svc_fwd_hold_gr_release()/svc_fwd_hold_expired() reads. svc_gr_rel_match()
+ * covers the just-resolved hold (done may already be true); the inflight
+ * variant additionally requires the SVC_FWD_INFLIGHT bit and an unresolved
+ * work item, i.e. the hardware copy the poll is waiting on is still running.
+ */
+static bool svc_gr_rel_match(u64 va, u32 *payload)
+{
+ if (!va || !svc_fwd_req.gr || svc_fwd_req.gr_rel_va != va)
+ return false;
+ if (payload)
+ *payload = svc_fwd_req.gr_rel_payload;
+ return true;
+}
+
+static bool svc_gr_rel_inflight(u64 va, u32 *payload)
+{
+ if (!svc_gr_rel_match(va, payload))
+ return false;
+ if (!test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state))
+ return false;
+ return !smp_load_acquire(&svc_fwd_req.done);
+}
+
+/*
* Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve
* the page-aligned GPA through the guest-RAM backing file; VID operands
* resolve the shadow byte offset through the vmalloc backing of BAR1 (the
@@ -11965,8 +12136,19 @@ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu,
bool pending = r->op != 0;
bool inc = DMA_SEM_REDUCTION_EN(r->dma) &&
DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC;
+ int rd = svc_sem_read(vgpu, r->va, &cur);
- if (svc_sem_read(vgpu, r->va, &cur)) {
+ /*
+ * C2 (GAP-1 H2/H3): prove the opdef apply handed the record to the
+ * helper and what the software read of the release word returned.
+ */
+ if (gsp_sema_dbg && svc_relx_log < 128) {
+ svc_relx_log++;
+ pr_info("sema-diag: relx kind=%u op=%u inc_claim=%u va=0x%llx pay=%u rd=%d\n",
+ (unsigned)r->kind, (unsigned)r->op,
+ (unsigned)r->inc_claimed, r->va, r->payload, rd);
+ }
+ if (rd) {
svc_pending_add(r->va, r->payload, inc ? 1 : 0);
svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending,
inc, r->inc_claimed);
@@ -12103,8 +12285,31 @@ static struct svc_opdef_ent *svc_opdef_alloc(struct nv_vgpu_device *vgpu,
return NULL;
}
-static void svc_opdef_clear(struct svc_opdef_ent *e)
+/* C3 caller tags for the opdef-clear witness (H2: which path cleared). */
+#define SVC_OPDEF_CLR_APPLY 0u
+#define SVC_OPDEF_CLR_FINISH 1u
+#define SVC_OPDEF_CLR_FLUSH 2u
+#define SVC_OPDEF_CLR_DROP 3u
+#define SVC_OPDEF_CLR_STALE 4u
+#define SVC_OPDEF_CLR_COUNTED 5u
+#define SVC_OPDEF_CLR_SWEEP 6u
+#define SVC_OPDEF_CLR_STOP 7u
+
+static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller)
{
+ /*
+ * C3 (GAP-1 H2): every clear is witnessed with its caller and the
+ * slot contents, so a "silent" clear (n=0, or a kind the apply
+ * switch ignores) is distinguishable from a delivery.
+ */
+ if (gsp_sema_dbg && e->active && svc_opdef_clr_log < 128) {
+ svc_opdef_clr_log++;
+ pr_info("sema-diag: opdef-clear caller=%u h=0x%x entry=%u n=%u kind0=%u va0=0x%llx pay0=%u\n",
+ caller, e->h_object, e->entry_idx, (unsigned)e->n,
+ e->n ? (unsigned)e->rel[0].kind : 0u,
+ e->n ? e->rel[0].va : 0ull,
+ e->n ? e->rel[0].payload : 0u);
+ }
memset(e, 0, sizeof(*e));
}
@@ -12142,7 +12347,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu,
ch->h_object, entry_idx, e->n);
}
}
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_APPLY);
}
/*
@@ -12174,7 +12379,7 @@ static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx)
ch->h_object, idx, e->n);
}
}
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_FINISH);
return;
}
}
@@ -12286,7 +12491,7 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
ch->h_object, entry_idx, e->n);
}
svc.opdef_expired++;
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_FLUSH);
}
/*
@@ -12338,7 +12543,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
(u32)(e->n - delivered), e->n);
}
}
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_DROP);
}
/*
@@ -12359,7 +12564,7 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu,
continue;
if (e->n)
svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n);
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_STALE);
}
}
@@ -12373,7 +12578,7 @@ static void svc_opdef_drop_counted(struct svc_opdef_ent *e)
pr_info_ratelimited("sema-opdef: LOSS entry h=0x%x idx=%u dropped with %u undelivered release(s)\n",
e->h_object, e->entry_idx, e->n);
}
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_COUNTED);
}
static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu,
@@ -12459,7 +12664,7 @@ static void svc_opdef_expire_sweep(void)
pr_info_ratelimited("sema-opdef: SWEEP-LOSS h=0x%x entry=%u n=%u owner channel gone (releases dropped)\n",
e->h_object, e->entry_idx, e->n);
}
- svc_opdef_clear(e);
+ svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP);
}
}
@@ -12500,6 +12705,16 @@ static void svc_gr_sw_release(struct nv_vgpu_device *vgpu, u32 h, u64 va,
return;
if (!svc_sem_read(vgpu, va, &cur) && cur >= payload)
return;
+ /*
+ * F1-a: this is the hold/fallback authorship of the forwarded launch's
+ * release word (src="hold"), budgeted with the sweep tag so one
+ * 256-line session window covers both authors.
+ */
+ if (svc_gr_rel_match(va, NULL) && svc.gr_rel_auth_log < 256) {
+ svc.gr_rel_auth_log++;
+ pr_info("sema-diag: GR-REL-AUTHOR src=hold va=0x%llx pay=%u h=0x%x\n",
+ va, payload, h);
+ }
if (!svc_sem_write(vgpu, h, va, payload)) {
svc.fwd_gr_sw_rel++;
pr_info("vgsp-work: GR launch release software-applied h=0x%x va=0x%llx pay=0x%x (no successful hardware copy; forward-only)\n",
@@ -12849,6 +13064,22 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
* done && hw_rc == -ETIMEDOUT; abort/cancel paths never write
* guest memory).
*/
+ /*
+ * C7 (GAP-2 A1): the false-completion edge - a hold whose
+ * hardware copy failed/timed out still software-applies the
+ * release set here.
+ */
+ if (gsp_sema_dbg && svc_hold_fb_log < 64) {
+ svc_hold_fb_log++;
+ pr_info("sema-diag: hold-fallback h=0x%x gr=%d done=%d hw_rc=%d nrel=%u rel0=0x%llx/%u -> sw apply\n",
+ ch->h_object, ch->engine_type == 0 ? 1 : 0,
+ done ? 1 : 0, svc_fwd_req.hw_rc,
+ (unsigned)svc_fwd_hold.nrel,
+ svc_fwd_hold.nrel ?
+ svc_fwd_hold.rel[0].va : 0ull,
+ svc_fwd_hold.nrel ?
+ svc_fwd_hold.rel[0].payload : 0u);
+ }
if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
svc_fwd_hold.relay_armed)
svc_fwd_fault_deliver(hv, ch);
@@ -13846,6 +14077,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
* in every launch and starved the real operands under the old
* window. Skip them as data candidates; they are
* shadowed/rewritten through gr_pages/Tier 2, not this path.
+ * P6: the QMD program pair (word 48/49) is launch-internal in
+ * exactly the same way and is skipped with them.
*/
if (svc_fwd_req.gr_param_va &&
(base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
@@ -13859,7 +14092,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
&svc_fwd_req.gr_pages[i];
if (pp->qmd_word >= 32u &&
- pp->qmd_word <= 47u &&
+ pp->qmd_word <= 49u &&
(base & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
(pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
launch_internal = true;
@@ -14846,6 +15079,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
int rrc;
u32 put = 0, get = 0, guard = 0;
u32 get_before;
+ u32 put_prev_wit;
u64 applied_before;
bool held = false;
@@ -15066,6 +15300,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
}
+ put_prev_wit = ch->put_prev;
ch->put_prev = put;
/*
@@ -15103,6 +15338,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
get_before = ch->last_get;
applied_before = svc.releases_applied;
+ /*
+ * C4 (GAP-1 H1): the GR walk witness - does put advance, and
+ * does last_get reach it? Budget 512/session; fires on a put
+ * change or while last_get is in the stalled 80..100 band.
+ */
+ if (gsp_sema_dbg && svc.walk_dbg < 512 &&
+ (put != put_prev_wit ||
+ ((u32)ch->last_get >= 80 && (u32)ch->last_get <= 100))) {
+ svc.walk_dbg++;
+ pr_info("sema-diag: walk h=0x%x put=%u get=%u last=%u prev=%u\n",
+ ch->h_object, put, get, (u32)ch->last_get,
+ put_prev_wit);
+ }
+
while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
u8 entry[SVC_GPFIFO_ENTRY_SIZE];
u64 va, idx = ch->last_get;
@@ -15693,8 +15942,56 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
fwd_guest_queued =
svc_fwd_request(vgpu,
ch);
- if (fwd_guest_queued)
+ if (fwd_guest_queued) {
svc.fwd_gr_defer++;
+ } else if (ch->gr_hold_idx != (u32)idx ||
+ time_before(jiffies,
+ ch->gr_hold_until)) {
+ u32 hold_ms = gsp_sema_gr_grace_ms ?
+ gsp_sema_gr_grace_ms : 8000;
+
+ /*
+ * P2 (GAP-2 A3): the wait
+ * expired without queueing.
+ * Do NOT fall through to the
+ * software-release consume -
+ * the launch was never
+ * submitted, so authoring
+ * its release would be a
+ * false completion. Hold
+ * exactly like the
+ * data-stalled path
+ * (m150/m156); the one-shot
+ * window lets the historical
+ * consume resume after it
+ * expires.
+ */
+ ch->gr_rehold_idx = (u32)idx;
+ ch->gr_hold_idx = (u32)idx;
+ ch->gr_hold_until = jiffies +
+ msecs_to_jiffies(hold_ms);
+ svc.fwd_gr_defer++;
+ if (svc_busy_defer_log < 64) {
+ svc_busy_defer_log++;
+ pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> hold %u ms\n",
+ ch->h_object,
+ (unsigned long long)idx,
+ hold_ms);
+ }
+ kfree(pb);
+ break;
+ } else if (svc_busy_defer_log < 64) {
+ /*
+ * C8 (GAP-2 A3): the bounded
+ * hold window is over; the
+ * historical consume with the
+ * software release resumes.
+ */
+ svc_busy_defer_log++;
+ pr_info("vgsp-work: GR busy-defer timeout h=0x%x idx=%llu -> consume w/ sw release\n",
+ ch->h_object,
+ (unsigned long long)idx);
+ }
}
/*
* m150: the GR fill held this
@@ -17832,7 +18129,7 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu)
/* Held-operand release slots of this guest cannot be delivered. */
for (i = 0; i < SVC_OPDEF_SLOTS; i++)
if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu)
- svc_opdef_clear(&svc_opdef[i]);
+ svc_opdef_clear(&svc_opdef[i], SVC_OPDEF_CLR_STOP);
/* Free the per-guest session and scrubber records. */
spin_lock_irqsave(&svc_guest_lock, flags);
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index b8c4dfb..e79064c 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -285,4 +285,15 @@ extern bool gsp_sema_pool_hold_fresh;
*/
extern bool gsp_sema_copy_dst_pin_guard;
+/*
+ * F1-b: authorship gate for the GR completion release the forwarded launch's
+ * poll waits on. When on (default), a RAM-sweep release whose VA equals the
+ * in-flight forward request's gr_rel_va is deferred (skipped, not applied)
+ * while the hardware copy is still in flight, so the poll can only break on
+ * an engine-authored word; the deferred release is re-seen by later sweep
+ * passes and the hold/fallback paths apply it if the hardware copy fails.
+ * 0 restores the unconditional sweep apply.
+ */
+extern bool gsp_sema_gr_swrel_gate;
+
#endif /* _GSP_SEMA_SERVICE_H_ */
diff --git a/kernel/main.c b/kernel/main.c
index d2dc57a..60bab26 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2771,6 +2771,16 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644);
MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)");
/*
+ * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a
+ * RAM-sweep release whose VA is that launch's QMD completion word, so the
+ * backend poll can only break on an engine-authored ack. Deferred releases
+ * are re-seen by later sweep passes; the hold/fallback path applies them when
+ * the hardware copy fails. 0 restores the unconditional sweep apply.
+ */
+module_param_named(gsp_sema_gr_swrel_gate, gsp_sema_gr_swrel_gate, bool, 0644);
+MODULE_PARM_DESC(gsp_sema_gr_swrel_gate, "Defer a sweep-authored in-flight GR release word (default 1; 0 = unconditional sweep apply)");
+
+/*
* m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode
* saw QMD words but built no launch candidate (replay-suppressed capture,
* partial/split QMD, or an unreadable push page). The entry stays unconsumed
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 4bfaaf8..f645f76 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -1627,15 +1627,126 @@ def main():
pool_reemit_next = 0.0
_want_state = {"mtime": None, "done": set()}
loop = 0
+
+ def _send(lines):
+ """Flush control/pin lines to the host, connecting the vsock if needed.
+
+ Returns True when the lines were sent (or there were none to send)."""
+ nonlocal sock
+ if not lines:
+ return True
+ payload = ''.join(lines).encode()
+ if sock is None:
+ try:
+ sock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
+ sock.settimeout(5)
+ sock.connect((CID, PORT))
+ sock.settimeout(None)
+ if last_pid is not None:
+ sock.sendall(("#pid %d\n" % last_pid).encode())
+ except OSError:
+ if sock:
+ sock.close()
+ sock = None
+ return False
+ try:
+ sock.sendall(payload)
+ except OSError:
+ sock.close()
+ sock = None
+ return False
+ return True
+
while True:
pid = cdt_pid()
loop += 1
+ lines = []
+ # Pid-independent work first: the scrubber words are kernel objects
+ # that exist before any client, and the pool/demand re-asserts heal
+ # the host pin table between clients. Gating these on a live pid kept
+ # the host unregistered/unhealed whenever no client was present.
+ if kc_ok and time.time() >= demand_next:
+ demand_next = time.time() + 1.0
+ try:
+ st_d = os.stat("/tmp/guest_want_vas.txt")
+ if st_d.st_mtime != _want_state.get("mtime"):
+ with open("/tmp/guest_want_vas.txt") as fw:
+ wvas = [int(t, 16) for t in fw.read().split()
+ if t.startswith("0x")]
+ _want_state["mtime"] = st_d.st_mtime
+ for wv in wvas[-64:]:
+ if wv in _want_state["done"]:
+ continue
+ got = []
+ if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI:
+ # m165 (D-F1a): pool/waiter VAs have no method
+ # stream content and the CE offset math does not
+ # apply; route them to the structural tracker-sem
+ # walk instead. m168: the tracker sems are
+ # long-lived objects (allocated once) whose page
+ # pointer follows the pool across clients, so try
+ # every sem we have ever resolved a chain for -
+ # seen_sems alone can miss the live generation.
+ _psems = set(seen_sems) | set(KC_PB_STATE.keys()) \
+ | set(resolved_kc_pb)
+ try:
+ ok = resolve_pool_va(kc, wv, got,
+ _psems)
+ except Exception:
+ ok = False
+ got = []
+ dbg('demand-pool-va: 0x%x %s'
+ % (wv, 'ok' if ok else 'pending'))
+ if ok and got:
+ pool_pages.setdefault(wv & ~0xFFF, True)
+ else:
+ try:
+ ok = resolve_va_by_content(kc, wv, got)
+ except Exception:
+ ok = False
+ got = []
+ if ok and got:
+ lines.extend(got)
+ _want_state["done"].add(wv)
+ except OSError:
+ pass
+ # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S.
+ # The pool page is boot-stable (UVM frees it only at GPU deinit) but
+ # the host pin table is wiped at every session flush, so a one-shot
+ # pin leaves later clients starving until the slow structural walks
+ # answer; the periodic re-assert heals the wipe within seconds (the
+ # same argument as the CE mem sweep above).
+ if kc_ok and pool_pages and time.time() >= pool_reemit_next:
+ pool_reemit_next = time.time() + KC_POOL_REEMIT_S
+ for pv in list(pool_pages):
+ got = []
+ try:
+ ok = resolve_pool_va(kc, pv, got, seen_sems)
+ except Exception:
+ ok = False
+ got = []
+ if ok and got:
+ lines.extend(got)
+ # PMA scrubber buffer discovery (throttled): resolve the finish/host-
+ # semaphore word PAs and stream them for host registration. This runs
+ # with or without a client (the words are kernel objects).
+ if kc_ok and time.time() >= scrub_next:
+ scrub_next = time.time() + 5.0
+ try:
+ scrub_scan(kc, lines)
+ except Exception as e:
+ dbg('scrub scan failed: %s' % e)
if pid is None:
- last_pid = None
+ if last_pid is not None:
+ last_pid = None
+ # Notify the host so it drops its per-pair dedupe map; the
+ # next client's pairs must be injected unconditionally.
+ lines.append('#nopid\n')
seen = {}
seen_sems = set()
seen_pbs = set()
seen_chs = set()
+ _send(lines)
time.sleep(INTERVAL)
continue
if pid != last_pid:
@@ -1703,85 +1814,6 @@ def main():
cur[0x20400f000] = cpa
new = [(va, pa) for va, pa in cur.items() if seen.get(va) != pa]
pending_cur = cur
- lines = []
- # Demand pins (host -> this scanner): the host appends the CE
- # push-buffer VAs of GPFIFO entries it cannot read to
- # /tmp/guest_want_vas.txt (written via the guest agent by
- # pin_gr_stream); resolve each by content against the
- # nv_vm_map_pages areas and emit the pair. This is the
- # event-independent hole filler for the stall class where the
- # one-shot #sem kprobe event was lost and the object-chain
- # resolver never learns the client.
- if kc_ok and time.time() >= demand_next:
- demand_next = time.time() + 1.0
- try:
- st_d = os.stat("/tmp/guest_want_vas.txt")
- if st_d.st_mtime != _want_state.get("mtime"):
- with open("/tmp/guest_want_vas.txt") as fw:
- wvas = [int(t, 16) for t in fw.read().split()
- if t.startswith("0x")]
- _want_state["mtime"] = st_d.st_mtime
- for wv in wvas[-64:]:
- if wv in _want_state["done"]:
- continue
- got = []
- if POOL_SEM_LO <= (wv & ~0xFFF) < POOL_SEM_HI:
- # m165 (D-F1a): pool/waiter VAs have no method
- # stream content and the CE offset math does not
- # apply; route them to the structural tracker-sem
- # walk instead. m168: the tracker sems are
- # long-lived objects (allocated once) whose page
- # pointer follows the pool across clients, so try
- # every sem we have ever resolved a chain for -
- # seen_sems alone can miss the live generation.
- _psems = set(seen_sems) | set(KC_PB_STATE.keys()) \
- | set(resolved_kc_pb)
- try:
- ok = resolve_pool_va(kc, wv, got,
- _psems)
- except Exception:
- ok = False
- got = []
- dbg('demand-pool-va: 0x%x %s'
- % (wv, 'ok' if ok else 'pending'))
- if ok and got:
- pool_pages.setdefault(wv & ~0xFFF, True)
- else:
- try:
- ok = resolve_va_by_content(kc, wv, got)
- except Exception:
- ok = False
- got = []
- if ok and got:
- lines.extend(got)
- _want_state["done"].add(wv)
- except OSError:
- pass
- # m165 (D-F1a): re-assert the pool page pair every KC_POOL_REEMIT_S.
- # The pool page is boot-stable (UVM frees it only at GPU deinit) but
- # the host pin table is wiped at every session flush, so a one-shot
- # pin leaves later clients starving until the slow structural walks
- # answer; the periodic re-assert heals the wipe within seconds (the
- # same argument as the CE mem sweep above).
- if kc_ok and pool_pages and time.time() >= pool_reemit_next:
- pool_reemit_next = time.time() + KC_POOL_REEMIT_S
- for pv in list(pool_pages):
- got = []
- try:
- ok = resolve_pool_va(kc, pv, got, seen_sems)
- except Exception:
- ok = False
- got = []
- if ok and got:
- lines.extend(got)
- # PMA scrubber buffer discovery (throttled): resolve the finish/host-
- # semaphore word PAs and stream them for host registration.
- if kc_ok and time.time() >= scrub_next:
- scrub_next = time.time() + 5.0
- try:
- scrub_scan(kc, lines)
- except Exception as e:
- dbg('scrub scan failed: %s' % e)
for ctl in tracer.drain():
parts = ctl.split()
if ctl.startswith("#sem") and len(parts) >= 2:
@@ -2017,27 +2049,8 @@ def main():
lines.append('0x%x,0x%x,%d\n' % (va0, pa0, n))
va0, pa0, n = va, pa, 1
lines.append('0x%x,0x%x,%d\n' % (va0, pa0, n))
- payload = ''.join(lines).encode()
- if sock is None:
- try:
- sock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
- sock.settimeout(5)
- sock.connect((CID, PORT))
- sock.settimeout(None)
- if last_pid is not None:
- sock.sendall(("#pid %d\n" % last_pid).encode())
- except OSError:
- if sock:
- sock.close()
- sock = None
- time.sleep(INTERVAL)
- continue
- try:
- sock.sendall(payload)
+ if _send(lines):
seen = pending_cur
- except OSError:
- sock.close()
- sock = None
time.sleep(INTERVAL)
diff --git a/tools/guest-diag/pin_gr_stream.py b/tools/guest-diag/pin_gr_stream.py
index 0072b20..1cd3eb1 100644
--- a/tools/guest-diag/pin_gr_stream.py
+++ b/tools/guest-diag/pin_gr_stream.py
@@ -32,7 +32,8 @@ LO = os.environ.get("PINWIN_LO", "0x200000000")
HI = os.environ.get("PINWIN_HI", "0x206000000")
PAMAX = os.environ.get("PINWIN_PA_MAX", "0x180000000")
SEMSTREAM = os.environ.get("SEM_STREAM_FILE", "/tmp/opencode/sem_stream.txt")
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
+# supervisor grants a 120 s startup grace on top).
HEARTBEAT_FILE = os.environ.get("PIN_GR_HEARTBEAT", "/tmp/opencode/pin_gr_heartbeat")
@@ -48,6 +49,16 @@ def _heartbeat():
pass
+def _hb_loop():
+ """Refresh the heartbeat on a fixed cadence, independent of data-path
+ activity: recv can be legally idle for minutes between client phases and
+ the supervisor used to read that silence as a hung tool (r198 restarts at
+ 34 s staleness while injection bursts were running)."""
+ while True:
+ _heartbeat()
+ time.sleep(5)
+
+
def _mirror(line):
"""Append guest control lines (#pid/#sem) to the shared host file so the
single-QMP watcher can consume them without guest-agent round trips."""
@@ -67,6 +78,17 @@ def vexec(cmd, timeout=40):
return ""
+def _scanner_alive():
+ """True when the in-guest scanner process still exists.
+
+ install_and_start() pkill's and restarts the scanner, which resets every
+ in-guest latch; probing first avoids killing a healthy scanner that is
+ merely silent because no client exists yet (r198 pre-client churn)."""
+ out = vexec("sudo sh -c 'kill -0 $(cat /tmp/guest_pin_stream.pid) "
+ "2>/dev/null && echo ALIVE'")
+ return "ALIVE" in (out or "")
+
+
def _start_cmd():
with open(GUEST_SCRIPT) as f:
b64 = subprocess.run(["base64", "-w0"], stdin=f,
@@ -187,11 +209,18 @@ def _demand_loop():
_DEMAND_STARTED = False
+_HB_STARTED = False
def main():
- global _DEMAND_STARTED
+ global _DEMAND_STARTED, _HB_STARTED
_heartbeat()
+ if not _HB_STARTED:
+ _HB_STARTED = True
+ try:
+ threading.Thread(target=_hb_loop, daemon=True).start()
+ except Exception as e:
+ log("WARN: heartbeat thread failed to start: %s" % e)
if not install_and_start():
log("WARN: could not start guest scanner (will retry via vsock wait)")
if not _DEMAND_STARTED:
@@ -218,9 +247,16 @@ def main():
now = time.time()
# No guest scanner connection for 30 s: it died or never started
# (a boot without it streams no pins and every client then hangs).
- # Re-run install_and_start() at most once per 15 s.
- if now - last_conn > 30.0 and now - last_reinstall >= 15.0:
+ # Probe the scanner first instead of reinstalling blindly:
+ # install_and_start() pkill's a live scanner and resets all
+ # in-guest latches, and pre-client silence is normal (r198 churn).
+ if now - last_conn > 30.0 and now - last_reinstall >= 60.0:
last_reinstall = now
+ if _scanner_alive():
+ _heartbeat()
+ log("no guest connection for %.0fs but scanner alive; "
+ "not reinstalling" % (now - last_conn))
+ continue
log("reinstall attempt: no guest connection for %.0fs"
% (now - last_conn))
try:
@@ -243,8 +279,9 @@ def main():
try:
chunk = conn.recv(1 << 20)
except socket.timeout:
- if runs:
- _flush(runs, seen)
+ # Run even with no pending runs: the periodic full
+ # re-assert inside _flush needs a quiet-period tick too.
+ if _flush(runs, seen):
runs = []
continue
except OSError:
@@ -265,6 +302,14 @@ def main():
% (m.group(1) if m else "?"))
_mirror(text)
continue
+ if text.startswith("#nopid"):
+ # Client exited: drop the per-pair dedupe map so the
+ # next client's pairs are injected unconditionally
+ # (the guest emits this on the pid -> None edge).
+ seen = {}
+ log("guest reports no client pid")
+ _mirror(text)
+ continue
if text.startswith("#sem"):
_mirror(text)
continue
@@ -303,6 +348,10 @@ def main():
pass
+_FULL_REASSERT_S = 5.0
+_full_reassert_next = 0.0
+
+
def _flush(runs, seen):
"""Inject run lines (one batched sudo pass), return number of pages pinned.
@@ -311,7 +360,18 @@ def _flush(runs, seen):
deadline. The batch writes all runs in a single shell, retrying only the
ones the kernel queue rejected (ENOSPC) individually.
"""
- todo = []
+ global _full_reassert_next
+ now = time.time()
+ # Every _FULL_REASSERT_S re-assert every pair pinned so far, not only the
+ # ones in this batch. The guest streams deltas only and the kernel session
+ # flush wipes the pin table when a client's channels close, so without
+ # this a flush while the same pid continues (a second context in one
+ # process) strands the GR-window pages for good. Writes are idempotent;
+ # contiguous pages are coalesced into one run line.
+ full = now >= _full_reassert_next
+ if full:
+ _full_reassert_next = now + _FULL_REASSERT_S
+ todo = {}
for va, pa, n in runs:
# The CE window (0x120000000..0x121000000) and the UVM semaphore
# pool page range (0x121000000..0x122000000) are re-asserted by the
@@ -324,9 +384,22 @@ def _flush(runs, seen):
# pool/tracker page (pending release starved, every client's first
# seconds).
ce_win = 0x120000000 <= va < 0x122000000
- if not ce_win and seen.get(va) == pa and n == 1:
+ if not full and not ce_win and seen.get(va) == pa and n == 1:
continue
- todo.append((va, pa, n))
+ todo[(va, pa, n)] = True
+ if full:
+ items = sorted(seen.items())
+ i = 0
+ while i < len(items):
+ va, pa = items[i]
+ n = 1
+ while (i + n < len(items)
+ and items[i + n][0] == va + n * 0x1000
+ and items[i + n][1] == pa + n * 0x1000):
+ n += 1
+ todo[(va, pa, n)] = True
+ i += n
+ todo = list(todo)
if not todo:
return 0
diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh
index 5bbab0c..f1f4eae 100755
--- a/tools/guest-diag/pin_supervisor.sh
+++ b/tools/guest-diag/pin_supervisor.sh
@@ -6,10 +6,12 @@
# first cudaMalloc.
#
# Both tools write a heartbeat file; this script checks, every 5 s, that each
-# process is alive (exact script path) and that its heartbeat mtime is < 30 s.
-# On failure it kills the tool (exact path match) and restarts it, appending
-# RESTART lines to the supervisor log. After 10 restarts in one hour for a
-# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
+# process is alive (exact script path) and that its heartbeat mtime is < 75 s.
+# A startup grace (120 s since the tool started, skipped early when the tool
+# log already shows first successful work) keeps a slow cold start from being
+# restarted. On failure it kills the tool (exact path match) and restarts it,
+# appending RESTART lines to the supervisor log. After 10 restarts in one hour
+# for a tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
# started (QEMU's QMP socket accepts a single client).
#
# Usage: pin_supervisor.sh LOGDIR TAG
@@ -30,6 +32,8 @@ PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log"
SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log"
WINDOW=3600
CAP=10
+STALE=75
+GRACE=120
mkdir -p "$LOGDIR" /tmp/opencode
log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; }
@@ -57,6 +61,7 @@ start_pin() {
touch "$PIN_HB" 2>/dev/null || true
( setsid nohup env SEMFIX_COUNTER_RETRY=10 \
python3 "$PIN" >> "$PIN_LOG" 2>&1 & )
+ pin_started=$(date +%s)
log "START pin_gr_stream"
}
@@ -70,6 +75,7 @@ start_semfix() {
touch "$SEMFIX_HB" 2>/dev/null || true
( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \
python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & )
+ semfix_started=$(date +%s)
log "START semfix_uvmt"
}
@@ -89,6 +95,8 @@ count_list() { # $1 list -> element count
restarts_pin=""
restarts_semfix=""
+pin_started=$(date +%s)
+semfix_started=$(date +%s)
restart() { # $1 label, $2 script path, $3 start function
local now list n
@@ -114,16 +122,26 @@ restart() { # $1 label, $2 script path, $3 start function
"$3"
}
-check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function
- local age
+check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function,
+ # $5 tool start TS, $6 tool log file, $7 first-work pattern
+ local age runtime
age=$(hb_age "$3")
- if tool_alive "$2" && [ "$age" -le 30 ]; then
+ if tool_alive "$2" && [ "$age" -le "$STALE" ]; then
return 0
fi
- if ! tool_alive "$2"; then
- log "$1 not running (heartbeat age ${age}s) - restarting"
- else
+ if tool_alive "$2"; then
+ # Startup grace: a cold tool can spend its first ~2 min installing the
+ # guest scanner or connecting QMP with a lagging heartbeat; only
+ # enforce staleness once the grace has passed or the log shows the
+ # tool has already done successful work.
+ runtime=$(( $(date +%s) - $5 ))
+ if [ "$runtime" -lt "$GRACE" ] && \
+ ! grep -q -- "$7" "$6" 2>/dev/null; then
+ return 0
+ fi
log "$1 heartbeat stale (${age}s) - restarting"
+ else
+ log "$1 not running (heartbeat age ${age}s) - restarting"
fi
restart "$1" "$2" "$4"
}
@@ -133,6 +151,8 @@ start_pin
start_semfix
while true; do
sleep 5
- check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin
- check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix
+ check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \
+ "$PIN_LOG" "listening on vsock"
+ check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \
+ "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching"
done
diff --git a/tools/guest-diag/semfix_uvmt.py b/tools/guest-diag/semfix_uvmt.py
index fa3ae01..812076e 100755
--- a/tools/guest-diag/semfix_uvmt.py
+++ b/tools/guest-diag/semfix_uvmt.py
@@ -53,7 +53,8 @@ SKIP_PB = os.environ.get("SEMFIX_SKIP_PB", "0") == "1"
# Seconds between counter-page pin attempts (the guest-agent exec blocks for
# its full timeout when the client saturates the guest).
COUNTER_RETRY_S = float(os.environ.get("SEMFIX_COUNTER_RETRY", "10"))
-# Liveness file checked by pin_supervisor.sh (mtime must stay < 30 s).
+# Liveness file checked by pin_supervisor.sh (mtime must stay < 75 s; the
+# supervisor grants a 120 s startup grace on top).
SEMFIX_HB_FILE = os.environ.get("SEMFIX_HEARTBEAT_FILE",
"/tmp/opencode/semfix_heartbeat")
# Every client pid ever observed. Module-level on purpose: the per-client
@@ -239,6 +240,23 @@ def cdt_pid():
return int(pids[-1]) if pids else None
+def pid_alive(pid):
+ """Probe /proc/<pid> in the guest before a blocking exec on it.
+
+ A killed client's pid stays latched in the watcher state; retrying its
+ counter pin every COUNTER_RETRY_S ran a full 25 s agent exec per attempt
+ and starved the scanner (the post-kill agent hammering). Assume alive on
+ a failed probe so a transient agent error cannot drop a live client.
+ """
+ try:
+ r = subprocess.run(
+ [VMEXEC, "exec", "test -d /proc/%d && echo ALIVE" % pid],
+ capture_output=True, text=True, timeout=15)
+ except Exception:
+ return True
+ return "ALIVE" in (r.stdout or "")
+
+
def pin_counter_page(pid, pinned_pids):
"""Pin the client's completion-counter page from its own pagemap.
@@ -282,10 +300,9 @@ def main():
last_seen_t = {}
dseen = set()
counter_pids = set()
+ counter_dead = set()
counter_next_try = {}
last_pid = None
- pending_pid = None
- pending_count = 0
# Stale-sem tracking: consecutive unreadable reads, first-unreadable time,
# and the per-sem read skip deadline once a sem has failed 3 times.
unread_count = {}
@@ -332,7 +349,6 @@ def main():
if last_pid is None:
seen_pids.add(p)
last_pid = p
- pending_pid = None
continue
if p == last_pid:
seen_pids.add(p)
@@ -343,20 +359,13 @@ def main():
log("semfix: pid %d seen before/backwards - no reset" % p)
seen_pids.add(p)
continue
- if pending_pid != p:
- pending_pid = p
- pending_count = 1
- log("semfix: pid change %d -> %d (debounce; reset if it "
- "repeats)" % (last_pid, p))
- continue
- pending_count += 1
- if pending_count < 2:
- continue
- # Same unseen pid reported on two consecutive polls: real new
- # client. seen_pids keeps old pids from ever triggering this.
+ # Any unseen pid above the current one is a real new client.
+ # Stream.poll dedupes pids against its own list, so each pid
+ # arrives in exactly one poll batch and never repeats - the old
+ # two-poll debounce therefore never fired after the first client
+ # (the post-kill counter latch bug). seen_pids still guards
+ # replays of old pids; reset immediately instead.
seen_pids.add(p)
- pending_pid = None
- pending_count = 0
stream.reset()
known_sems.clear()
seen_sem_pin.clear()
@@ -366,6 +375,7 @@ def main():
seen_ch.clear()
seen_pb.clear()
counter_pids.clear()
+ counter_dead.clear()
sem_pin_fail.clear()
sem_pin_skip_until.clear()
unread_count.clear()
@@ -382,6 +392,7 @@ def main():
# as a fallback for channel/pushbuffer discovery and to recover sems
# the scanner may have missed.
if loop % TRACE_EVERY == 0:
+ _hb()
try:
t_sems, t_pbs, t_chans = guest_trace()
except Exception:
@@ -542,6 +553,7 @@ def main():
log("semfix: pb=0x%x pushbuf miss [%s]" % (pb, diag_str(d)))
if DLOG:
+ _hb()
dmesg_releases(dseen)
# Retry incomplete push-buffer pins: gva2gpa of the kernel vmalloc
@@ -559,20 +571,32 @@ def main():
# block for its whole timeout under client load, so retry on a slow
# cadence: hammering the agent here starves it and wedges the run.
if last_pid is None and loop % TRACE_EVERY == 0:
+ _hb()
last_pid = cdt_pid()
_hb()
if last_pid:
seen_pids.add(last_pid)
log("counter: client pid %d (agent)" % last_pid)
- if last_pid and last_pid not in counter_pids:
+ if last_pid and last_pid not in counter_pids \
+ and last_pid not in counter_dead:
now = time.time()
if now >= counter_next_try.get(last_pid, 0):
- try:
- pin_counter_page(last_pid, counter_pids)
- except Exception as e:
- log("counter: pin failed: %s" % e)
_hb()
- counter_next_try[last_pid] = now + COUNTER_RETRY_S
+ if not pid_alive(last_pid):
+ # Killed clients stay latched as last_pid forever; probe
+ # first so the 10 s cadence does not block the agent for a
+ # full 25 s exec on every retry (the post-kill hammering
+ # that starved the scanner). A new client's #pid resets
+ # counter_dead and takes over below.
+ log("counter: pid %d is gone - dropping" % last_pid)
+ counter_dead.add(last_pid)
+ else:
+ try:
+ pin_counter_page(last_pid, counter_pids)
+ except Exception as e:
+ log("counter: pin failed: %s" % e)
+ _hb()
+ counter_next_try[last_pid] = now + COUNTER_RETRY_S
time.sleep(interval)
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-18 04:53:03 +0000