summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c390
1 files changed, 365 insertions, 25 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 931669c..ea96595 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -420,6 +420,18 @@ struct svc_channel {
u64 gr_nocap_va; /* push VA the window was opened for */
unsigned long gr_nocap_since; /* absolute first-seen stamp */
/*
+ * r471: the completion-counter publish must never advance the
+ * guest's word past a push whose content was still unwritten when
+ * the entry was consumed (the streams_events staging-reuse
+ * corruption: the publish satisfied the client's D2H wait before
+ * its copy ran, and the readback drained a staging buffer that
+ * still held the PREVIOUS copy's bytes). Lowest ring slot consumed
+ * with all-zero content; 0 = none. Cleared when a later decode of
+ * a GR push reads non-zero content (the content landed - the rewinds
+ * re-read entries constantly, so this heals within a pass or two).
+ */
+ u32 gr_bad_put;
+ /*
* STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A
* channel that cannot make forward progress (last_get pinned behind
* put, or a residual session put that never drops) is detected here
@@ -486,6 +498,12 @@ struct svc_vaspace {
u64 pdb;
u32 entries; /* root entry count */
int ap; /* SVC_APERTURE_* of the directory itself */
+ unsigned long stamp; /* r459: last registration/use; LRU evict
+ * when the table is full (a stale entry from
+ * a dead client otherwise blocks every new
+ * client after the 8th registration: the
+ * mid-boot walk failure that made the 10th+
+ * client's copies fail rc=-2) */
};
struct svc_lat {
@@ -819,6 +837,7 @@ struct sema_svc {
u64 pushtrace_all; /* fail4: de-windowed push decode
* trace (every push, bounded) */
u64 opadd_log; /* fail4: every op appended */
+ u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */
u64 dmatrace_log; /* fail4: denied-transfer trace */
u64 pend_dbg_log; /* fail4 r305: pending/hold witness */
u64 snap_gate_log; /* fail4 r311: capture-gate witness */
@@ -1599,12 +1618,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
if (!vs && !svc.vas[i].valid)
vs = &svc.vas[i];
}
+ /*
+ * r459: the table is 8 slots and the guest driver does not always
+ * send a teardown for a client's VA space (observed: 23 distinct
+ * registrations, 0 drops in one boot). Once all slots hold a dead
+ * client's directory, a NEW client's registration found no slot and
+ * every walk of its VAs failed (copies rc=-2, launches reading
+ * stale pages - the 10th-client mid-boot degradation). Evict the
+ * least-recently-registered entry instead of refusing the new one;
+ * the old client is gone and its directory pointer is dead guest
+ * memory, so the eviction can only remove stale authority.
+ */
+ if (!vs) {
+ unsigned long oldest = 0;
+ int oi = 0;
+
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (!svc.vas[i].valid)
+ continue;
+ if (!oldest || time_before(svc.vas[i].stamp, oldest)) {
+ oldest = svc.vas[i].stamp;
+ oi = i;
+ }
+ }
+ vs = &svc.vas[oi];
+ pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n",
+ vs->h_vaspace, vs->pdb, desc->h_vaspace);
+ }
if (vs) {
vs->valid = true;
vs->h_vaspace = desc->h_vaspace;
vs->pdb = desc->pdb;
vs->entries = desc->num_entries;
vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ vs->stamp = jiffies;
/* A new directory invalidates what we learned from the old one. */
memset(svc.lat, 0, sizeof(svc.lat));
}
@@ -1932,6 +1979,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
}
spin_unlock_irqrestore(&svc.vas_lock, flags);
+ /*
+ * r459: try the NEWEST registration first. A dead client's stale
+ * directory (the table outlives clients; only 8 slots) can still
+ * resolve a VA - through guest memory the driver has since reused -
+ * to a wrong page, and the first success wins. Ordering by the
+ * registration stamp makes the live client's directory the
+ * authority and only falls through to older ones when it cannot
+ * answer.
+ */
+ for (i = 1; i < n; i++) {
+ struct svc_vaspace k = local[i];
+ int j = i - 1;
+
+ while (j >= 0 && time_before(local[j].stamp, k.stamp)) {
+ local[j + 1] = local[j];
+ j--;
+ }
+ local[j + 1] = k;
+ }
+
for (i = 0; i < n; i++) {
int alt;
@@ -4820,7 +4887,8 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value,
u64 *done_out);
static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src,
size_t total, u32 dst_phys, u32 dst_aperture,
- u32 src_phys, u32 src_aperture, u64 *done_out);
+ u32 src_phys, u32 src_aperture, u64 *done_out,
+ u32 owner_ch, u32 owner_idx, u64 owner_hash);
/*
* r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per
@@ -5938,12 +6006,29 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va,
struct svc_copy_snap {
u64 dst, src;
size_t total;
+ /*
+ * Owner identity (r444): the channel, ring entry and push-content
+ * hash of the decode pass that captured this snapshot. A snapshot
+ * exists to serve ONE entry's retried copy (the held staging read);
+ * keying it by (dst,src,total) alone let a snapshot captured for an
+ * earlier entry poison a later copy with the same operand triple -
+ * the wedge_ab "bad = mutated-element count" class: step N's H2D
+ * executed from step N-1's snapshot because the re-decode of the
+ * completed step N-1 entry re-captured its (unchanged) source after
+ * its copy had already run, and step N's copy then consumed it.
+ * The pass hash changes whenever the guest rewrites the push, so a
+ * ring slot reused by a fresh push can never match a stale owner.
+ */
+ u64 owner_hash;
+ u32 owner_ch;
+ u32 owner_idx;
u8 *buf;
unsigned long stamp;
};
static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N];
-static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total)
+static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total,
+ u32 owner_ch, u32 owner_idx, u64 owner_hash)
{
u32 i;
@@ -5951,12 +6036,16 @@ static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total)
if (svc_copy_snaps[i].buf &&
svc_copy_snaps[i].dst == dst &&
svc_copy_snaps[i].src == src &&
- svc_copy_snaps[i].total == total)
+ svc_copy_snaps[i].total == total &&
+ svc_copy_snaps[i].owner_ch == owner_ch &&
+ svc_copy_snaps[i].owner_idx == owner_idx &&
+ svc_copy_snaps[i].owner_hash == owner_hash)
return svc_copy_snaps[i].buf;
return NULL;
}
-static void svc_copy_snap_free(u64 dst, u64 src, size_t total)
+static void svc_copy_snap_free(u64 dst, u64 src, size_t total,
+ u32 owner_ch, u32 owner_idx, u64 owner_hash)
{
u32 i;
@@ -5964,16 +6053,45 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total)
if (svc_copy_snaps[i].buf &&
svc_copy_snaps[i].dst == dst &&
svc_copy_snaps[i].src == src &&
- svc_copy_snaps[i].total == total) {
+ svc_copy_snaps[i].total == total &&
+ svc_copy_snaps[i].owner_ch == owner_ch &&
+ svc_copy_snaps[i].owner_idx == owner_idx &&
+ svc_copy_snaps[i].owner_hash == owner_hash) {
kfree(svc_copy_snaps[i].buf);
svc_copy_snaps[i].buf = NULL;
return;
}
}
+/*
+ * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's
+ * own successful consumption (svc_copy) -> free on ring-slot reuse with a
+ * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an
+ * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was
+ * removed from every opdef_clear caller: an entry whose release was
+ * delivered can still be REPLAYED (the pin-batch rewind) with its copy
+ * still owed, and freeing at that point let the replay re-capture the
+ * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's
+ * bytes into db1). A stale snapshot can only ever serve its own
+ * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed.
+ */
+static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_COPY_SNAP_N; i++)
+ if (svc_copy_snaps[i].buf &&
+ svc_copy_snaps[i].owner_ch == owner_ch &&
+ svc_copy_snaps[i].owner_idx == owner_idx) {
+ kfree(svc_copy_snaps[i].buf);
+ svc_copy_snaps[i].buf = NULL;
+ }
+}
+
/* Capture the source of a virtual-operand copy about to be held. */
static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst,
- u64 src, size_t total)
+ u64 src, size_t total, u32 owner_ch,
+ u32 owner_idx, u64 owner_hash)
{
struct svc_copy_snap *s = NULL;
u8 *buf;
@@ -5982,8 +6100,27 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst,
if (!total || total > SVC_COPY_SNAP_MAX)
return;
- if (svc_copy_snap_lookup(dst, src, total))
+ if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx,
+ owner_hash))
return; /* already captured */
+ /*
+ * r456: evict any snapshot left by the SAME ring slot under a
+ * DIFFERENT push hash (the slot was reused by a fresh push). The
+ * surviving-across-retries snapshot (svc_opdef_clear APPLY no
+ * longer frees it) must never be readable by the next occupant of
+ * the slot; the pass hash changes whenever the guest rewrites the
+ * push, so a same-slot different-hash snapshot is by definition
+ * stale for the new push.
+ */
+ for (i = 0; i < SVC_COPY_SNAP_N; i++) {
+ if (svc_copy_snaps[i].buf &&
+ svc_copy_snaps[i].owner_ch == owner_ch &&
+ svc_copy_snaps[i].owner_idx == owner_idx &&
+ svc_copy_snaps[i].owner_hash != owner_hash) {
+ kfree(svc_copy_snaps[i].buf);
+ svc_copy_snaps[i].buf = NULL;
+ }
+ }
for (i = 0; i < SVC_COPY_SNAP_N; i++) {
if (!svc_copy_snaps[i].buf) {
s = &svc_copy_snaps[i];
@@ -6019,10 +6156,14 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst,
s->dst = dst;
s->src = src;
s->total = total;
+ s->owner_ch = owner_ch;
+ s->owner_idx = owner_idx;
+ s->owner_hash = owner_hash;
s->stamp = jiffies;
s->buf = buf;
- pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu first4=%08x %08x %08x %08x\n",
- dst, src, total,
+ pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n",
+ dst, src, total, owner_ch, owner_idx,
+ owner_hash,
total >= 4 ? *(const u32 *)buf : 0,
total >= 8 ? *(const u32 *)(buf + 4) : 0,
total >= 12 ? *(const u32 *)(buf + 8) : 0,
@@ -6039,7 +6180,8 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
u64 *pa_out, u32 *ap_out);
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
- u64 *done_out, u32 rows, u32 dpitch, u32 spitch)
+ u64 *done_out, u32 rows, u32 dpitch, u32 spitch,
+ u32 owner_ch, u32 owner_idx, u64 owner_hash)
{
if (rows > 1 && dpitch) {
u64 done = 0;
@@ -6052,7 +6194,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch,
src + (u64)r * spitch, total,
dst_phys, dst_aperture, src_phys,
- src_aperture, &row_done);
+ src_aperture, &row_done,
+ owner_ch, owner_idx, owner_hash);
done += row_done;
if (rc)
break;
@@ -6062,12 +6205,13 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
return rc;
}
return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture,
- src_phys, src_aperture, done_out);
+ src_phys, src_aperture, done_out,
+ owner_ch, owner_idx, owner_hash);
}
static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture,
- u64 *done_out)
+ u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash)
{
u8 bounce[256];
size_t o;
@@ -6098,7 +6242,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot
svc.cur_ch, dst, src, total, dst_phys, dst_aperture,
src_phys, src_aperture);
}
- snap = svc_copy_snap_lookup(dst_orig, src_orig, total);
+ snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch,
+ owner_idx, owner_hash);
if (!dst_phys) {
u64 pa;
@@ -6232,7 +6377,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot
*/
if (!src_phys && src_orig >= SVC_VAS_FLOOR)
svc_copy_snap_capture(vgpu, dst_orig, src_orig,
- total);
+ total, owner_ch, owner_idx,
+ owner_hash);
return -ENOENT;
}
/*
@@ -6261,7 +6407,9 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot
dst_orig, dst_prov, ipa);
if (!src_phys && src_orig >= SVC_VAS_FLOOR)
svc_copy_snap_capture(vgpu, dst_orig,
- src_orig, total);
+ src_orig, total,
+ owner_ch, owner_idx,
+ owner_hash);
return -ENOENT;
}
}
@@ -7036,7 +7184,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot
svc_ledger_store(dst_orig + po, dst + po, cap);
}
if (snap)
- svc_copy_snap_free(dst_orig, src_orig, total);
+ svc_copy_snap_free(dst_orig, src_orig, total, owner_ch,
+ owner_idx, owner_hash);
if (done_out)
*done_out = (u64)o; /* o == total on the success path */
return 0;
@@ -7238,7 +7387,8 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
}
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
- u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base)
+ u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base,
+ u32 entry_idx, u64 pass_hash)
{
u32 i;
static u64 bigop_log;
@@ -7391,6 +7541,18 @@ svc_wait_done:
*/
if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) {
svc.ops_once_skipped++;
+ /*
+ * r456: this op already ran in an earlier pass. Any
+ * staging snapshot a later re-decode pass re-captured
+ * for it (the capture sites run on every decode) is
+ * a leak that the r444 APPLY-free used to sweep; with
+ * that free removed (r456 keeps the snapshot alive
+ * across retries) drop it here so it can never serve
+ * a re-push of the ring slot.
+ */
+ if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX)
+ svc_copy_snap_free(m->dst, m->src, m->bytes,
+ ch_h, entry_idx, pass_hash);
continue;
}
@@ -7404,7 +7566,8 @@ svc_wait_done:
m->phys, m->aperture,
m->src_phys, m->src_aperture,
&ce_done, m->rows, m->dpitch,
- m->spitch);
+ m->spitch, ch_h, entry_idx,
+ pass_hash);
else
rc = svc_fill(vgpu, m->dst, m->value, m->elem,
m->count, m->phys, m->aperture,
@@ -9385,7 +9548,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
* REPORT_SEM comment below. */
if (allow_ops && n_ops && !gsp_sema_skip_ops) {
svc_run_ops(vgpu, ops, n_ops, ch->h_object,
- ops_done, ops_seq - n_ops);
+ ops_done, ops_seq - n_ops,
+ entry_idx, svc_pass_hash);
svc.ring_ops++;
ops_runs++;
}
@@ -9585,7 +9749,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
*/
if (allow_ops && n_ops && !gsp_sema_skip_ops) {
svc_run_ops(vgpu, ops, n_ops, ch->h_object,
- ops_done, ops_seq - n_ops);
+ ops_done, ops_seq - n_ops,
+ entry_idx, svc_pass_hash);
svc.ring_ops++;
ops_runs++;
}
@@ -9809,6 +9974,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
break;
case CE_M_LINE_LENGTH_IN:
st.line_length = v0;
+ /*
+ * r458 THE memset2d root cause: the driver encodes
+ * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT
+ * (0x41c) as ONE count=2 method group - observed raw
+ * push bytes `20028106 00001000 00000100` = header
+ * (subch 4, count 2, addr 0x418), line length 4096,
+ * line count 0x100 = 256. Reading only v0 left
+ * line_count = 0, so the 2D-copy builder below took
+ * the single-row path: one 4 KB op per push, and a
+ * 4 MB cudaMemcpy2D delivered exactly one page per
+ * 1 MB block (memset2d probe: per-MB ok=4096,
+ * bad=12.5M/12.58M). The GRCE variant below has
+ * always decoded the count=2 form; the CE path did
+ * not.
+ */
+ if (count >= 2)
+ st.line_count = v1;
break;
/* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */
case CE_M_LINE_COUNT:
@@ -10116,6 +10298,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
m->spitch = spitch;
}
}
+ /*
+ * r458 memset2d forensics: the
+ * driver's cudaMemcpy2D pushes decode
+ * LINE_LENGTH=4096 with PITCH_IN/OUT
+ * present but LINE_COUNT absent
+ * (lc=0), so only one 4 KB row per
+ * push executes and a 4 MB copy
+ * delivers 4 pages. Dump the raw push
+ * (bounded) for every pitch-shaped
+ * copy so the method stream that
+ * carries the row count is on record.
+ */
+ if (gsp_sema_dbg &&
+ (st.have_pitch_out ||
+ st.have_pitch_in) &&
+ svc.op2d_raw_log < 12) {
+ u32 dl = len < 220 ? len : 220;
+
+ svc.op2d_raw_log++;
+ pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n",
+ ch->h_object, push_va,
+ len, off,
+ st.line_length,
+ st.line_count,
+ st.have_pitch_out,
+ st.pitch_out,
+ st.have_pitch_in,
+ st.pitch_in,
+ m->dst, m->src,
+ (u32)m->bytes, dma);
+ pr_info("sema-op2d-raw: raw %*phN\n",
+ dl, buf);
+ }
n_ops++;
ops_seq++;
} else if (gsp_sema_dbg && svc.dma_skip_log < 32) {
@@ -10295,7 +10510,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
svc_copy_snap_capture(vgpu,
st.offset_out,
st.offset_in,
- st.line_length);
+ st.line_length,
+ ch->h_object,
+ entry_idx,
+ svc_pass_hash);
}
/*
* r308: the `!replay` gate above kept this
@@ -10386,7 +10604,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
svc_copy_snap_capture(vgpu,
lm->dst,
lm->src,
- lm->bytes);
+ lm->bytes,
+ ch->h_object,
+ entry_idx,
+ svc_pass_hash);
if (gsp_sema_dbg &&
svc.snap_gate_log < 256) {
svc.snap_gate_log++;
@@ -10707,7 +10928,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
gsp_sema_skip_ops);
}
svc_run_ops(vgpu, ops, n_ops, ch->h_object,
- ops_done, ops_seq - n_ops);
+ ops_done, ops_seq - n_ops,
+ entry_idx, svc_pass_hash);
svc.ring_ops++;
ops_runs++;
}
@@ -14887,6 +15109,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
return;
val = put;
+ /*
+ * r471: never advance the guest's completion word past a push whose
+ * content was still unwritten when its entry was consumed. The
+ * clamp position is the lowest all-zero GR slot (gr_bad_put = slot
+ * + 1; 0 = none). Without it the publish can satisfy a D2H wait
+ * before its copy ran and the client drains a reused staging buffer
+ * holding the previous copy's bytes (streams_events h_c[2] = c0).
+ * The clamp heals when the rewinds re-read the slot with real
+ * content; the real completions still arrive through the decoded
+ * releases.
+ */
+ if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) {
+ val = ch->gr_bad_put - 1;
+ if (gsp_sema_dbg && svc.counter_pub_dbg < 96) {
+ svc.counter_pub_dbg++;
+ pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n",
+ ch->h_object, pa, val, ch->gr_bad_put, cur);
+ }
+ }
if (val == ch->counter_last)
return;
/*
@@ -15805,6 +16046,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller)
e->n ? e->rel[0].va : 0ull,
e->n ? e->rel[0].payload : 0u);
}
+ /*
+ * r456/r457: the snapshot lifecycle is now "capture once, consume
+ * on the copy's success, evict on ring-slot reuse, TTL". Every
+ * opdef_clear caller can still re-decode the entry:
+ * - APPLY (0): the pass's own op execution (which consumes the
+ * snapshot on success) follows the apply;
+ * - FINISH (1): the entry can be REPLAYED after a consume (the
+ * pin-batch rewind replays entries while mappings stream in),
+ * and an owed copy still needs the pipeline-order bytes;
+ * - FLUSH (2): the op stays skipped this pass but a later replay
+ * may run it.
+ * The r457 mode-C evidence: the b1 H2D's snapshot was captured at
+ * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...),
+ * FINISH freed it on the entry's consume, the rewind replay
+ * re-captured at 191608.798838 from the REFILLED staging
+ * (first4=00000000 3d800000 ... = b2) and the deferred execution
+ * wrote b2's bytes into db1 (dst16_after=000000000000803d...).
+ * The r444 leak the old free guarded against is already covered
+ * by the owner key (a stale snapshot can only ever serve its own
+ * (ch, idx, hash)), the slot-reuse eviction in
+ * svc_copy_snap_capture, the ops_done skip free in svc_run_ops
+ * and the 30 s TTL.
+ */
memset(e, 0, sizeof(*e));
}
@@ -21234,10 +21498,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* through to the historical
* consume so the channel can
* never livelock.
+ *
+ * r470: an ALL-ZERO push page is
+ * the same "not the client's work
+ * yet" condition - the ring entry
+ * is written (the previous
+ * session's or the client's own
+ * slot reuse) but the push
+ * CONTENT has not landed, so the
+ * decode sees no QMD and the
+ * entry would be consumed as a
+ * no-op; the end-of-pass counter
+ * publish then advances the
+ * guest's completion word while
+ * the real push was never read,
+ * and the client drains a
+ * staging buffer that still holds
+ * the PREVIOUS copy's bytes
+ * (streams_events: h_c[2] = c0
+ * because the c2 D2H's
+ * completion was published 4.5 s
+ * before its copy ran, the ring
+ * slots having been decoded as
+ * all-zero pushes). Deliberately
+ * NARROWER than the CE-shaped
+ * plausibility verdict: compute
+ * pushes legitimately fail that
+ * matcher, so only an entirely
+ * zero page counts as unwritten.
*/
+ {
+ bool all_zero = false;
+
+ if (!ch->engine_type &&
+ va && len) {
+ u32 zi;
+
+ all_zero = true;
+ for (zi = 0; zi < len;
+ zi++)
+ if (pb[zi]) {
+ all_zero =
+ false;
+ break;
+ }
+ }
+ /*
+ * r471: publish clamp bookkeeping.
+ * A zero-content GR entry that is
+ * held or consumed must never let
+ * the end-of-pass counter publish
+ * advance the guest's completion
+ * word past it (the client would
+ * satisfy its readback wait before
+ * the real push/copy ran - the
+ * staging buffer still holds the
+ * previous copy's bytes). Record
+ * the lowest zero slot; a decode
+ * that reads non-zero content
+ * clears the clamp.
+ */
+ if (!ch->engine_type) {
+ if (all_zero) {
+ if (!ch->gr_bad_put ||
+ (u32)idx + 1 <
+ ch->gr_bad_put)
+ ch->gr_bad_put =
+ (u32)idx + 1;
+ } else if (va && len) {
+ ch->gr_bad_put = 0;
+ }
+ }
if (!ch->engine_type && va && len &&
gsp_phys_gr_forward_enabled() &&
- svc_gr_qmd_seen > 0 &&
+ (svc_gr_qmd_seen > 0 ||
+ all_zero) &&
!entry_replay &&
!(svc_gr_cand.valid &&
svc_gr_cand.h_object ==
@@ -21289,6 +21624,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->h_object,
(u32)idx, va, len);
}
+ }
/*
* A CE push that should carry this
* channel's next tracking release but
@@ -22024,6 +22360,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].push_hold_until = ch->push_hold_until;
svc.chan[k].defer_since = ch->defer_since;
svc.chan[k].put_prev = ch->put_prev;
+ /* r471: the publish clamp watermark is set
+ * and cleared on the pump thread; persist it
+ * so the next pass still clamps. */
+ svc.chan[k].gr_bad_put = ch->gr_bad_put;
/*
* Persist the cheap-wait window and the content
* watermark: the pump works on a snapshot, so
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-22 19:01:07 +0000