summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rm_emulation.c249
-rw-r--r--kernel/gsp_rm_emulation.h1
-rw-r--r--kernel/gsp_sema_service.c274
3 files changed, 489 insertions, 35 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index eddfab5..90556f2 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -1169,6 +1169,7 @@ static const char *gsp_rm_rpc_name(u32 function)
case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL";
case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER: return "UNLOADING_GUEST_DRIVER";
case GSPRM_RPC_FUNC_UPDATE_BAR_PDE: return "UPDATE_BAR_PDE";
+ case GSPRM_RPC_FUNC_DUP_OBJECT: return "DUP_OBJECT";
default: return "UNKNOWN";
}
}
@@ -2955,6 +2956,115 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx,
}
/*
+ * r519 experiment switch, independent of the P3 whitelist: forward ONLY the
+ * BINAPI reserved-interface query 0x20810107 to the live GSP session. The P3
+ * whitelist may be off (its mixed real/emulated device answers perturb the
+ * boot); this one command is what the GL surface init needs answered.
+ */
+static bool gsp_rm_binapi07_fwd_param;
+module_param_named(gsp_rm_binapi07_fwd, gsp_rm_binapi07_fwd_param, bool, 0644);
+MODULE_PARM_DESC(gsp_rm_binapi07_fwd,
+ "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)");
+
+/*
+ * The live session's oracle client/object for the 0x20810107 experiment,
+ * created on demand; 0 until the first query builds it. One device per module
+ * load, so file-statics are enough; reset by unload.
+ */
+static u32 gsp_rm_binapi07_be_client;
+static u32 gsp_rm_binapi07_be_object;
+
+/*
+ * One GSP_RM_ALLOC (fn 103) against the live backend, built exactly like the
+ * rm-call ladder's helper (rpc_gsp_rm_alloc_v03_00: hClient, hParent,
+ * hObject, hClass, status, paramsSize, flags, reserved[4] + params).
+ * object_is_out selects the reply read: a ROOT alloc returns the
+ * server-chosen client handle in params[0] (payload +32), anything else
+ * echoes the client-chosen hObject at +8; status is at +16 in both cases.
+ */
+static int gsp_rm_binapi07_backend_alloc(u32 h_client, u32 h_parent,
+ u32 h_object, u32 h_class,
+ const void *params, u32 params_len,
+ bool object_is_out,
+ u32 *out_object, u32 *out_status)
+{
+ u8 *areq;
+ void *arsp = NULL;
+ u32 arsp_len = 0, st = 0xffffffff, ho = 0;
+ int ret;
+
+ areq = kzalloc(32 + params_len, GFP_KERNEL);
+ if (!areq)
+ return -ENOMEM;
+ memcpy(areq + 0, &h_client, 4);
+ memcpy(areq + 4, &h_parent, 4);
+ memcpy(areq + 8, &h_object, 4);
+ memcpy(areq + 12, &h_class, 4);
+ memcpy(areq + 20, &params_len, 4);
+ if (params_len)
+ memcpy(areq + 32, params, params_len);
+
+ ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_ALLOC, areq,
+ 32 + params_len, &arsp, &arsp_len, 500);
+ kfree(areq);
+ if (!ret && arsp) {
+ if (arsp_len >= 20)
+ memcpy(&st, (u8 *)arsp + 16, 4);
+ if (object_is_out) {
+ if (arsp_len >= 36)
+ memcpy(&ho, (u8 *)arsp + 32, 4);
+ } else if (arsp_len >= 12) {
+ memcpy(&ho, (u8 *)arsp + 8, 4);
+ }
+ }
+ kfree(arsp);
+ if (out_object)
+ *out_object = ho;
+ if (out_status)
+ *out_status = st;
+ return ret;
+}
+
+/*
+ * Build the oracle object on the live session: a class-0x2081 child of the
+ * session's internal subdevice, since the server dispatches the method from
+ * the OBJECT's class (the subdevice attempt alone answered
+ * NV_ERR_NOT_SUPPORTED, 0x56, and a fresh root client at runtime answered
+ * NV_ERR_INVALID_CLIENT, 0x23 - the ladder "adopts" the server's own
+ * internal client instead of creating one). A small candidate list of
+ * likely-free handles in that client's namespace is tried; the first
+ * success wins. Returns 0 with *out_object set.
+ */
+static int gsp_rm_binapi07_backend_setup(struct gsp_emu_context *ctx,
+ struct gsp_phys_backend_static_config *snap,
+ u32 *out_object)
+{
+ static const u32 cand[] = { 0xabcd2081u, 0xabcd2001u, 0xabcd2a2au,
+ 0xabcd5f5fu, 0xabcd1234u };
+ u8 p4[4] = { 0, 0, 0, 0 };
+ u32 st = 0xffffffff, obj = 0;
+ int i, ret;
+
+ for (i = 0; i < (int)ARRAY_SIZE(cand); i++) {
+ u32 want = cand[i];
+
+ obj = want;
+ ret = gsp_rm_binapi07_backend_alloc(snap->h_internal_client,
+ snap->h_internal_subdevice,
+ want, 0x2081, p4, 4,
+ false, &obj, &st);
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: BINAPI-EXPERIMENT oracle BINAPI alloc attempt %d: hObject=0x%x ret=%d status=0x%x",
+ i, want, ret, st);
+ if (!ret && st == 0 && obj) {
+ *out_object = obj;
+ return 0;
+ }
+ }
+ return -EIO;
+}
+
+/*
* Bounded logging for the P3 forward counters: the first four fallbacks (so a
* session that cannot forward is visible) and every 64th event. Never called
* when the parameter is off.
@@ -3003,12 +3113,37 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
u32 out_params_size;
int ret;
+ bool guest_handles = false;
+
/* Wave 1 whitelist: compile-time side-effect-free queries only. */
switch (req->cmd) {
case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
case TU104_CTRL_BUS_GET_INFO_V2:
case TU104_CTRL_FB_GET_INFO_V2:
case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO:
+ if (!gsp_phys_ctrl_forward_enabled())
+ return false;
+ break;
+ case 0x20810107: /* NV2081_BINAPI reserved-interface method 7 */
+ /*
+ * EXPERIMENT (r519): the guest GL core allocates a BINAPI
+ * object (class 0x2081) and sends this 2-byte query during
+ * 3D-surface init; our {0,0} generic reply leaves the
+ * driver's feature flag clear and the EGL surface init tears
+ * down (EGL_BAD_ALLOC at eglMakeCurrent). The guest-side
+ * handles (hClient 0xc1d..., a 0xbeef object) have no backend
+ * counterpart: allocate a real BINAPI object (class 0x2081) on
+ * the live session once, then send the query to it. The first
+ * attempt (subdevice translation) came back NV_ERR_NOT_SUPPORTED
+ * (0x56) - the server dispatches the method by the OBJECT's
+ * class, so a subdevice cannot answer a BINAPI method. The
+ * reply is logged either way; an error falls back to the
+ * emulated generic success until the semantics are known.
+ * Gated by its own parameter so the P3 whitelist can stay off.
+ */
+ if (!gsp_rm_binapi07_fwd_param)
+ return false;
+ guest_handles = true;
break;
default:
return false;
@@ -3020,24 +3155,44 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE)
goto fallback;
+ /* The experiment is scoped to the observed shape: 2-byte in/out. */
+ if (guest_handles && req->paramsSize != 2)
+ goto fallback;
+
/* A valid snapshot implies a READY session and the adopted handles. */
if (gsp_phys_backend_static_get(&snap))
goto fallback;
+ if (guest_handles && !gsp_rm_binapi07_be_object) {
+ u32 obj = 0;
+
+ if (gsp_rm_binapi07_backend_setup(ctx, &snap, &obj))
+ goto fallback;
+ /*
+ * The object lives in the internal client's namespace; the
+ * control is sent as that client (snap.h_internal_client).
+ */
+ gsp_rm_binapi07_be_client = snap.h_internal_client;
+ gsp_rm_binapi07_be_object = obj;
+ }
+
/* Translate the guest's emulated internal handles; any other handle
* domain stays emulated. */
- if (req->hClient != TU104_H_INTERNAL_CLIENT ||
- (req->hObject != TU104_H_INTERNAL_DEVICE &&
- req->hObject != TU104_H_INTERNAL_SUBDEVICE))
+ if (!guest_handles &&
+ (req->hClient != TU104_H_INTERNAL_CLIENT ||
+ (req->hObject != TU104_H_INTERNAL_DEVICE &&
+ req->hObject != TU104_H_INTERNAL_SUBDEVICE)))
goto fallback;
fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL);
if (!fwd_req)
goto fallback;
memset(&fwd_hdr, 0, sizeof(fwd_hdr));
- fwd_hdr.hClient = snap.h_internal_client;
- fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ?
- snap.h_internal_device : snap.h_internal_subdevice;
+ fwd_hdr.hClient = guest_handles ? gsp_rm_binapi07_be_client :
+ snap.h_internal_client;
+ fwd_hdr.hObject = guest_handles ? gsp_rm_binapi07_be_object :
+ (req->hObject == TU104_H_INTERNAL_DEVICE ?
+ snap.h_internal_device : snap.h_internal_subdevice);
fwd_hdr.cmd = req->cmd;
fwd_hdr.paramsSize = req->paramsSize;
fwd_hdr.flags = req->flags;
@@ -3057,6 +3212,27 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
}
/*
+ * r519 experiment evidence: log the live GSP's own answer to the
+ * BINAPI query (status + first params bytes) before deciding.
+ */
+ if (req->cmd == 0x20810107) {
+ u32 st = 0xffffffff, ps = 0xffffffff;
+ u8 b[8] = { 0 };
+
+ if (rsp && rsp_len >= GSP_RM_CTRL_HDR_SIZE) {
+ memcpy(&st, (u8 *)rsp + 12, 4);
+ memcpy(&ps, (u8 *)rsp + 16, 4);
+ if (ps <= 8 && ps <= rsp_len - GSP_RM_CTRL_HDR_SIZE)
+ memcpy(b, (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE,
+ ps);
+ }
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: BINAPI-EXPERIMENT 0x20810107 forwarded to live GSP: rsp_len=%u status=0x%x outSize=%u bytes=%02x %02x %02x %02x %02x %02x %02x %02x",
+ rsp_len, st, ps, b[0], b[1], b[2], b[3], b[4],
+ b[5], b[6], b[7]);
+ }
+
+ /*
* The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize
* at +16, echoed params at +24 - the same 24-byte prefix the emulated
* handler builds below. The RPC-level rpc_result was already checked
@@ -3231,7 +3407,7 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
* return means nothing was sent and the emulated switch below runs
* byte-identically to the parameter-off path.
*/
- if (gsp_phys_ctrl_forward_enabled() &&
+ if ((gsp_phys_ctrl_forward_enabled() || gsp_rm_binapi07_fwd_param) &&
gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE,
reply, reply_data_len, sequence)) {
kfree(reply);
@@ -3623,6 +3799,61 @@ rm_control_reply:
}
/*
+ * r510gl: DUP_OBJECT (function 21) - NVOS55_PARAMETERS echo with status=NV_OK.
+ *
+ * The guest driver dups an object into a second client at EGL/GLX
+ * MakeCurrent time (memory and VASpace objects: the GL context's client
+ * shares the device's allocations), and the historical generic-SUCCESS
+ * default replied with an EMPTY payload - the driver's reply must carry the
+ * params struct back (rpc_dup_object_v03_00 = NVOS55_PARAMETERS
+ * {hClient, hParent, hObject, hClientSrc, hObjectSrc, flags, status}), so
+ * an empty reply failed the call and the driver tore the whole EGL session
+ * down (guest witness: 'RPC UNKNOWN(21) seq=0 len=60 -> generic SUCCESS',
+ * immediately followed by the session's FREE cascade and
+ * eglMakeCurrent -> EGL_BAD_ALLOC (0x3003), 2026-10-23 r510).
+ *
+ * The emulation does not model per-object state for these classes (controls
+ * and allocs are answered from the request), so the echo with NV_OK is the
+ * complete answer; the dup'd handle is used by later calls exactly like the
+ * source handle.
+ */
+static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx,
+ const u8 *elem, u32 sequence)
+{
+ struct gsp_rm_emu *rm = ctx->rm_emu;
+ const struct gsprm_rpc_header *hdr =
+ (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
+ const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
+ u8 params[28]; /* NVOS55_PARAMETERS_v03_00 */
+ u32 hDstClient = 0, hDstObject = 0, hSrcClient = 0, hSrcObject = 0;
+
+ if (hdr->length >= sizeof(*hdr) + sizeof(params))
+ memcpy(params, req, sizeof(params));
+ else
+ memset(params, 0, sizeof(params));
+
+ memcpy(&hDstClient, params + 0, 4);
+ memcpy(&hDstObject, params + 8, 4);
+ memcpy(&hSrcClient, params + 12, 4);
+ memcpy(&hSrcObject, params + 16, 4);
+ memset(params + 24, 0, 4); /* status = NV_OK */
+
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo",
+ hDstClient, hDstObject, hSrcClient, hSrcObject, sequence);
+
+ /* Small delay before the reply: same reply-matching race mitigation as
+ * FREE/UNLOADING. */
+ udelay(2000);
+
+ if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_DUP_OBJECT, sequence, 0,
+ params, sizeof(params)))
+ nv_vgpu_err(ctx->vgpu, "vgsp-rm: DUP_OBJECT reply send failed");
+ else
+ rm->rpcs_replied++;
+}
+
+/*
* GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
*
* rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
@@ -3903,6 +4134,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
gsp_rm_handle_free(ctx, elem, sequence);
return;
+ case GSPRM_RPC_FUNC_DUP_OBJECT:
+ gsp_rm_handle_dup_object(ctx, elem, sequence);
+ return;
+
case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:
gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length);
return;
diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
index 96ea16a..45fcc64 100644
--- a/kernel/gsp_rm_emulation.h
+++ b/kernel/gsp_rm_emulation.h
@@ -45,6 +45,7 @@ struct gsp_emu_context;
#define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76
#define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103
#define GSPRM_RPC_FUNC_FREE 10
+#define GSPRM_RPC_FUNC_DUP_OBJECT 21
#define GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER 47
#define GSPRM_RPC_FUNC_UPDATE_BAR_PDE 70
#define GSPRM_RPC_EVENT_POST_EVENT 0x1003
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 1e9f981..42e8d76 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -394,6 +394,16 @@ struct svc_channel {
* even if still unresolved - bounded zeros, never
* a hang); m156: re-held while the window is
* live so a busy forward gets its real retry */
+ /*
+ * r509: one-shot verdict of the absolute bound. The expiry call
+ * pushes the deadline (so the hold window restarts) and latches this
+ * flag; the r351 retry-forward block consumes it within the same
+ * pass. Without the latch the two call sites in one pass could not
+ * both observe the expiry (the first push of the deadline made the
+ * second call see a fresh window - r213-r215's arming bug in
+ * reverse).
+ */
+ bool gr_hold_bound_hit;
unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data-
* stalled hold for gr_hold_dead_va.
* Fixed once on the first hold of that
@@ -2248,8 +2258,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
}
if (n < SVC_PD_MIN_HITS) {
n = 0;
- if (svc.map_n < SVC_PD_SAMPLES)
- return; /* no evidence yet: retry when pairs appear */
+ /*
+ * r510: a LARGE map is a mixed set of VA spaces cached over
+ * the whole guest boot (the comment above already names the
+ * consequence: no single directory reproduces a mixed set), so
+ * scanning with its stride subsample burns the attempts budget
+ * on evidence that cannot succeed - and by the time the
+ * push-buffer pairs (the strongest samples) exist, the budget
+ * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram
+ * cands=39621 best=0/8)' during the first client, eight RAM
+ * walks consumed, every later client's high argument pointers
+ * unresolvable - the terminal class). Only a SMALL map, which
+ * is one session's own pairs, still falls back; otherwise wait
+ * for the push-buffer pairs and keep the attempts.
+ */
+ if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) ||
+ svc.map_n < SVC_PD_SAMPLES)
+ return; /* no coherent evidence yet: retry later */
step = svc.map_n / SVC_PD_SAMPLES;
for (i = 0; i < SVC_PD_SAMPLES; i++) {
sva[i] = svc.map[i * step].gpu_page;
@@ -7527,6 +7552,79 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
return free_slot;
}
+/*
+ * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op
+ * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode
+ * pass and sets svc.op_tr_pending each time, so its push holds for the full
+ * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op
+ * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx
+ * after the first client, dst never translated) ground its whole stream
+ * at ~10 s per push ("hang after DEVICE" for matmul/multikernel).
+ *
+ * The window must still cover the legitimate pin-late retry (observed
+ * pin latency 0.5-2 s: the H2D copy's operands land a moment after the
+ * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes),
+ * so the latch is TIME-based: the FIRST failure of a VA arms a
+ * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as
+ * before; after a full bound with no success the op still attempts the
+ * write but no longer forces the hold - the entry consumes promptly and
+ * its withheld release applies, exactly the outcome the expired bound
+ * already sanctioned (op stays skipped, release delivered), minus the
+ * 10 s tax per subsequent pass. A success clears the record (a later
+ * failure gets a fresh window). Service-thread only, like every other op
+ * path.
+ */
+#define SVC_OPFAIL_MAX 32
+/*
+ * r507a: the op-failure window rides the opdef hold bound (10000 ms,
+ * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the
+ * latch (used by svc_run_ops) precedes that define.
+ */
+#define SVC_OPFAIL_WINDOW_MS 10000u
+static struct {
+ u64 va;
+ unsigned long since;
+} svc_opfail[SVC_OPFAIL_MAX];
+static u32 svc_opfail_log;
+
+static void svc_opfail_clear(u64 va)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_OPFAIL_MAX; i++)
+ if (svc_opfail[i].va == va)
+ svc_opfail[i].va = 0;
+}
+
+static bool svc_opfail_latched(u64 va)
+{
+ u32 i;
+ u32 free_i = SVC_OPFAIL_MAX;
+
+ for (i = 0; i < SVC_OPFAIL_MAX; i++) {
+ if (svc_opfail[i].va == va) {
+ if (time_after_eq(jiffies,
+ svc_opfail[i].since +
+ msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) {
+ if (svc_opfail_log < 32) {
+ svc_opfail_log++;
+ pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n",
+ va, SVC_OPFAIL_WINDOW_MS);
+ }
+ return true;
+ }
+ return false;
+ }
+ if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX)
+ free_i = i;
+ }
+ if (free_i < SVC_OPFAIL_MAX) {
+ svc_opfail[free_i].va = va;
+ svc_opfail[free_i].since = jiffies;
+ }
+ return false;
+}
+
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base,
u32 entry_idx, u64 pass_hash)
@@ -7784,9 +7882,11 @@ svc_wait_done:
*/
if (rc == SVC_COPY_FORWARDED)
continue;
- if (rc == 0)
+ if (rc == 0) {
svc.memsets_done++;
- else if (rc == -ENOENT)
+ /* r507a: a success re-arms the failure window. */
+ svc_opfail_clear(m->dst);
+ } else if (rc == -ENOENT)
svc.memsets_nomap++;
else
svc.memsets_fail++;
@@ -7817,7 +7917,8 @@ svc_wait_done:
if (rc != 0 && rc != SVC_COPY_FORWARDED &&
(m->bytes >= (256u << 10) ||
(m->type == 1 && m->dst >= SVC_VAS_FLOOR) ||
- (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)))
+ (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) &&
+ !svc_opfail_latched(m->dst))
svc.op_tr_pending = 1;
if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) {
svc.sem256_log++;
@@ -8777,7 +8878,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
inc && inc_apply);
return false;
}
- if (pending || inc_apply) {
+ /*
+ * r506w: generic absolute releases are forward-only by VALUE, whichever
+ * pass discovers the gap. The recorded pending bit came from the
+ * decode's read of the release word, and that read can be blinded by a
+ * stale/crossed landing (r506 witness: a withheld tracker release went
+ * to the flush with pending=0 while the fresh read still showed 0, and
+ * the release was then dropped by the counter fallback below - the
+ * client's tracker never advanced and its stream ground at the 10 s
+ * bound per push). Increments keep the one-shot claim gate (`inc_apply`).
+ */
+ if (pending || inc_apply || (!inc && payload > cur)) {
svc.pushes_pending++;
if (inc)
cur += payload;
@@ -8799,10 +8910,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
return true;
}
}
- if (ch->engine_type == 0)
- svc_counter_learn(vgpu, ch->h_object, sem_va);
- svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
- inc && inc_apply);
+ /*
+ * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its
+ * `release_pa ?: counter_pa` fallback writes (and forward-only-skips
+ * against) the channel's REGISTERED slot, which belongs to a different
+ * word - the r506 witness: releases to the tracker VA 0x20480fef0 were
+ * skipped against the polluted PUT counter (cur=96) and lost. For a
+ * non-counter VA the generic write above is the delivery path.
+ */
+ if (svc_counter_va(sem_va)) {
+ if (ch->engine_type == 0)
+ svc_counter_learn(vgpu, ch->h_object, sem_va);
+ svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+ inc && inc_apply);
+ }
/*
* Stall-class fix (R2): when the decode said this absolute release is
* genuinely pending and every write attempt above failed without
@@ -9536,17 +9657,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
bytes,
ch->h_object);
/*
- * fail4: hardware semantics - a
- * LOAD_INLINE_DATA run advances the
- * destination cursor by the bytes it
- * wrote. Without this, the consecutive
- * runs of a split 64 KB upload all land
- * at the first destination and the page
- * ends up with only the last run's
- * words (the RAWA junk).
- */
- st.offset_out += bytes;
- /*
* fail4: name the header that follows
* each big LID run - the split 64 KB
* upload's other payload carriers sit
@@ -9574,6 +9684,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
trusted, allow_ops, st.have_out,
st.line_length, ch->h_object);
}
+ /*
+ * fail4 (r526): hardware semantics - a
+ * LOAD_INLINE_DATA run advances the destination
+ * cursor by the bytes it carries whether or not
+ * the once-only gate let the write run. Without
+ * this, the consecutive runs of a split upload
+ * all land at the first destination (the old
+ * RAWA bug), and worse: on a re-decode pass where
+ * the gate suppresses the FIRST run, the cursor
+ * stays at the first run's destination and the
+ * SECOND run executes there - its data (the
+ * buffer's bytes 8..16) overwriting offset 0
+ * (the observed tgemm XDIFF: X[0..1] <-
+ * X[2..3], deterministic, every pass). The gate
+ * suppresses the write, not the method stream's
+ * positional semantics.
+ */
+ if (bytes)
+ st.offset_out += bytes;
}
/*
* Record the destination and the payload bytes for
@@ -19065,6 +19194,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
* forever (the same identity bug class as the opdef=955/13/0
* streams2 wedge). The first hold arms; others ride it; a fresh
* bound arms once the previous one has passed.
+ *
+ * r509: the bound must actually REPORT expiry. The historical arming
+ * rule re-armed the deadline on the very call that detected expiry
+ * and then compared against the NEW deadline - so the function could
+ * never return true, and a data-stalled launch whose arguments never
+ * resolved held its entry forever (r508 witness: matmul plus ten
+ * sibling channels in "GR hold ... data args unresolved" with zero
+ * GR-STALL-hold-bound lines for 10+ minutes; the designed
+ * retry-forward + consume never ran). The expiry is now one-shot:
+ * push the deadline, latch gr_hold_bound_hit for the r351 block of
+ * the same pass, return true. A fresh arm clears the latch (and the
+ * once-per-launch retry log).
*/
if (!ch->gr_hold_dead_va ||
time_after_eq(jiffies, ch->gr_hold_deadline)) {
@@ -19072,6 +19213,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
ch->gr_hold_dead_since = jiffies;
ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
ch->gr_hold_exp_logged = false;
+ ch->gr_hold_bound_hit = false;
+ return false;
}
/*
* r420: pause the per-launch consumption bound while the backend
@@ -19094,7 +19237,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
launch_va);
return false;
}
- return time_after_eq(jiffies, ch->gr_hold_deadline);
+ if (time_after_eq(jiffies, ch->gr_hold_deadline)) {
+ /*
+ * r509: one-shot expiry. Push the deadline so the hold window
+ * restarts for the caller's retry, latch the verdict for the
+ * second call site of this pass (the r351 retry-forward
+ * block), and report expired.
+ */
+ ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
+ ch->gr_hold_bound_hit = true;
+ return true;
+ }
+ return false;
}
static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
@@ -20729,8 +20883,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
bool stalled = false;
spin_lock_irqsave(&svc.chan_lock, flags);
+ /*
+ * r507o stream-order: walk the CE channels BEFORE the GR channels.
+ * The client's stream order places its H2D copies before the kernel
+ * that consumes them, but the channels register in handle order (the
+ * RM's GR channel early, the copy-ce channels later), and a pass that
+ * decoded the launch before the CE channel's entries ran let the
+ * forward workqueue execute the launch's copy-in against pre-copy
+ * data (r480-sep: copy-in 115 ms before its stream's H2D soft
+ * copies; r481 proved no host semaphore links the two, so the
+ * emulation must enforce stream order itself). The CE copies execute
+ * on this same service thread, so consuming the CE entries first
+ * fixes the inversion directly - no cross-channel hold, no latency
+ * tax. Stable partition; relative order within each class preserved.
+ */
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
- if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu)
+ if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+ svc.chan[i].engine_type)
+ local[n++] = svc.chan[i];
+ }
+ for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+ if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+ !svc.chan[i].engine_type)
local[n++] = svc.chan[i];
}
spin_unlock_irqrestore(&svc.chan_lock, flags);
@@ -21263,10 +21437,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
{
u64 ident = (u64)e0 | ((u64)e1 << 32);
- bool same = ident &&
+ /*
+ * r505b: the per-slot op bookkeeping arrays are
+ * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL
+ * channel's ring is 8192 slots. Bound every slot
+ * index here or a large ring writes out of
+ * bounds.
+ */
+ bool same = ident && idx < SVC_GPFIFO_MAX_BITS &&
ch->slot_ops_ident[idx] == ident;
- entry_ident = ident;
+ entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0;
/*
* A complete prior pass suppresses every op.
* A partial pass (an operand's targeted scan
@@ -21720,7 +21901,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
kfree(pb);
break;
}
- if (svc.op_tr_pending)
+ if (svc.op_tr_pending) {
/*
* Deadline hit: the
* operand never
@@ -21732,6 +21913,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
svc_opdef_flush(vgpu, ch,
(u32)idx);
+ /*
+ * r505b: mark the slot's
+ * ops complete. The op was
+ * permanently skipped, but
+ * the decode above stored
+ * complete=0 (op_tr_pending
+ * was set), so every pin-
+ * batch rewind replay of
+ * this slot re-decoded it,
+ * re-set op_tr_pending and
+ * re-held the entry in a
+ * livelock (multikernel on
+ * r505: opdef held=18359,
+ * expired=61, the client
+ * waiting forever while the
+ * pump looped on slot 1).
+ * Nothing is owed after the
+ * flush, so a replay must
+ * skip the ops.
+ */
+ if ((u32)idx <
+ SVC_GPFIFO_MAX_BITS) {
+ ch->slot_ops_complete[idx] =
+ 1;
+ ch->slot_ops_done[idx] =
+ entry_done;
+ }
+ }
rel_seen = (svc_rel_count != rel_n_before);
{
bool fwd_guest_queued;
@@ -21967,10 +22176,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (!fwd_guest_queued &&
!ch->engine_type &&
svc_gr_data_stalled &&
- svc_gr_hold_bound_expired(ch,
- svc_gr_launch_va(ch)) &&
+ ch->gr_hold_bound_hit &&
!ch->gr_hold_exp_logged) {
ch->gr_hold_exp_logged = true;
+ /* r509: the latch is this pass's only
+ * report of the expiry; clear it so the
+ * NEXT expiry cycle (gr_hold_exp_logged
+ * now set) falls through to the
+ * historical consume. */
+ ch->gr_hold_bound_hit = false;
svc.fwd_gr_hold_exp++;
/*
* r351: the bound firing used to
@@ -23040,6 +23254,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->gr_hold_deadline;
svc.chan[k].gr_hold_exp_logged =
ch->gr_hold_exp_logged;
+ /* r509: the one-shot expiry latch is snapshot
+ * state too. */
+ svc.chan[k].gr_hold_bound_hit =
+ ch->gr_hold_bound_hit;
/* r361: the opdef absolute-bound record is
* snapshot state too (the GRFIX lesson) -
* without the write-back the deadline would
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-23 11:07:42 +0000