summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rm_emulation.c293
-rw-r--r--kernel/gsp_rm_emulation.h1
-rw-r--r--kernel/gsp_rm_tu104.h13
-rw-r--r--kernel/gsp_sema_service.c518
-rw-r--r--kernel/gsp_sema_service.h8
5 files changed, 789 insertions, 44 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 63a4338..90556f2 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -1169,6 +1169,7 @@ static const char *gsp_rm_rpc_name(u32 function)
case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL";
case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER: return "UNLOADING_GUEST_DRIVER";
case GSPRM_RPC_FUNC_UPDATE_BAR_PDE: return "UPDATE_BAR_PDE";
+ case GSPRM_RPC_FUNC_DUP_OBJECT: return "DUP_OBJECT";
default: return "UNKNOWN";
}
}
@@ -2456,6 +2457,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx,
#define GSPRM_COPY_PDES_MAX_LEVELS 6
static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
+ u32 h_object,
const u8 *params, u32 params_size)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
@@ -2506,6 +2508,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
"vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u",
i, phys, size, aperture, page_shift);
}
+
+ /*
+ * r496j: this control IS the kernel RM publishing the page-table
+ * chain of its internal VAS (the level[0..] instances with their
+ * phys addresses and page shifts). The chain can name the X/GL
+ * channel's pushbuffer pages and progress semaphore, for which no
+ * other translation source exists (the fn-54 SET_PAGE_DIRECTORY
+ * path is external-VAS/UVM only). It is NOT registered directly:
+ * a blast registration of every client's directory let a stale
+ * kernel directory answer another client's VA in the walker's
+ * first-success order (the r459 stale-directory class; observed
+ * as a CUDA client hanging after DEVICE with r496g). Instead the
+ * root is offered as a CANDIDATE; the progress service registers
+ * it only after it has been VERIFIED against a content-validated
+ * pair of that very client (the ring found by the RAM content
+ * scan).
+ */
+ if (num_levels >= 1 && rm->pt.levels[0].phys &&
+ rm->pt.levels[0].size >= 16 &&
+ rm->pt.levels[0].size <= 4096 * 8) {
+ /*
+ * The level phys values are guest PAs on this platform
+ * (observed 4+ GiB, above any FB shadow); aperture 3 in the
+ * message does not match the walker's FB/SYS encoding, so
+ * classify by address: values inside the emulated VRAM are
+ * FB, everything else sysmem. The walker also retries the
+ * other aperture, so a misclassification only costs a walk.
+ */
+ bool vidmem = rm->pt.levels[0].phys <
+ (u64)ctx->vgpu->bars[1].size;
+
+ gsp_sema_service_vaspace_candidate_add(
+ h_object, rm->pt.levels[0].phys,
+ (u32)(rm->pt.levels[0].size / 8), vidmem);
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)",
+ h_object, (u64)rm->pt.levels[0].phys,
+ (u32)(rm->pt.levels[0].size / 8),
+ vidmem ? "vidmem" : "sysmem");
+ }
}
static bool gsp_rm_alloc_interesting(u32 class_id);
@@ -2914,6 +2956,115 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx,
}
/*
+ * r519 experiment switch, independent of the P3 whitelist: forward ONLY the
+ * BINAPI reserved-interface query 0x20810107 to the live GSP session. The P3
+ * whitelist may be off (its mixed real/emulated device answers perturb the
+ * boot); this one command is what the GL surface init needs answered.
+ */
+static bool gsp_rm_binapi07_fwd_param;
+module_param_named(gsp_rm_binapi07_fwd, gsp_rm_binapi07_fwd_param, bool, 0644);
+MODULE_PARM_DESC(gsp_rm_binapi07_fwd,
+ "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)");
+
+/*
+ * The live session's oracle client/object for the 0x20810107 experiment,
+ * created on demand; 0 until the first query builds it. One device per module
+ * load, so file-statics are enough; reset by unload.
+ */
+static u32 gsp_rm_binapi07_be_client;
+static u32 gsp_rm_binapi07_be_object;
+
+/*
+ * One GSP_RM_ALLOC (fn 103) against the live backend, built exactly like the
+ * rm-call ladder's helper (rpc_gsp_rm_alloc_v03_00: hClient, hParent,
+ * hObject, hClass, status, paramsSize, flags, reserved[4] + params).
+ * object_is_out selects the reply read: a ROOT alloc returns the
+ * server-chosen client handle in params[0] (payload +32), anything else
+ * echoes the client-chosen hObject at +8; status is at +16 in both cases.
+ */
+static int gsp_rm_binapi07_backend_alloc(u32 h_client, u32 h_parent,
+ u32 h_object, u32 h_class,
+ const void *params, u32 params_len,
+ bool object_is_out,
+ u32 *out_object, u32 *out_status)
+{
+ u8 *areq;
+ void *arsp = NULL;
+ u32 arsp_len = 0, st = 0xffffffff, ho = 0;
+ int ret;
+
+ areq = kzalloc(32 + params_len, GFP_KERNEL);
+ if (!areq)
+ return -ENOMEM;
+ memcpy(areq + 0, &h_client, 4);
+ memcpy(areq + 4, &h_parent, 4);
+ memcpy(areq + 8, &h_object, 4);
+ memcpy(areq + 12, &h_class, 4);
+ memcpy(areq + 20, &params_len, 4);
+ if (params_len)
+ memcpy(areq + 32, params, params_len);
+
+ ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_ALLOC, areq,
+ 32 + params_len, &arsp, &arsp_len, 500);
+ kfree(areq);
+ if (!ret && arsp) {
+ if (arsp_len >= 20)
+ memcpy(&st, (u8 *)arsp + 16, 4);
+ if (object_is_out) {
+ if (arsp_len >= 36)
+ memcpy(&ho, (u8 *)arsp + 32, 4);
+ } else if (arsp_len >= 12) {
+ memcpy(&ho, (u8 *)arsp + 8, 4);
+ }
+ }
+ kfree(arsp);
+ if (out_object)
+ *out_object = ho;
+ if (out_status)
+ *out_status = st;
+ return ret;
+}
+
+/*
+ * Build the oracle object on the live session: a class-0x2081 child of the
+ * session's internal subdevice, since the server dispatches the method from
+ * the OBJECT's class (the subdevice attempt alone answered
+ * NV_ERR_NOT_SUPPORTED, 0x56, and a fresh root client at runtime answered
+ * NV_ERR_INVALID_CLIENT, 0x23 - the ladder "adopts" the server's own
+ * internal client instead of creating one). A small candidate list of
+ * likely-free handles in that client's namespace is tried; the first
+ * success wins. Returns 0 with *out_object set.
+ */
+static int gsp_rm_binapi07_backend_setup(struct gsp_emu_context *ctx,
+ struct gsp_phys_backend_static_config *snap,
+ u32 *out_object)
+{
+ static const u32 cand[] = { 0xabcd2081u, 0xabcd2001u, 0xabcd2a2au,
+ 0xabcd5f5fu, 0xabcd1234u };
+ u8 p4[4] = { 0, 0, 0, 0 };
+ u32 st = 0xffffffff, obj = 0;
+ int i, ret;
+
+ for (i = 0; i < (int)ARRAY_SIZE(cand); i++) {
+ u32 want = cand[i];
+
+ obj = want;
+ ret = gsp_rm_binapi07_backend_alloc(snap->h_internal_client,
+ snap->h_internal_subdevice,
+ want, 0x2081, p4, 4,
+ false, &obj, &st);
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: BINAPI-EXPERIMENT oracle BINAPI alloc attempt %d: hObject=0x%x ret=%d status=0x%x",
+ i, want, ret, st);
+ if (!ret && st == 0 && obj) {
+ *out_object = obj;
+ return 0;
+ }
+ }
+ return -EIO;
+}
+
+/*
* Bounded logging for the P3 forward counters: the first four fallbacks (so a
* session that cannot forward is visible) and every 64th event. Never called
* when the parameter is off.
@@ -2962,12 +3113,37 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
u32 out_params_size;
int ret;
+ bool guest_handles = false;
+
/* Wave 1 whitelist: compile-time side-effect-free queries only. */
switch (req->cmd) {
case TU104_CTRL_BUS_GET_PCI_BAR_INFO:
case TU104_CTRL_BUS_GET_INFO_V2:
case TU104_CTRL_FB_GET_INFO_V2:
case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO:
+ if (!gsp_phys_ctrl_forward_enabled())
+ return false;
+ break;
+ case 0x20810107: /* NV2081_BINAPI reserved-interface method 7 */
+ /*
+ * EXPERIMENT (r519): the guest GL core allocates a BINAPI
+ * object (class 0x2081) and sends this 2-byte query during
+ * 3D-surface init; our {0,0} generic reply leaves the
+ * driver's feature flag clear and the EGL surface init tears
+ * down (EGL_BAD_ALLOC at eglMakeCurrent). The guest-side
+ * handles (hClient 0xc1d..., a 0xbeef object) have no backend
+ * counterpart: allocate a real BINAPI object (class 0x2081) on
+ * the live session once, then send the query to it. The first
+ * attempt (subdevice translation) came back NV_ERR_NOT_SUPPORTED
+ * (0x56) - the server dispatches the method by the OBJECT's
+ * class, so a subdevice cannot answer a BINAPI method. The
+ * reply is logged either way; an error falls back to the
+ * emulated generic success until the semantics are known.
+ * Gated by its own parameter so the P3 whitelist can stay off.
+ */
+ if (!gsp_rm_binapi07_fwd_param)
+ return false;
+ guest_handles = true;
break;
default:
return false;
@@ -2979,24 +3155,44 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE)
goto fallback;
+ /* The experiment is scoped to the observed shape: 2-byte in/out. */
+ if (guest_handles && req->paramsSize != 2)
+ goto fallback;
+
/* A valid snapshot implies a READY session and the adopted handles. */
if (gsp_phys_backend_static_get(&snap))
goto fallback;
+ if (guest_handles && !gsp_rm_binapi07_be_object) {
+ u32 obj = 0;
+
+ if (gsp_rm_binapi07_backend_setup(ctx, &snap, &obj))
+ goto fallback;
+ /*
+ * The object lives in the internal client's namespace; the
+ * control is sent as that client (snap.h_internal_client).
+ */
+ gsp_rm_binapi07_be_client = snap.h_internal_client;
+ gsp_rm_binapi07_be_object = obj;
+ }
+
/* Translate the guest's emulated internal handles; any other handle
* domain stays emulated. */
- if (req->hClient != TU104_H_INTERNAL_CLIENT ||
- (req->hObject != TU104_H_INTERNAL_DEVICE &&
- req->hObject != TU104_H_INTERNAL_SUBDEVICE))
+ if (!guest_handles &&
+ (req->hClient != TU104_H_INTERNAL_CLIENT ||
+ (req->hObject != TU104_H_INTERNAL_DEVICE &&
+ req->hObject != TU104_H_INTERNAL_SUBDEVICE)))
goto fallback;
fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL);
if (!fwd_req)
goto fallback;
memset(&fwd_hdr, 0, sizeof(fwd_hdr));
- fwd_hdr.hClient = snap.h_internal_client;
- fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ?
- snap.h_internal_device : snap.h_internal_subdevice;
+ fwd_hdr.hClient = guest_handles ? gsp_rm_binapi07_be_client :
+ snap.h_internal_client;
+ fwd_hdr.hObject = guest_handles ? gsp_rm_binapi07_be_object :
+ (req->hObject == TU104_H_INTERNAL_DEVICE ?
+ snap.h_internal_device : snap.h_internal_subdevice);
fwd_hdr.cmd = req->cmd;
fwd_hdr.paramsSize = req->paramsSize;
fwd_hdr.flags = req->flags;
@@ -3016,6 +3212,27 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx,
}
/*
+ * r519 experiment evidence: log the live GSP's own answer to the
+ * BINAPI query (status + first params bytes) before deciding.
+ */
+ if (req->cmd == 0x20810107) {
+ u32 st = 0xffffffff, ps = 0xffffffff;
+ u8 b[8] = { 0 };
+
+ if (rsp && rsp_len >= GSP_RM_CTRL_HDR_SIZE) {
+ memcpy(&st, (u8 *)rsp + 12, 4);
+ memcpy(&ps, (u8 *)rsp + 16, 4);
+ if (ps <= 8 && ps <= rsp_len - GSP_RM_CTRL_HDR_SIZE)
+ memcpy(b, (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE,
+ ps);
+ }
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: BINAPI-EXPERIMENT 0x20810107 forwarded to live GSP: rsp_len=%u status=0x%x outSize=%u bytes=%02x %02x %02x %02x %02x %02x %02x %02x",
+ rsp_len, st, ps, b[0], b[1], b[2], b[3], b[4],
+ b[5], b[6], b[7]);
+ }
+
+ /*
* The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize
* at +16, echoed params at +24 - the same 24-byte prefix the emulated
* handler builds below. The RPC-level rpc_result was already checked
@@ -3190,7 +3407,7 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
* return means nothing was sent and the emulated switch below runs
* byte-identically to the parameter-off path.
*/
- if (gsp_phys_ctrl_forward_enabled() &&
+ if ((gsp_phys_ctrl_forward_enabled() || gsp_rm_binapi07_fwd_param) &&
gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE,
reply, reply_data_len, sequence)) {
kfree(reply);
@@ -3219,7 +3436,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
break;
case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */
- gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE,
+ gsp_rm_handle_copy_server_pdes(ctx, req.hObject,
+ req_data + GSP_RM_CTRL_HDR_SIZE,
req.paramsSize);
out_status = 0;
out_params_size = 0;
@@ -3581,6 +3799,61 @@ rm_control_reply:
}
/*
+ * r510gl: DUP_OBJECT (function 21) - NVOS55_PARAMETERS echo with status=NV_OK.
+ *
+ * The guest driver dups an object into a second client at EGL/GLX
+ * MakeCurrent time (memory and VASpace objects: the GL context's client
+ * shares the device's allocations), and the historical generic-SUCCESS
+ * default replied with an EMPTY payload - the driver's reply must carry the
+ * params struct back (rpc_dup_object_v03_00 = NVOS55_PARAMETERS
+ * {hClient, hParent, hObject, hClientSrc, hObjectSrc, flags, status}), so
+ * an empty reply failed the call and the driver tore the whole EGL session
+ * down (guest witness: 'RPC UNKNOWN(21) seq=0 len=60 -> generic SUCCESS',
+ * immediately followed by the session's FREE cascade and
+ * eglMakeCurrent -> EGL_BAD_ALLOC (0x3003), 2026-10-23 r510).
+ *
+ * The emulation does not model per-object state for these classes (controls
+ * and allocs are answered from the request), so the echo with NV_OK is the
+ * complete answer; the dup'd handle is used by later calls exactly like the
+ * source handle.
+ */
+static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx,
+ const u8 *elem, u32 sequence)
+{
+ struct gsp_rm_emu *rm = ctx->rm_emu;
+ const struct gsprm_rpc_header *hdr =
+ (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE);
+ const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr);
+ u8 params[28]; /* NVOS55_PARAMETERS_v03_00 */
+ u32 hDstClient = 0, hDstObject = 0, hSrcClient = 0, hSrcObject = 0;
+
+ if (hdr->length >= sizeof(*hdr) + sizeof(params))
+ memcpy(params, req, sizeof(params));
+ else
+ memset(params, 0, sizeof(params));
+
+ memcpy(&hDstClient, params + 0, 4);
+ memcpy(&hDstObject, params + 8, 4);
+ memcpy(&hSrcClient, params + 12, 4);
+ memcpy(&hSrcObject, params + 16, 4);
+ memset(params + 24, 0, 4); /* status = NV_OK */
+
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo",
+ hDstClient, hDstObject, hSrcClient, hSrcObject, sequence);
+
+ /* Small delay before the reply: same reply-matching race mitigation as
+ * FREE/UNLOADING. */
+ udelay(2000);
+
+ if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_DUP_OBJECT, sequence, 0,
+ params, sizeof(params)))
+ nv_vgpu_err(ctx->vgpu, "vgsp-rm: DUP_OBJECT reply send failed");
+ else
+ rm->rpcs_replied++;
+}
+
+/*
* GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK.
*
* rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the
@@ -3861,6 +4134,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
gsp_rm_handle_free(ctx, elem, sequence);
return;
+ case GSPRM_RPC_FUNC_DUP_OBJECT:
+ gsp_rm_handle_dup_object(ctx, elem, sequence);
+ return;
+
case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY:
gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length);
return;
diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h
index 96ea16a..45fcc64 100644
--- a/kernel/gsp_rm_emulation.h
+++ b/kernel/gsp_rm_emulation.h
@@ -45,6 +45,7 @@ struct gsp_emu_context;
#define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76
#define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103
#define GSPRM_RPC_FUNC_FREE 10
+#define GSPRM_RPC_FUNC_DUP_OBJECT 21
#define GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER 47
#define GSPRM_RPC_FUNC_UPDATE_BAR_PDE 70
#define GSPRM_RPC_EVENT_POST_EVENT 0x1003
diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h
index b5ee4d7..3382939 100644
--- a/kernel/gsp_rm_tu104.h
+++ b/kernel/gsp_rm_tu104.h
@@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = {
TU104_CE_CAPS0_SYSMEM | \
TU104_CE_CAPS0_P2P | \
TU104_CE_CAPS0_BL_GT_64K)
-#define TU104_CE_CAPS1 0x00
+/*
+ * Caps byte 1: the 2D/3D copy path requires a CE that supports the
+ * blocklinear access modes (hardware bug 200288104): the X driver's
+ * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL
+ * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then
+ * fails with "Failed to find a suitable copy object" - the observed wall
+ * after the DMA pushbuffer test started passing. Turing's CEs are fully
+ * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03.
+ * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/
+ * 1:0x02.)
+ */
+#define TU104_CE_CAPS1 0x03
/* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */
#define TU104_ENGINE_TYPE_COPY0 0x00000009
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 75c7645..bc21e95 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -394,6 +394,16 @@ struct svc_channel {
* even if still unresolved - bounded zeros, never
* a hang); m156: re-held while the window is
* live so a busy forward gets its real retry */
+ /*
+ * r509: one-shot verdict of the absolute bound. The expiry call
+ * pushes the deadline (so the hold window restarts) and latches this
+ * flag; the r351 retry-forward block consumes it within the same
+ * pass. Without the latch the two call sites in one pass could not
+ * both observe the expiry (the first push of the deadline made the
+ * second call see a fresh window - r213-r215's arming bug in
+ * reverse).
+ */
+ bool gr_hold_bound_hit;
unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data-
* stalled hold for gr_hold_dead_va.
* Fixed once on the first hold of that
@@ -441,6 +451,16 @@ struct svc_channel {
u8 gr_ram_passes; /* slices walked so far */
bool gr_ram_logged; /* one discovery line per channel */
/*
+ * r496e: true when the ring was found by the RAM content scan (a
+ * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's
+ * pushes are 2D/3D method streams the emulation does not execute;
+ * the correct model is consume-and-apply-releases: no QMD capture,
+ * no launch forward, no no-capture hold. Also allows an all-zero
+ * NOP push (the driver's pushbuffer sanity test writes exactly one
+ * zero dword) to be consumed as the no-op it is.
+ */
+ bool gr_ram_ring;
+ /*
* r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the
* registered/ring-derived candidates all fail; the scan validates the
* hit against the ring entries the put claims, so a foreign channel's
@@ -1630,6 +1650,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
* the page tables itself and hands their base address to the GPU). Store it so
* the service can translate that space's VAs directly.
*/
+/*
+ * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A
+ * kernel-RM internal VAS publishes its chain there, but registering every
+ * client's directory directly lets a stale kernel directory answer another
+ * client's VA in the walker's first-success order (the r459 stale-directory
+ * class; observed as a CUDA client hanging after DEVICE). Candidates are
+ * parked here and registered only after a walk of the client's own
+ * content-validated ring VA through the candidate reproduces that ring's PA
+ * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify).
+ */
+#define SVC_VAS_CAND_MAX 16
+static struct {
+ bool valid;
+ u32 h_vaspace;
+ u64 pdb;
+ u32 entries;
+ bool vidmem;
+} svc_vas_cand[SVC_VAS_CAND_MAX];
+
+static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
+ u64 va, int tap, u64 *addr, int *ap, bool trace);
+
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+ u32 entries, bool vidmem)
+{
+ int i;
+
+ if (!pdb || entries < 2 || entries > 4096)
+ return;
+ for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+ if (svc_vas_cand[i].valid &&
+ svc_vas_cand[i].h_vaspace == h_vaspace &&
+ svc_vas_cand[i].pdb == pdb)
+ return; /* already parked */
+ if (!svc_vas_cand[i].valid) {
+ svc_vas_cand[i].valid = true;
+ svc_vas_cand[i].h_vaspace = h_vaspace;
+ svc_vas_cand[i].pdb = pdb;
+ svc_vas_cand[i].entries = entries;
+ svc_vas_cand[i].vidmem = vidmem;
+ return;
+ }
+ }
+}
+
+/*
+ * Verify each parked candidate against one content-validated translation
+ * (the ring VA -> PA the RAM scan just proved): walk the ring page through
+ * the candidate and register only the directory that reproduces it. Runs on
+ * the service thread from the ring-scan latch; the walk itself is cheap (a
+ * few table reads).
+ */
+static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va,
+ u64 pa)
+{
+ int i;
+
+ for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+ u64 wpa = 0;
+ int wap = 0;
+ int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS;
+ struct sema_vaspace_desc d;
+
+ if (!svc_vas_cand[i].valid)
+ continue;
+ if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
+ svc_vas_cand[i].entries, va & ~0xfffULL,
+ ap, &wpa, &wap, false) ||
+ (wpa & ~0xfffULL) != (pa & ~0xfffULL))
+ continue;
+ memset(&d, 0, sizeof(d));
+ d.h_vaspace = svc_vas_cand[i].h_vaspace;
+ d.pdb = svc_vas_cand[i].pdb;
+ d.num_entries = svc_vas_cand[i].entries;
+ d.b_vidmem = svc_vas_cand[i].vidmem;
+ gsp_sema_service_vaspace_add(&d);
+ svc_vas_cand[i].valid = false;
+ pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n",
+ d.h_vaspace, d.pdb, d.num_entries, ap);
+ break;
+ }
+}
+
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
{
struct svc_vaspace *vs = NULL;
@@ -2154,8 +2258,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
}
if (n < SVC_PD_MIN_HITS) {
n = 0;
- if (svc.map_n < SVC_PD_SAMPLES)
- return; /* no evidence yet: retry when pairs appear */
+ /*
+ * r510: a LARGE map is a mixed set of VA spaces cached over
+ * the whole guest boot (the comment above already names the
+ * consequence: no single directory reproduces a mixed set), so
+ * scanning with its stride subsample burns the attempts budget
+ * on evidence that cannot succeed - and by the time the
+ * push-buffer pairs (the strongest samples) exist, the budget
+ * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram
+ * cands=39621 best=0/8)' during the first client, eight RAM
+ * walks consumed, every later client's high argument pointers
+ * unresolvable - the terminal class). Only a SMALL map, which
+ * is one session's own pairs, still falls back; otherwise wait
+ * for the push-buffer pairs and keep the attempts.
+ */
+ if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) ||
+ svc.map_n < SVC_PD_SAMPLES)
+ return; /* no coherent evidence yet: retry later */
step = svc.map_n / SVC_PD_SAMPLES;
for (i = 0; i < SVC_PD_SAMPLES; i++) {
sva[i] = svc.map[i * step].gpu_page;
@@ -7433,6 +7552,79 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
return free_slot;
}
+/*
+ * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op
+ * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode
+ * pass and sets svc.op_tr_pending each time, so its push holds for the full
+ * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op
+ * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx
+ * after the first client, dst never translated) ground its whole stream
+ * at ~10 s per push ("hang after DEVICE" for matmul/multikernel).
+ *
+ * The window must still cover the legitimate pin-late retry (observed
+ * pin latency 0.5-2 s: the H2D copy's operands land a moment after the
+ * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes),
+ * so the latch is TIME-based: the FIRST failure of a VA arms a
+ * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as
+ * before; after a full bound with no success the op still attempts the
+ * write but no longer forces the hold - the entry consumes promptly and
+ * its withheld release applies, exactly the outcome the expired bound
+ * already sanctioned (op stays skipped, release delivered), minus the
+ * 10 s tax per subsequent pass. A success clears the record (a later
+ * failure gets a fresh window). Service-thread only, like every other op
+ * path.
+ */
+#define SVC_OPFAIL_MAX 32
+/*
+ * r507a: the op-failure window rides the opdef hold bound (10000 ms,
+ * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the
+ * latch (used by svc_run_ops) precedes that define.
+ */
+#define SVC_OPFAIL_WINDOW_MS 10000u
+static struct {
+ u64 va;
+ unsigned long since;
+} svc_opfail[SVC_OPFAIL_MAX];
+static u32 svc_opfail_log;
+
+static void svc_opfail_clear(u64 va)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_OPFAIL_MAX; i++)
+ if (svc_opfail[i].va == va)
+ svc_opfail[i].va = 0;
+}
+
+static bool svc_opfail_latched(u64 va)
+{
+ u32 i;
+ u32 free_i = SVC_OPFAIL_MAX;
+
+ for (i = 0; i < SVC_OPFAIL_MAX; i++) {
+ if (svc_opfail[i].va == va) {
+ if (time_after_eq(jiffies,
+ svc_opfail[i].since +
+ msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) {
+ if (svc_opfail_log < 32) {
+ svc_opfail_log++;
+ pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n",
+ va, SVC_OPFAIL_WINDOW_MS);
+ }
+ return true;
+ }
+ return false;
+ }
+ if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX)
+ free_i = i;
+ }
+ if (free_i < SVC_OPFAIL_MAX) {
+ svc_opfail[free_i].va = va;
+ svc_opfail[free_i].since = jiffies;
+ }
+ return false;
+}
+
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base,
u32 entry_idx, u64 pass_hash)
@@ -7690,9 +7882,11 @@ svc_wait_done:
*/
if (rc == SVC_COPY_FORWARDED)
continue;
- if (rc == 0)
+ if (rc == 0) {
svc.memsets_done++;
- else if (rc == -ENOENT)
+ /* r507a: a success re-arms the failure window. */
+ svc_opfail_clear(m->dst);
+ } else if (rc == -ENOENT)
svc.memsets_nomap++;
else
svc.memsets_fail++;
@@ -7723,7 +7917,8 @@ svc_wait_done:
if (rc != 0 && rc != SVC_COPY_FORWARDED &&
(m->bytes >= (256u << 10) ||
(m->type == 1 && m->dst >= SVC_VAS_FLOOR) ||
- (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)))
+ (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) &&
+ !svc_opfail_latched(m->dst))
svc.op_tr_pending = 1;
if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) {
svc.sem256_log++;
@@ -8683,7 +8878,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
inc && inc_apply);
return false;
}
- if (pending || inc_apply) {
+ /*
+ * r506w: generic absolute releases are forward-only by VALUE, whichever
+ * pass discovers the gap. The recorded pending bit came from the
+ * decode's read of the release word, and that read can be blinded by a
+ * stale/crossed landing (r506 witness: a withheld tracker release went
+ * to the flush with pending=0 while the fresh read still showed 0, and
+ * the release was then dropped by the counter fallback below - the
+ * client's tracker never advanced and its stream ground at the 10 s
+ * bound per push). Increments keep the one-shot claim gate (`inc_apply`).
+ */
+ if (pending || inc_apply || (!inc && payload > cur)) {
svc.pushes_pending++;
if (inc)
cur += payload;
@@ -8705,10 +8910,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
return true;
}
}
- if (ch->engine_type == 0)
- svc_counter_learn(vgpu, ch->h_object, sem_va);
- svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
- inc && inc_apply);
+ /*
+ * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its
+ * `release_pa ?: counter_pa` fallback writes (and forward-only-skips
+ * against) the channel's REGISTERED slot, which belongs to a different
+ * word - the r506 witness: releases to the tracker VA 0x20480fef0 were
+ * skipped against the polluted PUT counter (cur=96) and lost. For a
+ * non-counter VA the generic write above is the delivery path.
+ */
+ if (svc_counter_va(sem_va)) {
+ if (ch->engine_type == 0)
+ svc_counter_learn(vgpu, ch->h_object, sem_va);
+ svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+ inc && inc_apply);
+ }
/*
* Stall-class fix (R2): when the decode said this absolute release is
* genuinely pending and every write attempt above failed without
@@ -13046,6 +13261,9 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
u64 off, lim;
u8 *base = vgpu->bars[1].vaddr;
+ u64 best = 0, best_dist = ~0ULL;
+ u32 best_put = 0, best_get = 0;
+ bool best_pending = false;
ch->userd_scans++;
lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
@@ -13054,6 +13272,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
u32 hp, hg;
bool ok = true;
u32 k;
+ u64 dist;
+ bool pending;
memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
@@ -13078,10 +13298,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
eva = ((u64)(a1 & 0xff) << 32) |
(u64)(a0 & 0xfffffffc);
elen = ((a1 >> 10) & 0x1fffff) << 2;
+ /*
+ * Every entry the put claims must name a
+ * VA in the ring's allocation window -
+ * pushbuffers below the ring, the
+ * progress tracker above it.
+ */
if (!eva ||
- eva >= ch->gpfifo_va ||
eva + SVC_GR_RAM_WIN / 2 <
ch->gpfifo_va ||
+ eva >= ch->gpfifo_va +
+ SVC_GR_RAM_WIN / 2 ||
elen < 4 || elen > 0x100000) {
ok = false;
break;
@@ -13089,13 +13316,43 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
}
if (!ok)
continue;
- ch->userd_pa = off;
+ /*
+ * Several blocks can validate: a previous
+ * session leaves its old USERD holding the same
+ * put/entry values while the new channel's block
+ * sits elsewhere (observed r504: the re-
+ * registered channel re-scanned, adopted the
+ * stale block, and its DMA test timed out).
+ * Prefer a block with work PENDING (get < put);
+ * among equals prefer the one nearest the
+ * registered memdesc offset (the driver's own
+ * hint). The pump's GP_GET write-back goes to
+ * the adopted block, so a stale adoption strands
+ * the client's poll.
+ */
+ dist = off > ch->userd_pa ? off - ch->userd_pa
+ : ch->userd_pa - off;
+ pending = hg < hp;
+ if (best &&
+ (best_pending > pending ||
+ (best_pending == pending && dist >= best_dist)))
+ continue;
+ best = off;
+ best_dist = dist;
+ best_put = hp;
+ best_get = hg;
+ best_pending = pending;
+ }
+ if (best) {
+ ch->userd_pa = best;
ch->userd_as = SVC_APERTURE_FB;
ch->userd_ok = true;
- *put = hp;
- *get = hg;
- pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n",
- ch->h_object, ch->gr_index, off, hp, hg);
+ *put = best_put;
+ *get = best_get;
+ pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
+ ch->h_object, ch->gr_index, best,
+ best_put, best_get,
+ best_pending ? "" : ", idle");
return 0;
}
}
@@ -14220,6 +14477,54 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
return false;
}
+/*
+ * r496f: discover a kernel-RM client VAS's page directory in the emulated
+ * VRAM by verification. The CPU RM builds internal-VAS page tables in
+ * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally
+ * owned / UVM-only RPC), so the pushbuffer pages and the progress
+ * semaphore of an X/GL channel cannot be translated from any existing
+ * source. Try each 4 KB page as the walk root and accept the first whose
+ * walk reproduces the ring VA -> ring PA pair we already validated by
+ * content; registering it with the VAS table makes every other VA of the
+ * same client resolvable through the existing walker. Bounded to the
+ * vidmem shadow; verified, so a wrong candidate cannot be registered.
+ */
+static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
+ u64 ring_pa)
+{
+ static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 };
+ u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+ u64 off;
+ unsigned int ei;
+
+ if (!vgpu->bars[1].vaddr || !lim)
+ return;
+ for (off = 0; off + 0x1000 <= lim; off += 0x1000) {
+ for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) {
+ u64 pa = 0;
+ int ap = 0;
+
+ if (!svc_gmmu_walk(vgpu, off, ent_tab[ei],
+ ring_va & ~0xfffULL,
+ SVC_APERTURE_FB, &pa, &ap, false) &&
+ (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) {
+ struct sema_vaspace_desc d;
+
+ memset(&d, 0, sizeof(d));
+ d.h_vaspace = 0x5a5a0000u |
+ (u32)(ring_va >> 20);
+ d.pdb = off;
+ d.num_entries = ent_tab[ei];
+ d.b_vidmem = true;
+ gsp_sema_service_vaspace_add(&d);
+ pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n",
+ off, ent_tab[ei], ring_va, ring_pa);
+ return;
+ }
+ }
+ }
+}
+
static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
struct svc_channel *ch)
{
@@ -14552,24 +14857,63 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
u64 hit_pa = 0;
if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) {
- u64 win_va = ch->gpfifo_va - SVC_GR_RAM_WIN / 2;
- u64 win_pa = hit_pa - SVC_GR_RAM_WIN / 2;
- u64 off;
-
+ /*
+ * Pin ONLY the content-validated ring page (validated,
+ * not a window guess). The pushbuffer pages and the
+ * tracker push of this client resolve through the VAS
+ * page directory registered from COPY_SERVER_PDES
+ * (r496g) - the surface is NONCONTIGUOUS, so a linear
+ * window guess is wrong for them and, being a
+ * pin_lookup_any hit, would shadow the correct walk in
+ * svc_lookup_trusted (observed r502: the tracker push
+ * read zeros through the wrong window page and the
+ * driver's DMA poll never completed).
+ */
svc_pin_store(ch->gpfifo_va, hit_pa,
SVC_APERTURE_SYS);
svc_lat_store(ch->gpfifo_va, hit_pa,
SVC_APERTURE_SYS);
- for (off = 0; off < SVC_GR_RAM_WIN;
- off += SEMA_PAGE_SIZE) {
- svc_pin_store_win(win_va + off, win_pa + off,
- SVC_APERTURE_SYS);
- svc_lat_store(win_va + off, win_pa + off,
- SVC_APERTURE_SYS);
- }
ch->gr_ram_logged = true;
+ ch->gr_ram_ring = true;
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
+ /*
+ * r496j: with the ring content-validated, verify the
+ * parked COPY_SERVER_PDES candidates against it and
+ * register the directory that reproduces the ring's
+ * PA; its walk then resolves the client's other VAs
+ * (pushbuffer pages, the progress semaphore the
+ * driver polls). r496f's blind vidmem sweep stays as
+ * a fallback for directories the control never
+ * published.
+ */
+ svc_vas_candidates_verify(vgpu, ch->gpfifo_va,
+ hit_pa);
+ svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa);
+ /*
+ * Persist the latch immediately: a later stalled
+ * path in this pass may `continue` past the
+ * end-of-pass write-back, and the next pass's
+ * snapshot must carry gr_ram_ring (the zero-NOP
+ * exception and the no-capture-hold skip key on
+ * it).
+ */
+ {
+ unsigned long flags2;
+ int i2;
+
+ spin_lock_irqsave(&svc.chan_lock, flags2);
+ for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++)
+ if (svc.chan[i2].valid &&
+ svc.chan[i2].vgpu == vgpu &&
+ svc.chan[i2].h_object ==
+ ch->h_object) {
+ svc.chan[i2].gr_ram_ring = true;
+ svc.chan[i2].gr_ram_logged = true;
+ break;
+ }
+ spin_unlock_irqrestore(&svc.chan_lock, flags2);
+ }
pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n",
ch->h_object, ch->gr_index, ch->gpfifo_va,
hit_pa);
@@ -18842,6 +19186,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
* forever (the same identity bug class as the opdef=955/13/0
* streams2 wedge). The first hold arms; others ride it; a fresh
* bound arms once the previous one has passed.
+ *
+ * r509: the bound must actually REPORT expiry. The historical arming
+ * rule re-armed the deadline on the very call that detected expiry
+ * and then compared against the NEW deadline - so the function could
+ * never return true, and a data-stalled launch whose arguments never
+ * resolved held its entry forever (r508 witness: matmul plus ten
+ * sibling channels in "GR hold ... data args unresolved" with zero
+ * GR-STALL-hold-bound lines for 10+ minutes; the designed
+ * retry-forward + consume never ran). The expiry is now one-shot:
+ * push the deadline, latch gr_hold_bound_hit for the r351 block of
+ * the same pass, return true. A fresh arm clears the latch (and the
+ * once-per-launch retry log).
*/
if (!ch->gr_hold_dead_va ||
time_after_eq(jiffies, ch->gr_hold_deadline)) {
@@ -18849,6 +19205,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
ch->gr_hold_dead_since = jiffies;
ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
ch->gr_hold_exp_logged = false;
+ ch->gr_hold_bound_hit = false;
+ return false;
}
/*
* r420: pause the per-launch consumption bound while the backend
@@ -18871,7 +19229,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
launch_va);
return false;
}
- return time_after_eq(jiffies, ch->gr_hold_deadline);
+ if (time_after_eq(jiffies, ch->gr_hold_deadline)) {
+ /*
+ * r509: one-shot expiry. Push the deadline so the hold window
+ * restarts for the caller's retry, latch the verdict for the
+ * second call site of this pass (the r351 retry-forward
+ * block), and report expired.
+ */
+ ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms);
+ ch->gr_hold_bound_hit = true;
+ return true;
+ }
+ return false;
}
static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
@@ -20506,8 +20875,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
bool stalled = false;
spin_lock_irqsave(&svc.chan_lock, flags);
+ /*
+ * r507o stream-order: walk the CE channels BEFORE the GR channels.
+ * The client's stream order places its H2D copies before the kernel
+ * that consumes them, but the channels register in handle order (the
+ * RM's GR channel early, the copy-ce channels later), and a pass that
+ * decoded the launch before the CE channel's entries ran let the
+ * forward workqueue execute the launch's copy-in against pre-copy
+ * data (r480-sep: copy-in 115 ms before its stream's H2D soft
+ * copies; r481 proved no host semaphore links the two, so the
+ * emulation must enforce stream order itself). The CE copies execute
+ * on this same service thread, so consuming the CE entries first
+ * fixes the inversion directly - no cross-channel hold, no latency
+ * tax. Stable partition; relative order within each class preserved.
+ */
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
- if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu)
+ if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+ svc.chan[i].engine_type)
+ local[n++] = svc.chan[i];
+ }
+ for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+ if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+ !svc.chan[i].engine_type)
local[n++] = svc.chan[i];
}
spin_unlock_irqrestore(&svc.chan_lock, flags);
@@ -21040,10 +21429,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
{
u64 ident = (u64)e0 | ((u64)e1 << 32);
- bool same = ident &&
+ /*
+ * r505b: the per-slot op bookkeeping arrays are
+ * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL
+ * channel's ring is 8192 slots. Bound every slot
+ * index here or a large ring writes out of
+ * bounds.
+ */
+ bool same = ident && idx < SVC_GPFIFO_MAX_BITS &&
ch->slot_ops_ident[idx] == ident;
- entry_ident = ident;
+ entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0;
/*
* A complete prior pass suppresses every op.
* A partial pass (an operand's targeted scan
@@ -21156,7 +21552,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
injected = (svc_ipin_lookup(va, &ipa,
&iap) == 0);
- if (!svc_method_stream_plausible(pb, len)) {
+ /*
+ * r496e: an all-zero push on a
+ * content-scanned-ring channel is the
+ * driver's NOP sanity push (one zero
+ * dword - NVA06F_DMA_NOP), not a bad
+ * translation: accept it and let the
+ * decode find no ops.
+ */
+ bool zero_nop = ch->gr_ram_ring &&
+ len >= 4 &&
+ !memchr_inv(pb, 0, len);
+
+ if (!svc_method_stream_plausible(pb, len) &&
+ !zero_nop) {
svc.false_trans++;
if (injected) {
u8 *fb = kmalloc(len,
@@ -21484,7 +21893,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
kfree(pb);
break;
}
- if (svc.op_tr_pending)
+ if (svc.op_tr_pending) {
/*
* Deadline hit: the
* operand never
@@ -21496,6 +21905,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
svc_opdef_flush(vgpu, ch,
(u32)idx);
+ /*
+ * r505b: mark the slot's
+ * ops complete. The op was
+ * permanently skipped, but
+ * the decode above stored
+ * complete=0 (op_tr_pending
+ * was set), so every pin-
+ * batch rewind replay of
+ * this slot re-decoded it,
+ * re-set op_tr_pending and
+ * re-held the entry in a
+ * livelock (multikernel on
+ * r505: opdef held=18359,
+ * expired=61, the client
+ * waiting forever while the
+ * pump looped on slot 1).
+ * Nothing is owed after the
+ * flush, so a replay must
+ * skip the ops.
+ */
+ if ((u32)idx <
+ SVC_GPFIFO_MAX_BITS) {
+ ch->slot_ops_complete[idx] =
+ 1;
+ ch->slot_ops_done[idx] =
+ entry_done;
+ }
+ }
rel_seen = (svc_rel_count != rel_n_before);
{
bool fwd_guest_queued;
@@ -21731,10 +22168,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
if (!fwd_guest_queued &&
!ch->engine_type &&
svc_gr_data_stalled &&
- svc_gr_hold_bound_expired(ch,
- svc_gr_launch_va(ch)) &&
+ ch->gr_hold_bound_hit &&
!ch->gr_hold_exp_logged) {
ch->gr_hold_exp_logged = true;
+ /* r509: the latch is this pass's only
+ * report of the expiry; clear it so the
+ * NEXT expiry cycle (gr_hold_exp_logged
+ * now set) falls through to the
+ * historical consume. */
+ ch->gr_hold_bound_hit = false;
svc.fwd_gr_hold_exp++;
/*
* r351: the bound firing used to
@@ -21975,6 +22417,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
jiffies;
}
if (gsp_sema_gr_nocap_hold_ms &&
+ !ch->gr_ram_ring &&
time_before(jiffies,
ch->gr_nocap_since +
msecs_to_jiffies(
@@ -22724,6 +23167,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor;
svc.chan[k].gr_ram_passes = ch->gr_ram_passes;
svc.chan[k].gr_ram_logged = ch->gr_ram_logged;
+ svc.chan[k].gr_ram_ring = ch->gr_ram_ring;
svc.chan[k].userd_scans = ch->userd_scans;
svc.chan[k].logged = ch->logged;
svc.chan[k].logged_expected = ch->logged_expected;
@@ -22802,6 +23246,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
ch->gr_hold_deadline;
svc.chan[k].gr_hold_exp_logged =
ch->gr_hold_exp_logged;
+ /* r509: the one-shot expiry latch is snapshot
+ * state too. */
+ svc.chan[k].gr_hold_bound_hit =
+ ch->gr_hold_bound_hit;
/* r361: the opdef absolute-bound record is
* snapshot state too (the GRFIX lesson) -
* without the write-back the deadline would
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 305a027..ee5c48a 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -81,6 +81,14 @@ struct sema_vaspace_desc {
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
void gsp_sema_service_vaspace_del(u32 h_vaspace);
+/*
+ * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a
+ * content-validated walk proves it belongs to a live client; only then is
+ * it registered (a blind registration let stale kernel directories answer
+ * other clients' VAs in the walker's first-success order).
+ */
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+ u32 entries, bool vidmem);
/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state
* (direct-map base, content-scanned tables) on the svc thread. Client
* session flushes keep that state - see svc_session_flush. */
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-23 07:39:25 +0000