summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c106
-rw-r--r--kernel/gsp_phys_booter.h10
-rw-r--r--kernel/gsp_rm_emulation.c210
-rw-r--r--kernel/gsp_rm_tu104.h13
-rw-r--r--kernel/gsp_sema_service.c741
-rw-r--r--kernel/gsp_sema_service.h11
-rw-r--r--kernel/main.c2
7 files changed, 1073 insertions, 20 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 42a0f2a..734ebf2 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -6828,6 +6828,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out,
EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait);
/*
+ * Mode-35 KGR static-data snapshots. The user-lane query set
+ * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with
+ * the device's true GR static data; the replies are kept here so the
+ * guest-side emulation can serve the identical bytes instead of fabricating
+ * GR values. The concrete consumer is the X driver's 3D-engine selection:
+ * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table
+ * and fails with "Failed to select a 3D engine" when it reads back as
+ * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live
+ * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along
+ * with every other GR static field (warps, caps, topology, ctx buffers).
+ *
+ * One entry per captured cmd; the generation key ties the cache to the
+ * backend session exactly like gsp_phys_backend_static above, and the
+ * getter refuses entries once the session has moved on.
+ */
+#define GSP_PHYS_KGR_SNAP_ENTRIES 10
+#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */
+
+struct gsp_phys_kgr_snap {
+ u32 cmd;
+ u32 size;
+ bool valid;
+ u32 generation;
+ u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE];
+};
+
+static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES];
+
+/*
* Drop the cached static-config snapshot. Called when the backend session
* leaves READY (or a new claim starts): the cache generation key is bumped
* past the current backend generation so a concurrent getter cannot serve the
@@ -6836,9 +6865,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait);
*/
void gsp_phys_backend_static_invalidate(void)
{
+ unsigned int i;
+
mutex_lock(&gsp_phys_rm_lock);
gsp_phys_backend_static.valid = false;
gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1;
+ for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++)
+ gsp_phys_kgr_snaps[i].valid = false;
mutex_unlock(&gsp_phys_rm_lock);
}
@@ -6892,6 +6925,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out)
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get);
+/*
+ * Copy the captured reply for cmd into dst. Returns 0 on success (with
+ * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's
+ * generation no longer matches the backend session), -ENOSPC when the
+ * snapshot does not fit dst_len, -EAGAIN while the backend is not READY.
+ */
+int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len)
+{
+ unsigned int i;
+
+ if (!dst || !out_len)
+ return -EINVAL;
+
+ if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
+ return -EAGAIN;
+
+ for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) {
+ struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i];
+
+ if (s->cmd != cmd)
+ continue;
+ if (!s->valid ||
+ s->generation != gsp_phys_backend_generation())
+ return -ENODATA;
+ if (s->size > dst_len)
+ return -ENOSPC;
+ memcpy(dst, s->data, s->size);
+ *out_len = s->size;
+ return 0;
+ }
+ return -ENODATA;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get);
+
+/*
+ * Record one mode-35 KGR query reply. Called under the boot thread only
+ * (same thread as the query itself), so no lock is needed beyond the
+ * generation key.
+ */
+static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size)
+{
+ unsigned int i;
+ struct gsp_phys_kgr_snap *slot = NULL;
+
+ if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE)
+ return;
+
+ for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) {
+ if (gsp_phys_kgr_snaps[i].cmd == cmd) {
+ slot = &gsp_phys_kgr_snaps[i];
+ break;
+ }
+ if (!slot && !gsp_phys_kgr_snaps[i].valid)
+ slot = &gsp_phys_kgr_snaps[i];
+ }
+ if (!slot)
+ return;
+
+ slot->cmd = cmd;
+ slot->size = size;
+ slot->generation = gsp_phys_backend_generation();
+ memcpy(slot->data, data, size);
+ slot->valid = true;
+}
+
/* ============================================================
* Host RM client ladder (R1): gsp-phys-rm-call
* ============================================================ */
@@ -13606,6 +13704,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0)
* (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are
* zero-in best-effort getters at the capture's payload sizes; a failure
* warns and the lane continues.
+ *
+ * Every successful reply is retained via gsp_phys_kgr_snap_store(): the
+ * guest-side emulation serves these exact bytes for the same commands, so
+ * the guest's GL/3D stack reads the live GSP's true GR static values
+ * (SM_VERSION in particular - see the snapshot comment near
+ * gsp_phys_backend_static_get).
*/
static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0)
{
@@ -13644,6 +13748,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0)
q[i].name);
pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n",
q[i].name, q[i].size, status, rpc_result, ret);
+ if (ret == 0 && rpc_result == 0 && status == 0)
+ gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size);
kvfree(buf);
}
}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 80beb54..af74f7c 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -164,6 +164,16 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out,
void gsp_phys_backend_static_invalidate(void);
/*
+ * Mode-35 KGR static-data snapshots (see gsp_phys_booter.c): the live GSP's
+ * replies to the real client's KGR query set, captured at backend boot and
+ * served byte-identically by the guest-side emulation. Returns 0 with
+ * *out_len set, -ENODATA when cmd was not captured for the current backend
+ * generation, -ENOSPC when the snapshot exceeds dst_len, -EAGAIN when the
+ * backend is not READY.
+ */
+int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len);
+
+/*
* Run one fn1+fn65 probe on a READY backend to fill the static snapshot
* cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must
* run on a work item, never inline under a caller-held rm lock or on the mdev
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index cd9cbf2..eddfab5 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2456,6 +2456,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx,
#define GSPRM_COPY_PDES_MAX_LEVELS 6
static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
+ u32 h_object,
const u8 *params, u32 params_size)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
@@ -2506,6 +2507,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
"vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u",
i, phys, size, aperture, page_shift);
}
+
+ /*
+ * r496j: this control IS the kernel RM publishing the page-table
+ * chain of its internal VAS (the level[0..] instances with their
+ * phys addresses and page shifts). The chain can name the X/GL
+ * channel's pushbuffer pages and progress semaphore, for which no
+ * other translation source exists (the fn-54 SET_PAGE_DIRECTORY
+ * path is external-VAS/UVM only). It is NOT registered directly:
+ * a blast registration of every client's directory let a stale
+ * kernel directory answer another client's VA in the walker's
+ * first-success order (the r459 stale-directory class; observed
+ * as a CUDA client hanging after DEVICE with r496g). Instead the
+ * root is offered as a CANDIDATE; the progress service registers
+ * it only after it has been VERIFIED against a content-validated
+ * pair of that very client (the ring found by the RAM content
+ * scan).
+ */
+ if (num_levels >= 1 && rm->pt.levels[0].phys &&
+ rm->pt.levels[0].size >= 16 &&
+ rm->pt.levels[0].size <= 4096 * 8) {
+ /*
+ * The level phys values are guest PAs on this platform
+ * (observed 4+ GiB, above any FB shadow); aperture 3 in the
+ * message does not match the walker's FB/SYS encoding, so
+ * classify by address: values inside the emulated VRAM are
+ * FB, everything else sysmem. The walker also retries the
+ * other aperture, so a misclassification only costs a walk.
+ */
+ bool vidmem = rm->pt.levels[0].phys <
+ (u64)ctx->vgpu->bars[1].size;
+
+ gsp_sema_service_vaspace_candidate_add(
+ h_object, rm->pt.levels[0].phys,
+ (u32)(rm->pt.levels[0].size / 8), vidmem);
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)",
+ h_object, (u64)rm->pt.levels[0].phys,
+ (u32)(rm->pt.levels[0].size / 8),
+ vidmem ? "vidmem" : "sysmem");
+ }
}
static bool gsp_rm_alloc_interesting(u32 class_id);
@@ -2554,6 +2595,19 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4);
+ /*
+ * The wire carries NV2080_ENGINE_TYPE_*: NULL = 0, GR0 (== GRAPHICS)
+ * = 1, COPY0 = 9, NVDEC0 = 0x13, ... The progress service's
+ * convention is "engine_type == 0 means the GR channel" (CUDA's
+ * kernel-RM GR channels arrive with engineType NULL), so an X/GL 3D
+ * channel - which carries the real NV2080_ENGINE_TYPE_GRAPHICS = 1 -
+ * was misclassified as a CE and its push never executed ("Failed to
+ * initialize DMA" in the X driver's pushbuffer NOP test). Normalize
+ * GR0..GR7 (1..8) to 0: the emulated device has a single GR and every
+ * other engine id is >= COPY0 (9), so this cannot capture a CE.
+ */
+ if (desc.engine_type >= 1 && desc.engine_type <= 8)
+ desc.engine_type = 0;
memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4);
memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8);
memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8);
@@ -3048,6 +3102,101 @@ fallback:
return false;
}
+/*
+ * r489: a GSP_RM_CONTROL reply whose rpc record exceeds the driver's
+ * pRpc->maxRpcSize must be sent with the GSP's continuation-record protocol
+ * (nv gsp_rpc.c _rpcWriteContinuationEntries): the FIRST record carries
+ * exactly maxRpcSize bytes of [rpc header + ctrl header + params]; each
+ * following NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD record carries up to
+ * maxRpcSize - sizeof(hdr) bytes of the remainder. The driver's
+ * _issueRpcLarge consumes the first frame as min(bufSize, maxRpcSize) and
+ * then requires one continuation record per remaining chunk - without them
+ * it times out ("rpcRecvPoll timedout for fn 71 continuation record") and
+ * the control fails (NV_ERR_INVALID_PARAMETER), which is how the GL VASpace
+ * construction died on cmd 0x2080a0a4 (paramsSize 67392 > maxRpcSize).
+ *
+ * maxRpcSize is the guest driver's GSP_MSG_QUEUE_RPC_SIZE_MAX; the value is
+ * pinned by the driver's own arithmetic: for the 0x2080a0a4 reply
+ * (full_param_size = 32 + 24 + 67392 = 67448) it reported remainingSize
+ * 0x940 = 2368 after the first frame, i.e. maxRpcSize = 67448 - 2368 =
+ * 65080. A build whose constant differs shows up immediately as a
+ * mismatched continuation count in the guest log; adjust here.
+ */
+#define GSPRM_CRM_CONT_MAXRPC 65080u
+
+static int gsp_rm_send_rm_control(struct gsp_emu_context *ctx, u32 sequence,
+ const void *data, u32 data_len)
+{
+ u32 total = sizeof(struct gsprm_rpc_header) + data_len;
+ const u8 *p = data;
+ u32 off = 0;
+ int ret;
+
+ if (total <= GSPRM_CRM_CONT_MAXRPC)
+ return gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL,
+ sequence, 0, data, data_len);
+
+ /* Frame 1: rpc.length = maxRpcSize. */
+ ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
+ p, GSPRM_CRM_CONT_MAXRPC -
+ sizeof(struct gsprm_rpc_header));
+ if (ret)
+ return ret;
+ off = GSPRM_CRM_CONT_MAXRPC - sizeof(struct gsprm_rpc_header);
+
+ /* Continuations: rpc.length = hdr + chunk. */
+ while (off < data_len) {
+ u32 chunk = data_len - off;
+
+ if (chunk > GSPRM_CRM_CONT_MAXRPC -
+ sizeof(struct gsprm_rpc_header))
+ chunk = GSPRM_CRM_CONT_MAXRPC -
+ sizeof(struct gsprm_rpc_header);
+ ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_CONTINUATION_RECORD,
+ sequence, 0, p + off, chunk);
+ if (ret)
+ return ret;
+ off += chunk;
+ }
+ return 0;
+}
+
+/*
+ * Real-GSP KGR static-data snapshots (mode-35 boot capture; see
+ * gsp_phys_kgr_snap_get in gsp_phys_booter.c). The physical backend asks the
+ * live GSP the real client's KGR query set at boot and retains the replies;
+ * serve the identical bytes for the same commands so the guest's GL/3D stack
+ * reads the device's true GR static values. Concrete consumer: the X
+ * driver's 3D-engine selection maps GR_GET_INFO index SM_VERSION (0x0c)
+ * through a table and fails with "Failed to select a 3D engine" when it
+ * reads back as SM_VERSION_NONE (0); the live GSP's KGR_GET_INFO reply
+ * carries the real value (0x75 on TU104). Returns true when the reply was
+ * served from the snapshot (out_params_size set); false falls through to the
+ * emulated handler unchanged.
+ */
+static bool gsp_rm_kgr_snap_serve(struct gsp_emu_context *ctx, u32 cmd,
+ u32 req_params_size,
+ u8 *out_params, u32 out_len,
+ u32 *out_params_size)
+{
+ u32 n = 0;
+ int ret;
+
+ ret = gsp_phys_kgr_snap_get(cmd, out_params, out_len, &n);
+ if (ret || !n)
+ return false;
+
+ /* Never answer with more bytes than the guest asked for. */
+ if (n > req_params_size)
+ return false;
+
+ *out_params_size = n;
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: KGR cmd=0x%x served from the real-GSP snapshot (%u bytes)",
+ cmd, n);
+ return true;
+}
+
static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
const u8 *elem, u32 sequence)
{
@@ -3089,6 +3238,19 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
return;
}
+ /*
+ * KGR static-data snapshots (mode-35 boot capture): the live GSP's own
+ * replies to the real client's KGR query set are served byte-identically
+ * for the same commands, so the guest's GL/3D stack sees the device's
+ * true GR static values. Falls through to the emulated switch when no
+ * snapshot exists (backend not READY, other boot mode, cmd not captured).
+ */
+ if (gsp_rm_kgr_snap_serve(ctx, req.cmd, req.paramsSize,
+ reply + GSP_RM_CTRL_HDR_SIZE,
+ reply_data_len - GSP_RM_CTRL_HDR_SIZE,
+ &out_params_size))
+ goto rm_control_reply;
+
switch (req.cmd) {
case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE:
gsp_rm_handle_fifo_device_table(ctx, sequence, &req,
@@ -3098,7 +3260,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
break;
case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */
- gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE,
+ gsp_rm_handle_copy_server_pdes(ctx, req.hObject,
+ req_data + GSP_RM_CTRL_HDR_SIZE,
req.paramsSize);
out_status = 0;
out_params_size = 0;
@@ -3409,6 +3572,27 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
break;
}
+ /*
+ * r487/r489 THE GL/VULKAN VASPACE WALL (Xorg/EGL/GZDOOM, every
+ * session to date): the reply must follow the GSP's own large-reply
+ * framing. r487 clamped the echo to fit one 64 KB record; the guest
+ * then refused the record ("Incorrect message length 65528") and,
+ * when it fit, waited for CONTINUATION records that never came
+ * ("rpcRecvPoll timedout for fn 71 continuation record
+ * (remainingSize=0x940)") - rpc.c's _issueRpcLarge consumes the
+ * first frame up to pRpc->maxRpcSize (65080) and then requires
+ * NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD frames for the rest
+ * (the GSP's own _rpcWriteContinuationEntries). The size is left
+ * unclamped here; gsp_rm_send_rm_control() frames it per that
+ * protocol. Root evidence: cmd 0x2080a0a4 (PERF_VPSTATES_INFO,
+ * paramsSize 67392, FINN-serialized) sent during Xorg VASpace
+ * construction; its failure made vmmCreateVaspace return ffff and
+ * killed GL - and, after the length was fixed, the 3D-engine
+ * selection ("Failed to select a 3D engine") until the framing was
+ * correct.
+ */
+
+rm_control_reply:
/* Build the reply header */
memcpy(reply + 0, &req.hClient, 4);
memcpy(reply + 4, &req.hObject, 4);
@@ -3428,8 +3612,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
out_status, dt_us);
}
- if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0,
- reply, GSP_RM_CTRL_HDR_SIZE + out_params_size))
+ if (gsp_rm_send_rm_control(ctx, sequence, reply,
+ GSP_RM_CTRL_HDR_SIZE + out_params_size))
nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply send failed cmd=0x%x",
req.cmd);
else
@@ -3685,14 +3869,18 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
return;
case GSPRM_RPC_FUNC_CONTINUATION_RECORD:
- /* CONTINUATION_RECORD: driver expects a reply. Send SUCCESS. */
- nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> SUCCESS reply",
- gsp_rm_rpc_name(function), function, sequence,
- hdr->length);
- if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0))
- nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function);
- else
- rm->rpcs_replied++;
+ /*
+ * r489: a CONTINUATION_RECORD from the driver is ONE-WAY data
+ * - the tail of a multi-frame request (its _issueRpcLarge
+ * sends frame 1 then continuation records). The old reply
+ * here was a guess ("driver expects a reply"); it injected an
+ * extra fn-71 record into the reply stream that the driver
+ * drained as an event ("Unexpected RPC event from GPU0: 0x47")
+ * and it corrupted the framing of our own large replies
+ * (which use fn-71 continuations the other way). Consume
+ * silently: the driver waits only for the reply to frame 1.
+ */
+ rm->rpcs_replied++;
return;
case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO:
diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h
index b5ee4d7..3382939 100644
--- a/kernel/gsp_rm_tu104.h
+++ b/kernel/gsp_rm_tu104.h
@@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = {
TU104_CE_CAPS0_SYSMEM | \
TU104_CE_CAPS0_P2P | \
TU104_CE_CAPS0_BL_GT_64K)
-#define TU104_CE_CAPS1 0x00
+/*
+ * Caps byte 1: the 2D/3D copy path requires a CE that supports the
+ * blocklinear access modes (hardware bug 200288104): the X driver's
+ * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL
+ * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then
+ * fails with "Failed to find a suitable copy object" - the observed wall
+ * after the DMA pushbuffer test started passing. Turing's CEs are fully
+ * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03.
+ * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/
+ * 1:0x02.)
+ */
+#define TU104_CE_CAPS1 0x03
/* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */
#define TU104_ENGINE_TYPE_COPY0 0x00000009
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 28e5906..35abb15 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -160,6 +160,15 @@ struct sema_map_entry {
#define SVC_USERD_GP_GET 0x88
#define SVC_USERD_GP_PUT 0x8c
+/*
+ * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct
+ * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan
+ * slice per service pass so the service thread stays responsive.
+ */
+#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */
+#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */
+#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */
+
struct svc_channel {
bool valid;
u32 session_gen; /* m176: session generation this entry was
@@ -420,6 +429,49 @@ struct svc_channel {
u64 gr_nocap_va; /* push VA the window was opened for */
unsigned long gr_nocap_since; /* absolute first-seen stamp */
/*
+ * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in
+ * guest RAM. The CPU RM maps sysmem objects locally (no
+ * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so
+ * svc_lookup_trusted cannot name the ring page; the driver's kickoff
+ * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan
+ * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the
+ * service thread stays responsive.
+ */
+ u64 gr_ram_cursor; /* resume point of the content scan */
+ u8 gr_ram_passes; /* slices walked so far */
+ bool gr_ram_logged; /* one discovery line per channel */
+ /*
+ * r496e: true when the ring was found by the RAM content scan (a
+ * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's
+ * pushes are 2D/3D method streams the emulation does not execute;
+ * the correct model is consume-and-apply-releases: no QMD capture,
+ * no launch forward, no no-capture hold. Also allows an all-zero
+ * NOP push (the driver's pushbuffer sanity test writes exactly one
+ * zero dword) to be consumed as the no-op it is.
+ */
+ bool gr_ram_ring;
+ /*
+ * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the
+ * registered/ring-derived candidates all fail; the scan validates the
+ * hit against the ring entries the put claims, so a foreign channel's
+ * USERD cannot be adopted. Bounded retries: the driver kicks off a
+ * few passes after channel registration, so a scan before the first
+ * put must not retire the discovery.
+ */
+ u8 userd_scans;
+ /*
+ * r471: the completion-counter publish must never advance the
+ * guest's word past a push whose content was still unwritten when
+ * the entry was consumed (the streams_events staging-reuse
+ * corruption: the publish satisfied the client's D2H wait before
+ * its copy ran, and the readback drained a staging buffer that
+ * still held the PREVIOUS copy's bytes). Lowest ring slot consumed
+ * with all-zero content; 0 = none. Cleared when a later decode of
+ * a GR push reads non-zero content (the content landed - the rewinds
+ * re-read entries constantly, so this heals within a pass or two).
+ */
+ u32 gr_bad_put;
+ /*
* STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A
* channel that cannot make forward progress (last_get pinned behind
* put, or a residual session put that never drops) is detected here
@@ -1588,6 +1640,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
* the page tables itself and hands their base address to the GPU). Store it so
* the service can translate that space's VAs directly.
*/
+/*
+ * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A
+ * kernel-RM internal VAS publishes its chain there, but registering every
+ * client's directory directly lets a stale kernel directory answer another
+ * client's VA in the walker's first-success order (the r459 stale-directory
+ * class; observed as a CUDA client hanging after DEVICE). Candidates are
+ * parked here and registered only after a walk of the client's own
+ * content-validated ring VA through the candidate reproduces that ring's PA
+ * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify).
+ */
+#define SVC_VAS_CAND_MAX 16
+static struct {
+ bool valid;
+ u32 h_vaspace;
+ u64 pdb;
+ u32 entries;
+ bool vidmem;
+} svc_vas_cand[SVC_VAS_CAND_MAX];
+
+static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
+ u64 va, int tap, u64 *addr, int *ap, bool trace);
+
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+ u32 entries, bool vidmem)
+{
+ int i;
+
+ if (!pdb || entries < 2 || entries > 4096)
+ return;
+ for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+ if (svc_vas_cand[i].valid &&
+ svc_vas_cand[i].h_vaspace == h_vaspace &&
+ svc_vas_cand[i].pdb == pdb)
+ return; /* already parked */
+ if (!svc_vas_cand[i].valid) {
+ svc_vas_cand[i].valid = true;
+ svc_vas_cand[i].h_vaspace = h_vaspace;
+ svc_vas_cand[i].pdb = pdb;
+ svc_vas_cand[i].entries = entries;
+ svc_vas_cand[i].vidmem = vidmem;
+ return;
+ }
+ }
+}
+
+/*
+ * Verify each parked candidate against one content-validated translation
+ * (the ring VA -> PA the RAM scan just proved): walk the ring page through
+ * the candidate and register only the directory that reproduces it. Runs on
+ * the service thread from the ring-scan latch; the walk itself is cheap (a
+ * few table reads).
+ */
+static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va,
+ u64 pa)
+{
+ int i;
+
+ for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+ u64 wpa = 0;
+ int wap = 0;
+ int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB
+ : SVC_APERTURE_SYS;
+ struct sema_vaspace_desc d;
+
+ if (!svc_vas_cand[i].valid)
+ continue;
+ if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
+ svc_vas_cand[i].entries, va & ~0xfffULL,
+ ap, &wpa, &wap, false) ||
+ (wpa & ~0xfffULL) != (pa & ~0xfffULL))
+ continue;
+ memset(&d, 0, sizeof(d));
+ d.h_vaspace = svc_vas_cand[i].h_vaspace;
+ d.pdb = svc_vas_cand[i].pdb;
+ d.num_entries = svc_vas_cand[i].entries;
+ d.b_vidmem = svc_vas_cand[i].vidmem;
+ gsp_sema_service_vaspace_add(&d);
+ svc_vas_cand[i].valid = false;
+ pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n",
+ d.h_vaspace, d.pdb, d.num_entries, ap);
+ break;
+ }
+}
+
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
{
struct svc_vaspace *vs = NULL;
@@ -4124,6 +4260,23 @@ struct svc_wait_op {
};
static struct svc_wait_op svc_waits[SVC_WAIT_MAX];
static u32 svc_wait_n; /* acquires captured this entry */
+/*
+ * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an
+ * entry, the op can still have executed on an earlier decode whose gate did
+ * not trigger (svc_wait_n was 0 on that pass), and the once-only done
+ * bitmap then suppresses it on the first pass whose gate finally passes -
+ * the pass at which the dependency just became installed and the op's
+ * inputs changed. The release then applies and the guest drains a staging
+ * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H
+ * copy read dc2 before the vadd_into wrote it, ran early on gate-less
+ * passes, was once-only-skipped on the passing pass, and the client
+ * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel,
+ * entry) and force exactly ONE re-run of its ops on the first passing
+ * pass.
+ */
+static u32 svc_wait_hold_h;
+static u32 svc_wait_hold_idx = 0xffffffffu;
+static bool svc_wait_repass;
static u32 svc_wait_ovf; /* acquires dropped past the array */
static u64 svc_wait_held;
static u64 svc_wait_pass;
@@ -7382,6 +7535,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
static u64 bigop_log;
svc.cur_ch = ch_h;
+ /* r484: one-shot per call; set only by the hold->pass transition. */
+ svc_wait_repass = false;
/*
* m163: apply every queued scanner injection before resolving this
* pass's copy operands. Without this, a CE copy decoded in the window
@@ -7422,7 +7577,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
struct svc_wait_hold *gh;
bool hold = false;
u32 wi;
-
/*
* Kill switch: a wait that never satisfies (a counter our
* emulation does not advance the way the driver expects)
@@ -7507,9 +7661,22 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
}
if (hold) {
svc_wait_held++;
+ svc_wait_hold_h = ch_h;
+ svc_wait_hold_idx = entry_idx;
svc.op_tr_pending = 1;
return;
}
+ /*
+ * r484: the gate just PASSED for an entry it previously held -
+ * the dependency is installed now, so this pass's ops must
+ * re-run even if the once-only bitmap says they already did
+ * (they ran against the pre-dependency state).
+ */
+ if (ch_h == svc_wait_hold_h &&
+ entry_idx == svc_wait_hold_idx) {
+ svc_wait_repass = true;
+ svc_wait_hold_idx = 0xffffffffu;
+ }
svc_wait_pass++;
if (gh)
gh->active = false;
@@ -7527,7 +7694,8 @@ svc_wait_done:
* ran; re-running a CE copy would read a staging buffer the
* client has since refilled and overwrite finished data.
*/
- if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) {
+ if (!svc_wait_repass && ops_done && seq < 64 &&
+ ((*ops_done >> seq) & 1)) {
svc.ops_once_skipped++;
/*
* r456: this op already ran in an earlier pass. Any
@@ -8973,6 +9141,87 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */
+/*
+ * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method
+ * stream of the first few large GR-class pushes (len >= 1024) so the launch
+ * push's acquire/release method groups (or their absence) are visible
+ * verbatim. The streams_events class evidence shows the st2 vadd's copy-in
+ * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the
+ * driver relied on is only provable by reading the push's own methods.
+ * Bounded: gsp_sema_push_raw dumps, at most once per push VA.
+ */
+unsigned gsp_sema_push_raw;
+static u64 svc_pushraw_last_va;
+static u32 svc_pushraw_n;
+
+static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va,
+ u32 h_object)
+{
+ u32 off = 0, gi = 0;
+
+ if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw)
+ return;
+ if (len < 512 || len > 8192)
+ return;
+ if (push_va == svc_pushraw_last_va)
+ return;
+ svc_pushraw_last_va = push_va;
+ svc_pushraw_n++;
+ pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n",
+ h_object, push_va, len, svc_pushraw_n);
+ while (off + 4 <= len && gi < 160) {
+ u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n;
+
+ memcpy(&hdr, buf + off, 4);
+ op = (hdr >> 29) & 0x7;
+ cnt = (hdr >> 16) & 0xfff;
+ subch = (hdr >> 13) & 0x7;
+ addr = (hdr & 0x1fff) << 2;
+ n = cnt > 4 ? 4 : cnt;
+ for (k = 0; k < n && off + 8 + 4 * k <= len; k++)
+ memcpy(&v[k], buf + off + 4 + 4 * k, 4);
+ pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n",
+ gi, off, op, subch, addr, cnt,
+ v[0], v[1], v[2], v[3]);
+ if (op == 3 && cnt) {
+ off += 4 + cnt * 4;
+ gi++;
+ continue;
+ }
+ if (!cnt) {
+ gi++;
+ off += 4;
+ continue;
+ }
+ if (op == 1) {
+ off += 4 + cnt * 4;
+ gi++;
+ continue;
+ }
+ {
+ u32 p;
+ bool found = false;
+
+ for (p = off + 4; p + 4 <= len; p += 4) {
+ u32 h2;
+
+ memcpy(&h2, buf + p, 4);
+ if (((h2 >> 29) & 0x7) == 1 &&
+ ((h2 >> 16) & 0xfff)) {
+ off = p;
+ found = true;
+ break;
+ }
+ }
+ if (!found)
+ off = len;
+ gi++;
+ }
+ }
+ pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n",
+ h_object, push_va, off, len);
+}
+
static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
u32 len, u64 page_pa, bool allow_ops,
struct svc_channel *ch, u32 entry_idx,
@@ -9037,6 +9286,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
allow_ops = false;
}
+ /*
+ * r481: verbatim method-stream dump for large GR-class pushes
+ * (default 0 = off). See svc_push_raw_dump.
+ */
+ if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512)
+ svc_push_raw_dump(buf, len, push_va, ch->h_object);
+
memset(&st, 0, sizeof(st));
st.push_va = push_va;
/*
@@ -12867,6 +13123,119 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
return 0;
}
}
+
+ /*
+ * r496b: no candidate matched. For a GR channel whose ring is known
+ * (translated and pinned - the sysmem content scan or the VRAM
+ * conventions), the USERD may still be elsewhere: the X/GL driver
+ * allocates its USERD in VRAM and the CPU RM maps it through a BAR1
+ * window, so the driver's GP_PUT write lands at a BAR1 offset that is
+ * neither the registered memdesc base nor ring-derived. Scan the
+ * emulated VRAM for the USERD shape and validate each hit against the
+ * ring: the put it claims must correspond to entries whose pushbuffer
+ * VAs all sit in the ring's allocation window. A foreign channel's
+ * USERD cannot pass that test. One scan per channel (the result
+ * persists through the pump's write-back), logged on latch.
+ */
+ if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
+ u64 off, lim;
+ u8 *base = vgpu->bars[1].vaddr;
+ u64 best = 0, best_dist = ~0ULL;
+ u32 best_put = 0, best_get = 0;
+ bool best_pending = false;
+
+ ch->userd_scans++;
+ lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+ if (base && lim) {
+ for (off = 0; off + 0xa0 <= lim; off += 0x200) {
+ u32 hp, hg;
+ bool ok = true;
+ u32 k;
+ u64 dist;
+ bool pending;
+
+ memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
+ memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
+ if (!hp || hp > ch->gpfifo_entries || hg > hp)
+ continue;
+ /* Validate: every claimed entry decodes to a
+ * pushbuffer VA in the ring's window. */
+ for (k = 0; k < hp && k < 64; k++) {
+ u8 e[8];
+ u64 eva;
+ u32 elen, a0, a1;
+
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)k * 8,
+ e, sizeof(e))) {
+ ok = false;
+ break;
+ }
+ memcpy(&a0, e, 4);
+ memcpy(&a1, e + 4, 4);
+ eva = ((u64)(a1 & 0xff) << 32) |
+ (u64)(a0 & 0xfffffffc);
+ elen = ((a1 >> 10) & 0x1fffff) << 2;
+ /*
+ * Every entry the put claims must name a
+ * VA in the ring's allocation window -
+ * pushbuffers below the ring, the
+ * progress tracker above it.
+ */
+ if (!eva ||
+ eva + SVC_GR_RAM_WIN / 2 <
+ ch->gpfifo_va ||
+ eva >= ch->gpfifo_va +
+ SVC_GR_RAM_WIN / 2 ||
+ elen < 4 || elen > 0x100000) {
+ ok = false;
+ break;
+ }
+ }
+ if (!ok)
+ continue;
+ /*
+ * Several blocks can validate: a previous
+ * session leaves its old USERD holding the same
+ * put/entry values while the new channel's block
+ * sits elsewhere (observed r504: the re-
+ * registered channel re-scanned, adopted the
+ * stale block, and its DMA test timed out).
+ * Prefer a block with work PENDING (get < put);
+ * among equals prefer the one nearest the
+ * registered memdesc offset (the driver's own
+ * hint). The pump's GP_GET write-back goes to
+ * the adopted block, so a stale adoption strands
+ * the client's poll.
+ */
+ dist = off > ch->userd_pa ? off - ch->userd_pa
+ : ch->userd_pa - off;
+ pending = hg < hp;
+ if (best &&
+ (best_pending > pending ||
+ (best_pending == pending && dist >= best_dist)))
+ continue;
+ best = off;
+ best_dist = dist;
+ best_put = hp;
+ best_get = hg;
+ best_pending = pending;
+ }
+ if (best) {
+ ch->userd_pa = best;
+ ch->userd_as = SVC_APERTURE_FB;
+ ch->userd_ok = true;
+ *put = best_put;
+ *get = best_get;
+ pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
+ ch->h_object, ch->gr_index, best,
+ best_put, best_get,
+ best_pending ? "" : ", idle");
+ return 0;
+ }
+ }
+ }
return -ENOENT;
}
@@ -13902,6 +14271,139 @@ unsigned gsp_sema_push_hold_ms = 2500;
static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */
+static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch, u64 *pa_out)
+{
+ static u8 *buf;
+ u64 start = ch->gr_ram_cursor;
+ u64 end = start + SVC_GR_RAM_SLICE;
+ u64 gpa;
+ int i;
+ unsigned long flags;
+
+ if (end > SEMA_PA_MAX)
+ end = SEMA_PA_MAX;
+ if (!buf) {
+ buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+ if (!buf)
+ return false;
+ }
+ for (gpa = start; gpa + SEMA_PAGE_SIZE <= end;
+ gpa += SVC_SCAN_CHUNK) {
+ u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa);
+ u32 o;
+
+ if (svc_read(vgpu, gpa, buf, chunk))
+ continue;
+ for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) {
+ u32 e0, e1, f0, f1;
+ u64 va0, va1;
+ u32 len0, len1;
+
+ memcpy(&e0, buf + o, 4);
+ memcpy(&e1, buf + o + 4, 4);
+ va0 = ((u64)(e1 & 0xff) << 32) |
+ (u64)(e0 & 0xfffffffc);
+ len0 = ((e1 >> 10) & 0x1fffff) << 2;
+ /* entry0: main pushbuffer, just below the ring */
+ if (va0 >= ch->gpfifo_va ||
+ va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va)
+ continue;
+ if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI)
+ continue;
+ if (len0 < 4 || len0 > 0x100000)
+ continue;
+ memcpy(&f0, buf + o + 8, 4);
+ memcpy(&f1, buf + o + 12, 4);
+ va1 = ((u64)(f1 & 0xff) << 32) |
+ (u64)(f0 & 0xfffffffc);
+ len1 = ((f1 >> 10) & 0x1fffff) << 2;
+ /* entry1: progress tracker, above entry0 */
+ if (va1 <= va0 ||
+ va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2)
+ continue;
+ if (len1 < 4 || len1 > 0x100000)
+ continue;
+ *pa_out = gpa + o;
+ return true;
+ }
+ }
+ ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
+ ch->gr_ram_passes++;
+ if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) {
+ ch->gr_ram_logged = true;
+ pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n",
+ ch->h_object, ch->gr_index, ch->gr_ram_passes,
+ ch->gpfifo_va);
+ }
+ /*
+ * Persist the cursor immediately: the pump's stalled-userd path
+ * `continue`s past the end-of-pass write-back, so without this the
+ * scan would restart from slice 0 every pass and never reach the
+ * ring's page.
+ */
+ spin_lock_irqsave(&svc.chan_lock, flags);
+ for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+ if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+ svc.chan[i].h_object == ch->h_object) {
+ svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor;
+ svc.chan[i].gr_ram_passes = ch->gr_ram_passes;
+ svc.chan[i].gr_ram_logged = ch->gr_ram_logged;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&svc.chan_lock, flags);
+ return false;
+}
+
+/*
+ * r496f: discover a kernel-RM client VAS's page directory in the emulated
+ * VRAM by verification. The CPU RM builds internal-VAS page tables in
+ * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally
+ * owned / UVM-only RPC), so the pushbuffer pages and the progress
+ * semaphore of an X/GL channel cannot be translated from any existing
+ * source. Try each 4 KB page as the walk root and accept the first whose
+ * walk reproduces the ring VA -> ring PA pair we already validated by
+ * content; registering it with the VAS table makes every other VA of the
+ * same client resolvable through the existing walker. Bounded to the
+ * vidmem shadow; verified, so a wrong candidate cannot be registered.
+ */
+static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
+ u64 ring_pa)
+{
+ static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 };
+ u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+ u64 off;
+ unsigned int ei;
+
+ if (!vgpu->bars[1].vaddr || !lim)
+ return;
+ for (off = 0; off + 0x1000 <= lim; off += 0x1000) {
+ for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) {
+ u64 pa = 0;
+ int ap = 0;
+
+ if (!svc_gmmu_walk(vgpu, off, ent_tab[ei],
+ ring_va & ~0xfffULL,
+ SVC_APERTURE_FB, &pa, &ap, false) &&
+ (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) {
+ struct sema_vaspace_desc d;
+
+ memset(&d, 0, sizeof(d));
+ d.h_vaspace = 0x5a5a0000u |
+ (u32)(ring_va >> 20);
+ d.pdb = off;
+ d.num_entries = ent_tab[ei];
+ d.b_vidmem = true;
+ gsp_sema_service_vaspace_add(&d);
+ pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n",
+ off, ent_tab[ei], ring_va, ring_pa);
+ return;
+ }
+ }
+ }
+}
+
static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
struct svc_channel *ch)
{
@@ -14178,6 +14680,124 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n",
ch->h_object, hits, lim);
}
+
+ /*
+ * r495 diagnostic (X/GL channel): the X driver's GR channel
+ * registers its USERD at a memdesc base that may be an FB
+ * offset or a BAR1 window. The driver's kickoff writes GP_PUT
+ * through its CPU mapping of that object; if the write landed
+ * in the emulated FB shadow, scanning for the USERD shape
+ * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90,
+ * both nonzero somewhere in the 0x200 block) names the real
+ * offset. One-shot per channel, diagnostic only.
+ */
+ if (gsp_sema_dbg && vgpu->bars[1].vaddr &&
+ svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) {
+ u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
+ u64 off;
+ int hits = 0;
+
+ for (off = 0; off + 0xa0 <= lim && hits < 8;
+ off += 0x200) {
+ u32 hp, hg;
+
+ memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4);
+ memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4);
+ if (!hp || hp > ch->gpfifo_entries || hg > hp)
+ continue;
+ pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n",
+ off, hp, hg,
+ *(u32 *)(vgpu->bars[1].vaddr + off + 0x80),
+ *(u32 *)(vgpu->bars[1].vaddr + off + 0x84),
+ *(u32 *)(vgpu->bars[1].vaddr + off + 0x88),
+ *(u32 *)(vgpu->bars[1].vaddr + off + 0x94));
+ hits++;
+ }
+ pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n",
+ ch->h_object, hits, lim);
+ }
+ }
+
+ /*
+ * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor
+ * field comment). The driver's kickoff writes GPFIFO entries whose
+ * pushbuffer VAs sit just BELOW the ring VA in the same allocation
+ * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0
+ * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to
+ * ring_va + 0x100000 (the progress tracker). A page whose first two
+ * 8-byte words both name such VAs with sane lengths is the ring; pin
+ * it (validated) and its allocation window (guesses) so the ring
+ * entry and pushbuffer reads resolve. Sliced so the service thread
+ * stays responsive; gives up per channel after
+ * SVC_GR_RAM_PASS_MAX slices.
+ */
+ if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX &&
+ !test_bit(ch->gr_index, &svc_gr_ring_done)) {
+ u64 hit_pa = 0;
+
+ if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) {
+ /*
+ * Pin ONLY the content-validated ring page (validated,
+ * not a window guess). The pushbuffer pages and the
+ * tracker push of this client resolve through the VAS
+ * page directory registered from COPY_SERVER_PDES
+ * (r496g) - the surface is NONCONTIGUOUS, so a linear
+ * window guess is wrong for them and, being a
+ * pin_lookup_any hit, would shadow the correct walk in
+ * svc_lookup_trusted (observed r502: the tracker push
+ * read zeros through the wrong window page and the
+ * driver's DMA poll never completed).
+ */
+ svc_pin_store(ch->gpfifo_va, hit_pa,
+ SVC_APERTURE_SYS);
+ svc_lat_store(ch->gpfifo_va, hit_pa,
+ SVC_APERTURE_SYS);
+ ch->gr_ram_logged = true;
+ ch->gr_ram_ring = true;
+ set_bit(ch->gr_index, &svc_gr_ring_done);
+ svc_ring_n++;
+ /*
+ * r496j: with the ring content-validated, verify the
+ * parked COPY_SERVER_PDES candidates against it and
+ * register the directory that reproduces the ring's
+ * PA; its walk then resolves the client's other VAs
+ * (pushbuffer pages, the progress semaphore the
+ * driver polls). r496f's blind vidmem sweep stays as
+ * a fallback for directories the control never
+ * published.
+ */
+ svc_vas_candidates_verify(vgpu, ch->gpfifo_va,
+ hit_pa);
+ svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa);
+ /*
+ * Persist the latch immediately: a later stalled
+ * path in this pass may `continue` past the
+ * end-of-pass write-back, and the next pass's
+ * snapshot must carry gr_ram_ring (the zero-NOP
+ * exception and the no-capture-hold skip key on
+ * it).
+ */
+ {
+ unsigned long flags2;
+ int i2;
+
+ spin_lock_irqsave(&svc.chan_lock, flags2);
+ for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++)
+ if (svc.chan[i2].valid &&
+ svc.chan[i2].vgpu == vgpu &&
+ svc.chan[i2].h_object ==
+ ch->h_object) {
+ svc.chan[i2].gr_ram_ring = true;
+ svc.chan[i2].gr_ram_logged = true;
+ break;
+ }
+ spin_unlock_irqrestore(&svc.chan_lock, flags2);
+ }
+ pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n",
+ ch->h_object, ch->gr_index, ch->gpfifo_va,
+ hit_pa);
+ return;
+ }
}
}
@@ -14185,8 +14805,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
struct svc_channel *ch)
{
u64 pa = 0, va = 0;
- int ap = SVC_APERTURE_SYS;
- u8 e[SVC_GPFIFO_ENTRY_SIZE];
+ int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE];
u32 e0, e1, len;
if (!ch->engine_type) {
@@ -15097,6 +15716,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
return;
val = put;
+ /*
+ * r471: never advance the guest's completion word past a push whose
+ * content was still unwritten when its entry was consumed. The
+ * clamp position is the lowest all-zero GR slot (gr_bad_put = slot
+ * + 1; 0 = none). Without it the publish can satisfy a D2H wait
+ * before its copy ran and the client drains a reused staging buffer
+ * holding the previous copy's bytes (streams_events h_c[2] = c0).
+ * The clamp heals when the rewinds re-read the slot with real
+ * content; the real completions still arrive through the decoded
+ * releases.
+ */
+ if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) {
+ val = ch->gr_bad_put - 1;
+ if (gsp_sema_dbg && svc.counter_pub_dbg < 96) {
+ svc.counter_pub_dbg++;
+ pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n",
+ ch->h_object, pa, val, ch->gr_bad_put, cur);
+ }
+ }
if (val == ch->counter_last)
return;
/*
@@ -20625,10 +21263,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
{
u64 ident = (u64)e0 | ((u64)e1 << 32);
- bool same = ident &&
+ /*
+ * r505b: the per-slot op bookkeeping arrays are
+ * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL
+ * channel's ring is 8192 slots. Bound every slot
+ * index here or a large ring writes out of
+ * bounds.
+ */
+ bool same = ident && idx < SVC_GPFIFO_MAX_BITS &&
ch->slot_ops_ident[idx] == ident;
- entry_ident = ident;
+ entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0;
/*
* A complete prior pass suppresses every op.
* A partial pass (an operand's targeted scan
@@ -20741,7 +21386,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
injected = (svc_ipin_lookup(va, &ipa,
&iap) == 0);
- if (!svc_method_stream_plausible(pb, len)) {
+ /*
+ * r496e: an all-zero push on a
+ * content-scanned-ring channel is the
+ * driver's NOP sanity push (one zero
+ * dword - NVA06F_DMA_NOP), not a bad
+ * translation: accept it and let the
+ * decode find no ops.
+ */
+ bool zero_nop = ch->gr_ram_ring &&
+ len >= 4 &&
+ !memchr_inv(pb, 0, len);
+
+ if (!svc_method_stream_plausible(pb, len) &&
+ !zero_nop) {
svc.false_trans++;
if (injected) {
u8 *fb = kmalloc(len,
@@ -21069,7 +21727,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
kfree(pb);
break;
}
- if (svc.op_tr_pending)
+ if (svc.op_tr_pending) {
/*
* Deadline hit: the
* operand never
@@ -21081,6 +21739,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
*/
svc_opdef_flush(vgpu, ch,
(u32)idx);
+ /*
+ * r505b: mark the slot's
+ * ops complete. The op was
+ * permanently skipped, but
+ * the decode above stored
+ * complete=0 (op_tr_pending
+ * was set), so every pin-
+ * batch rewind replay of
+ * this slot re-decoded it,
+ * re-set op_tr_pending and
+ * re-held the entry in a
+ * livelock (multikernel on
+ * r505: opdef held=18359,
+ * expired=61, the client
+ * waiting forever while the
+ * pump looped on slot 1).
+ * Nothing is owed after the
+ * flush, so a replay must
+ * skip the ops.
+ */
+ if ((u32)idx <
+ SVC_GPFIFO_MAX_BITS) {
+ ch->slot_ops_complete[idx] =
+ 1;
+ ch->slot_ops_done[idx] =
+ entry_done;
+ }
+ }
rel_seen = (svc_rel_count != rel_n_before);
{
bool fwd_guest_queued;
@@ -21512,6 +22198,32 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
break;
}
}
+ /*
+ * r471: publish clamp bookkeeping.
+ * A zero-content GR entry that is
+ * held or consumed must never let
+ * the end-of-pass counter publish
+ * advance the guest's completion
+ * word past it (the client would
+ * satisfy its readback wait before
+ * the real push/copy ran - the
+ * staging buffer still holds the
+ * previous copy's bytes). Record
+ * the lowest zero slot; a decode
+ * that reads non-zero content
+ * clears the clamp.
+ */
+ if (!ch->engine_type) {
+ if (all_zero) {
+ if (!ch->gr_bad_put ||
+ (u32)idx + 1 <
+ ch->gr_bad_put)
+ ch->gr_bad_put =
+ (u32)idx + 1;
+ } else if (va && len) {
+ ch->gr_bad_put = 0;
+ }
+ }
if (!ch->engine_type && va && len &&
gsp_phys_gr_forward_enabled() &&
(svc_gr_qmd_seen > 0 ||
@@ -21534,6 +22246,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
jiffies;
}
if (gsp_sema_gr_nocap_hold_ms &&
+ !ch->gr_ram_ring &&
time_before(jiffies,
ch->gr_nocap_since +
msecs_to_jiffies(
@@ -22277,6 +22990,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].userd_pa = ch->userd_pa;
svc.chan[k].userd_as = ch->userd_as;
svc.chan[k].userd_ok = ch->userd_ok;
+ /* r496b: persist the RAM-scan cursor and the
+ * one-shot USERD-scan latch so they survive
+ * the per-pass snapshot copy. */
+ svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor;
+ svc.chan[k].gr_ram_passes = ch->gr_ram_passes;
+ svc.chan[k].gr_ram_logged = ch->gr_ram_logged;
+ svc.chan[k].gr_ram_ring = ch->gr_ram_ring;
+ svc.chan[k].userd_scans = ch->userd_scans;
svc.chan[k].logged = ch->logged;
svc.chan[k].logged_expected = ch->logged_expected;
svc.chan[k].notified = ch->notified;
@@ -22303,6 +23024,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc.chan[k].push_hold_until = ch->push_hold_until;
svc.chan[k].defer_since = ch->defer_since;
svc.chan[k].put_prev = ch->put_prev;
+ /* r471: the publish clamp watermark is set
+ * and cleared on the pump thread; persist it
+ * so the next pass still clamps. */
+ svc.chan[k].gr_bad_put = ch->gr_bad_put;
/*
* Persist the cheap-wait window and the content
* watermark: the pump works on a snapshot, so
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 3765cd2..ee5c48a 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -81,6 +81,14 @@ struct sema_vaspace_desc {
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
void gsp_sema_service_vaspace_del(u32 h_vaspace);
+/*
+ * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a
+ * content-validated walk proves it belongs to a live client; only then is
+ * it registered (a blind registration let stale kernel directories answer
+ * other clients' VAs in the walker's first-success order).
+ */
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+ u32 entries, bool vidmem);
/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state
* (direct-map base, content-scanned tables) on the svc thread. Client
* session flushes keep that state - see svc_session_flush. */
@@ -313,6 +321,9 @@ extern bool gsp_sema_wait_gate;
/* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the
* emulated launch completion latency for copy-after-launch ordering. */
extern unsigned gsp_sema_wait_hold_ms;
+/* r481: verbatim method-stream dump for the first N large GR pushes
+ * (default 0 off). Diagnostic only; no behaviour change. */
+extern unsigned gsp_sema_push_raw;
/*
* r384 (guest boot-corruption lottery): fill-destination guards (default
diff --git a/kernel/main.c b/kernel/main.c
index 0110372..3ee5c23 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -2832,6 +2832,8 @@ module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644);
MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)");
module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644);
MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)");
+module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644);
+MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)");
MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)");
/*
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-23 02:50:25 +0000