summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_rm_emulation.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_rm_emulation.c')
-rw-r--r--kernel/gsp_rm_emulation.c366
1 files changed, 314 insertions, 52 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 8318f56..cd9cbf2 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -452,8 +452,26 @@ static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size,
/*
* Write a status queue record (1+ elements) and advance the writePtr.
*/
-static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
- u32 sequence, u32 result, const void *data, u32 data_len)
+/*
+ * m171: the event burst budget is a rate over a short time window, not a
+ * monotonic session counter. Without the time reset a flood left the counter
+ * pinned at the cap and silently dropped the completion post of a client
+ * that only started waiting later - a minimal cudaEventSynchronize repro sat
+ * forever while its registrations were valid and 'completion hint rc=-22'
+ * repeated. Channel-completion posts get headroom over hint re-posts: a hint
+ * can be retried a moment later, a completion wake-up cannot.
+ */
+#define GSPRM_EVENT_BURST_CRIT_MAX 48
+#define GSPRM_EVENT_BURST_WINDOW (HZ)
+/* Strictly shorter than the service's 0.5 s hint cadence: an equal window
+ * races the boundary and every later hint skips forever (observed r181:
+ * 'completion hint rc=-2' forever after one success and the stream client
+ * still stalled - the late-waiter wake-up never fired again). */
+#define GSPRM_EVENT_HINT_DEDUP (HZ / 4)
+
+static int gsp_rm_send_flags(struct gsp_emu_context *ctx, u32 function,
+ u32 sequence, u32 result, const void *data,
+ u32 data_len, bool critical)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
u8 *buf;
@@ -489,13 +507,22 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
* newer record and the client aborts with "Bad sequence number".
*/
if (function == GSPRM_RPC_EVENT_POST_EVENT) {
- if (rm->event_burst >= GSPRM_EVENT_BURST_MAX) {
+ u32 cap = critical ? GSPRM_EVENT_BURST_CRIT_MAX
+ : GSPRM_EVENT_BURST_MAX;
+
+ /* m171: expire the budget by wall time so a long-ago
+ * flood can never starve a fresh completion post. */
+ if (time_after(jiffies, rm->event_burst_at +
+ GSPRM_EVENT_BURST_WINDOW))
+ rm->event_burst = 0;
+ rm->event_burst_at = jiffies;
+ if (rm->event_burst >= cap) {
rm->events_dropped++;
if (rm->events_dropped <= 8 ||
(rm->events_dropped % 1000) == 0)
nv_vgpu_info(ctx->vgpu,
- "vgsp-rm: event burst capped (%u), dropped total=%llu",
- GSPRM_EVENT_BURST_MAX,
+ "vgsp-rm: event burst capped (%u/%u), dropped total=%llu",
+ rm->event_burst, cap,
rm->events_dropped);
mutex_unlock(&rm->stat_lock);
kfree(buf);
@@ -504,6 +531,7 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
rm->event_burst++;
} else {
rm->event_burst = 0;
+ rm->event_burst_at = jiffies;
}
record_size = gsp_rm_build_record(rm, buf, buf_size, function,
@@ -524,6 +552,106 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
return -EOVERFLOW;
}
+ /*
+ * fail4: free-space discipline. The guest's consumption cursor for
+ * the status queue (msgqRxHeader.readPtr, at stat+rxHdrOff under the
+ * queue's SWAP_RX layout - same location msgqTxGetFreeSpace polls)
+ * gates this write. Overwriting unread records mixes entries in the
+ * guest's RPC stream: under a completion burst the guest then runs
+ * garbage RM callbacks (rm_run_rc_callback+0x0) and dies. Events are
+ * wake-ups and drop freely; a reply waits briefly for the ring to
+ * drain and is dropped (never overwritten) if it stays full.
+ */
+ {
+ u32 peer_read = 0;
+ bool have_peer = false;
+ int w;
+
+ for (w = 0; w < 51; w++) {
+ u32 used, space, rxoff = 0;
+ u8 shdr[32];
+
+ /*
+ * r332 SWAP_RX cursor fix: the guest's consumption
+ * cursor for the STATUS queue lives in the COMMAND
+ * queue's RX header, not in the status queue's own
+ * rxHdrOff. Under MSGQ_FLAGS_SWAP_RX (both sides set
+ * it: flags=1 in both headers) the driver's
+ * msgqRxMarkConsumed writes its status readPtr to
+ * pOurRxHdr->readPtr, and the driver's pOurRxHdr is
+ * the COMMAND-queue buffer's RX header (msgq.c
+ * msgqRxLink swap branch: pReadOutgoing =
+ * &pOurRxHdr->readPtr with pOurRxHdr from
+ * msgqTxCreate's pBackingStore = pCommandQueue). The
+ * status queue's own +rxHdrOff is the mirror we
+ * publish: OUR command-queue read position
+ * (written below after every consumed command
+ * record). Reading that mirror here made the
+ * free-space math track our own writes: the ring was
+ * declared "full" with the guest mid-drain, replies
+ * were written into it anyway, unread records were
+ * overwritten, and the guest RM read a newer record
+ * ("Bad sequence number. Expected N got N+63") - the
+ * first-client cudaMalloc then fails with "device
+ * busy or unavailable" (the boot lottery) and every
+ * later RPC fails until a fresh boot.
+ */
+ if (gsp_rm_read_buf(ctx, rm->cmd_off, shdr, sizeof(shdr)))
+ break;
+ memcpy(&rxoff, shdr + 24, 4);
+ if (!rxoff || rxoff >= 0x1000)
+ break;
+ if (gsp_rm_read_buf(ctx, rm->cmd_off + rxoff, &peer_read, 4) ||
+ peer_read >= rm->stat_msgcount)
+ break; /* no usable cursor: blind write */
+ have_peer = true;
+ used = (rm->stat_write + rm->stat_msgcount -
+ peer_read) % rm->stat_msgcount;
+ space = rm->stat_msgcount - used - 1u;
+ if (elem_count <= space)
+ break;
+ /*
+ * Only event wake-ups drop freely; a reply waits
+ * (bounded) and then proceeds - dropping replies
+ * breaks the guest's RM init handshake (observed:
+ * two fn=0x1 replies dropped at boot -> the device
+ * never comes up).
+ */
+ if (function == GSPRM_RPC_EVENT_POST_EVENT || w == 50)
+ break;
+ mutex_unlock(&rm->stat_lock);
+ msleep(1);
+ mutex_lock(&rm->stat_lock);
+ }
+ if (have_peer) {
+ u32 used = (rm->stat_write + rm->stat_msgcount -
+ peer_read) % rm->stat_msgcount;
+ u32 space = rm->stat_msgcount - used - 1u;
+
+ if (elem_count > space &&
+ function == GSPRM_RPC_EVENT_POST_EVENT) {
+ rm->events_dropped++;
+ if (rm->events_dropped <= 8 ||
+ (rm->events_dropped % 1000) == 0)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: status ring full (free=%u need=%u peer=%u write=%u mc=%u); event dropped, total=%llu",
+ space, elem_count,
+ peer_read, rm->stat_write,
+ rm->stat_msgcount,
+ rm->events_dropped);
+ kfree(buf);
+ mutex_unlock(&rm->stat_lock);
+ return -EAGAIN;
+ }
+ if (elem_count > space)
+ nv_vgpu_warn(ctx->vgpu,
+ "vgsp-rm: status ring full for reply fn=0x%x (free=%u need=%u peer=%u write=%u mc=%u); writing anyway after wait",
+ function, space, elem_count,
+ peer_read, rm->stat_write,
+ rm->stat_msgcount);
+ }
+ }
+
/* Write all elements to consecutive queue slots. Page-aware: the
* queue buffer is non-contiguous, so translate per page. */
slot = rm->stat_write % rm->stat_msgcount;
@@ -589,6 +717,14 @@ static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
return 0;
}
+/* Non-event sends: the historical entry point, budget untouched. */
+static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function,
+ u32 sequence, u32 result, const void *data, u32 data_len)
+{
+ return gsp_rm_send_flags(ctx, function, sequence, result, data,
+ data_len, false);
+}
+
/*
* Post the GPU event that tells the guest's CPU-side RM a channel's
* submitted work is complete. A real GPU raises an interrupt, the host GSP
@@ -627,9 +763,10 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
memcpy(payload + 8, &rm->events[i].notify_index, 4);
memcpy(payload + 12, &rm->events[i].data, 4);
/* info16@16, status@20, eventDataSize@24, bNotifyList@28 = 0 */
- if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
- payload, sizeof(payload)))
+ if (gsp_rm_send_flags(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+ payload, sizeof(payload), true))
continue;
+ rm->events[i].last_post = jiffies;
posted++;
}
@@ -648,9 +785,11 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src)
memcpy(payload + 4, &rm->events[i].h_event, 4);
memcpy(payload + 8, &rm->events[i].notify_index, 4);
memcpy(payload + 12, &rm->events[i].data, 4);
- if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
- payload, sizeof(payload)))
+ if (gsp_rm_send_flags(ctx, GSPRM_RPC_EVENT_POST_EVENT,
+ 0, 0, payload,
+ sizeof(payload), true))
continue;
+ rm->events[i].last_post = jiffies;
posted++;
}
}
@@ -675,6 +814,7 @@ static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
u8 payload[32];
+ int rc;
if (idx >= rm->events_n || !rm->events[idx].valid)
return -EINVAL;
@@ -685,8 +825,11 @@ static int gsp_rm_post_event_idx(struct gsp_emu_context *ctx, u32 idx)
memcpy(payload + 8, &rm->events[idx].notify_index, 4);
memcpy(payload + 12, &rm->events[idx].data, 4);
- return gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
- payload, sizeof(payload));
+ rc = gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+ payload, sizeof(payload));
+ if (!rc)
+ rm->events[idx].last_post = jiffies;
+ return rc;
}
int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx)
@@ -709,8 +852,14 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx)
}
for (i = 0; i < rm->events_n; i++) {
- int r = gsp_rm_post_event_idx(ctx, i);
+ int r;
+ /* m171: retired slots are not errors - skipping them
+ * keeps the hint's rc meaningful (a valid table must not
+ * report EINVAL because the last allocated slot is dead). */
+ if (!rm->events[i].valid)
+ continue;
+ r = gsp_rm_post_event_idx(ctx, i);
if (r)
last = r;
else
@@ -725,10 +874,53 @@ int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx)
* guest allocates carry no source object, so a per-channel association is
* not available in the ABI). Used by the service's completion hint so a
* client that starts waiting after the event fired still gets a wake-up.
+ *
+ * m171: this is a HINT, and hints repeat every 0.5 s while the window is
+ * open. Re-posting an event that a real completion post already woke moments
+ * ago burns the burst budget and floods the status ring without adding a
+ * wake-up, so skip events posted within GSPRM_EVENT_HINT_DEDUP. A waiter
+ * that arrives after the edge still gets a re-post within half a second.
*/
int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu)
{
- return gsp_rm_force_post(vgpu, 0xffffffffu);
+ struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL;
+ struct gsp_rm_emu *rm;
+ u8 payload[32];
+ u32 i;
+ int n = 0, skipped = 0;
+
+ if (!ctx)
+ return -ENODEV;
+ rm = ctx->rm_emu;
+ if (!rm)
+ return -ENODEV;
+
+ for (i = 0; i < rm->events_n; i++) {
+ if (!rm->events[i].valid)
+ continue;
+ if (time_before(jiffies, rm->events[i].last_post +
+ GSPRM_EVENT_HINT_DEDUP)) {
+ skipped++;
+ continue;
+ }
+
+ memset(payload, 0, sizeof(payload));
+ memcpy(payload + 0, &rm->events[i].h_client, 4);
+ memcpy(payload + 4, &rm->events[i].h_event, 4);
+ memcpy(payload + 8, &rm->events[i].notify_index, 4);
+ memcpy(payload + 12, &rm->events[i].data, 4);
+ if (gsp_rm_send(ctx, GSPRM_RPC_EVENT_POST_EVENT, 0, 0,
+ payload, sizeof(payload)))
+ continue;
+ rm->events[i].last_post = jiffies;
+ n++;
+ }
+
+ if (n)
+ return 0;
+ /* -EAGAIN: valid events existed but every one was dedup-skipped
+ * (normal during an active post burst); -ENOENT: nothing to post. */
+ return skipped ? -EAGAIN : -ENOENT;
}
/*
@@ -2357,6 +2549,7 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object,
return;
memset(&desc, 0, sizeof(desc));
+ desc.vgpu = ctx->vgpu;
desc.h_object = h_object;
memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8);
memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4);
@@ -2443,51 +2636,80 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
* notifyIndex, data}). The driver allocates one per channel and waits
* for the GPU to post the event before it considers submitted work
* complete - remember them so completed work can be signalled back.
+ *
+ * The table is append+retire: FREE retires records (valid=false) but
+ * never shrinks events_n, so long-lived boots with many client
+ * create/destroy cycles would otherwise exhaust the 128 slots and
+ * SILENTLY drop new registrations - after which the completion
+ * wake-ups (post_channel_event / post_all_events) fail with EINVAL
+ * and a later client waits forever for a completion that can no
+ * longer be notified (observed r162: `completion hint rc=-22`
+ * repeating from the first post-teardown client onward). Reclaim the
+ * first retired slot for a new registration, and log loudly when the
+ * table is genuinely full of live entries.
*/
- if (class_id == 0x79 && params_size >= 24 && rm->events_n < GSPRM_MAX_EVENTS) {
+ if (class_id == 0x79 && params_size >= 24) {
u32 h_src = 0, notify_index = 0, hclass = 0;
u64 data64 = 0;
int hc_off = -1, i;
+ u32 slot = rm->events_n;
- /*
- * Locate hClass (0x79) inside the params instead of trusting a
- * fixed offset: NV0005_ALLOC_PARAMETERS is
- * {hParentClient, hSrcResource, hClass, notifyIndex, data},
- * with hSrcResource immediately before hClass.
- */
- for (i = 0; i + 4 <= (int)params_size; i += 4) {
- u32 v;
-
- memcpy(&v, params + i, 4);
- if (v == 0x79) {
- hc_off = i;
+ for (i = 0; i < (int)rm->events_n; i++) {
+ if (!rm->events[i].valid) {
+ slot = (u32)i;
break;
}
}
- if (hc_off >= 4) {
- memcpy(&h_src, params + hc_off - 4, 4);
- memcpy(&hclass, params + hc_off, 4);
- if (hc_off + 8 <= (int)params_size)
- memcpy(&notify_index, params + hc_off + 4, 4);
- if (hc_off + 16 <= (int)params_size)
- memcpy(&data64, params + hc_off + 8, 8);
- } else {
- memcpy(&h_src, params + 4, 4);
- memcpy(&notify_index, params + 12, 4);
- memcpy(&data64, params + 16, 8);
+ if (slot >= GSPRM_MAX_EVENTS)
+ nv_vgpu_err(ctx->vgpu,
+ "vgsp-rm: event table full (%u live records); dropping registration hObject=0x%x - completion wake-ups will not reach this client",
+ rm->events_n, ho);
+ else {
+ /*
+ * Locate hClass (0x79) inside the params instead of
+ * trusting a fixed offset: NV0005_ALLOC_PARAMETERS is
+ * {hParentClient, hSrcResource, hClass, notifyIndex,
+ * data}, with hSrcResource immediately before hClass.
+ */
+ for (i = 0; i + 4 <= (int)params_size; i += 4) {
+ u32 v;
+
+ memcpy(&v, params + i, 4);
+ if (v == 0x79) {
+ hc_off = i;
+ break;
+ }
+ }
+ if (hc_off >= 4) {
+ memcpy(&h_src, params + hc_off - 4, 4);
+ memcpy(&hclass, params + hc_off, 4);
+ if (hc_off + 8 <= (int)params_size)
+ memcpy(&notify_index,
+ params + hc_off + 4, 4);
+ if (hc_off + 16 <= (int)params_size)
+ memcpy(&data64,
+ params + hc_off + 8, 8);
+ } else {
+ memcpy(&h_src, params + 4, 4);
+ memcpy(&notify_index, params + 12, 4);
+ memcpy(&data64, params + 16, 8);
+ }
+ rm->events[slot].h_src = h_src;
+ rm->events[slot].h_event = ho;
+ rm->events[slot].h_client = hc;
+ rm->events[slot].notify_index = notify_index;
+ rm->events[slot].data = (u32)data64;
+ rm->events[slot].valid = true;
+ if (slot == rm->events_n)
+ rm->events_n++;
+ if (gsp_sema_dbg)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d, slot %u) params=%*phN",
+ ho, h_src, notify_index, hc_off,
+ slot,
+ (int)min(params_size, (u32)24),
+ params);
}
- rm->events[rm->events_n].h_src = h_src;
- rm->events[rm->events_n].h_event = ho;
- rm->events[rm->events_n].h_client = hc;
- rm->events[rm->events_n].notify_index = notify_index;
- rm->events[rm->events_n].data = (u32)data64;
- rm->events[rm->events_n].valid = true;
- rm->events_n++;
- if (gsp_sema_dbg)
- nv_vgpu_info(ctx->vgpu,
- "vgsp-rm: event registered hEvent=0x%x hSrc=0x%x notifyIndex=%u (off %d) params=%*phN",
- ho, h_src, notify_index, hc_off,
- (int)min(params_size, (u32)24), params);
}
/* Echo the fixed alloc header + as many param bytes as are contiguous
@@ -3244,7 +3466,7 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
memcpy(&hObjOld, params + 8, 4);
memset(params + 12, 0, 4); /* status = NV_OK */
- gsp_sema_service_channel_del(hObjOld);
+ gsp_sema_service_channel_del(ctx->vgpu, hObjOld);
/*
* Retire event records that refer to the object being freed. Posting an
@@ -3253,11 +3475,19 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem,
* client's notifier events (hSrcResource 0) linger across runs and the
* fallback wake-up nudge would hit them. Also retire a client's events
* when its root object is freed (hObjOld == hRoot).
+ *
+ * m176: the hEvent match must also match the owning client. Handle
+ * values are recycled across runs, so a late FREE of an old client's
+ * hEvent must never retire a new client's record with the same numeric
+ * handle - the new client's completion wake-up would be lost and it
+ * would wait forever (late-client stall class). hRoot is the freeing
+ * client's own handle in NVOS00_PARAMETERS.
*/
for (i = 0; i < rm->events_n; i++) {
if (!rm->events[i].valid)
continue;
- if (rm->events[i].h_event == hObjOld ||
+ if ((rm->events[i].h_event == hObjOld &&
+ (!hRoot || rm->events[i].h_client == hRoot)) ||
(hObjOld && hObjOld == hRoot &&
rm->events[i].h_client == hRoot)) {
rm->events[i].valid = false;
@@ -3657,8 +3887,40 @@ void gsp_rm_emu_fini(struct gsp_emu_context *ctx)
void gsp_rm_emu_reset(struct gsp_emu_context *ctx)
{
struct gsp_rm_emu *rm = ctx->rm_emu;
+ u32 i, stale;
- if (!rm || !rm->parsed)
+ if (!rm)
+ return;
+
+ /*
+ * m176: a reset means the emulated GSP is gone and every RM client
+ * with it (probe/unload, engine reset). Event records left valid from
+ * the previous session would be posted to with stale
+ * (hClient, hEvent) pairs - the guest RM asserts in CliGetEventInfo
+ * and drops the record - and a recycled hEvent can absorb a wake-up
+ * meant for the new client. Drop them regardless of parse state.
+ */
+ stale = 0;
+ for (i = 0; i < GSPRM_MAX_EVENTS; i++) {
+ if (rm->events[i].valid) {
+ rm->events[i].valid = false;
+ stale++;
+ }
+ }
+ if (stale)
+ nv_vgpu_info(ctx->vgpu,
+ "vgsp-rm: reset dropped %u stale event record(s)",
+ stale);
+
+ /*
+ * m177: RM reload means a new guest boot: the guest's direct-map base
+ * and every content-scanned translation may have changed. Raise the
+ * boot-scoped wipe (executed on the sema svc thread; client session
+ * flushes intentionally keep that state - see svc_session_flush).
+ */
+ gsp_sema_service_guest_reset();
+
+ if (!rm->parsed)
return;
nv_vgpu_info(ctx->vgpu, "vgsp-rm: reset (rpcs=%llu replied=%llu unknown=%llu)",
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-21 13:19:14 +0000