diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 106 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 10 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 166 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 654 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 3 | ||||
| -rw-r--r-- | kernel/main.c | 2 |
6 files changed, 898 insertions, 43 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 42a0f2a..734ebf2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -6828,6 +6828,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6836,9 +6865,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6892,6 +6925,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -13606,6 +13704,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13644,6 +13748,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 80beb54..af74f7c 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -164,6 +164,16 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, void gsp_phys_backend_static_invalidate(void); /* + * Mode-35 KGR static-data snapshots (see gsp_phys_booter.c): the live GSP's + * replies to the real client's KGR query set, captured at backend boot and + * served byte-identically by the guest-side emulation. Returns 0 with + * *out_len set, -ENODATA when cmd was not captured for the current backend + * generation, -ENOSPC when the snapshot exceeds dst_len, -EAGAIN when the + * backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len); + +/* * Run one fn1+fn65 probe on a READY backend to fill the static snapshot * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must * run on a work item, never inline under a caller-held rm lock or on the mdev diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index cd9cbf2..63a4338 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2554,6 +2554,19 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4); + /* + * The wire carries NV2080_ENGINE_TYPE_*: NULL = 0, GR0 (== GRAPHICS) + * = 1, COPY0 = 9, NVDEC0 = 0x13, ... The progress service's + * convention is "engine_type == 0 means the GR channel" (CUDA's + * kernel-RM GR channels arrive with engineType NULL), so an X/GL 3D + * channel - which carries the real NV2080_ENGINE_TYPE_GRAPHICS = 1 - + * was misclassified as a CE and its push never executed ("Failed to + * initialize DMA" in the X driver's pushbuffer NOP test). Normalize + * GR0..GR7 (1..8) to 0: the emulated device has a single GR and every + * other engine id is >= COPY0 (9), so this cannot capture a CE. + */ + if (desc.engine_type >= 1 && desc.engine_type <= 8) + desc.engine_type = 0; memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); @@ -3048,6 +3061,101 @@ fallback: return false; } +/* + * r489: a GSP_RM_CONTROL reply whose rpc record exceeds the driver's + * pRpc->maxRpcSize must be sent with the GSP's continuation-record protocol + * (nv gsp_rpc.c _rpcWriteContinuationEntries): the FIRST record carries + * exactly maxRpcSize bytes of [rpc header + ctrl header + params]; each + * following NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD record carries up to + * maxRpcSize - sizeof(hdr) bytes of the remainder. The driver's + * _issueRpcLarge consumes the first frame as min(bufSize, maxRpcSize) and + * then requires one continuation record per remaining chunk - without them + * it times out ("rpcRecvPoll timedout for fn 71 continuation record") and + * the control fails (NV_ERR_INVALID_PARAMETER), which is how the GL VASpace + * construction died on cmd 0x2080a0a4 (paramsSize 67392 > maxRpcSize). + * + * maxRpcSize is the guest driver's GSP_MSG_QUEUE_RPC_SIZE_MAX; the value is + * pinned by the driver's own arithmetic: for the 0x2080a0a4 reply + * (full_param_size = 32 + 24 + 67392 = 67448) it reported remainingSize + * 0x940 = 2368 after the first frame, i.e. maxRpcSize = 67448 - 2368 = + * 65080. A build whose constant differs shows up immediately as a + * mismatched continuation count in the guest log; adjust here. + */ +#define GSPRM_CRM_CONT_MAXRPC 65080u + +static int gsp_rm_send_rm_control(struct gsp_emu_context *ctx, u32 sequence, + const void *data, u32 data_len) +{ + u32 total = sizeof(struct gsprm_rpc_header) + data_len; + const u8 *p = data; + u32 off = 0; + int ret; + + if (total <= GSPRM_CRM_CONT_MAXRPC) + return gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, + sequence, 0, data, data_len); + + /* Frame 1: rpc.length = maxRpcSize. */ + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, + p, GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)); + if (ret) + return ret; + off = GSPRM_CRM_CONT_MAXRPC - sizeof(struct gsprm_rpc_header); + + /* Continuations: rpc.length = hdr + chunk. */ + while (off < data_len) { + u32 chunk = data_len - off; + + if (chunk > GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)) + chunk = GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header); + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_CONTINUATION_RECORD, + sequence, 0, p + off, chunk); + if (ret) + return ret; + off += chunk; + } + return 0; +} + +/* + * Real-GSP KGR static-data snapshots (mode-35 boot capture; see + * gsp_phys_kgr_snap_get in gsp_phys_booter.c). The physical backend asks the + * live GSP the real client's KGR query set at boot and retains the replies; + * serve the identical bytes for the same commands so the guest's GL/3D stack + * reads the device's true GR static values. Concrete consumer: the X + * driver's 3D-engine selection maps GR_GET_INFO index SM_VERSION (0x0c) + * through a table and fails with "Failed to select a 3D engine" when it + * reads back as SM_VERSION_NONE (0); the live GSP's KGR_GET_INFO reply + * carries the real value (0x75 on TU104). Returns true when the reply was + * served from the snapshot (out_params_size set); false falls through to the + * emulated handler unchanged. + */ +static bool gsp_rm_kgr_snap_serve(struct gsp_emu_context *ctx, u32 cmd, + u32 req_params_size, + u8 *out_params, u32 out_len, + u32 *out_params_size) +{ + u32 n = 0; + int ret; + + ret = gsp_phys_kgr_snap_get(cmd, out_params, out_len, &n); + if (ret || !n) + return false; + + /* Never answer with more bytes than the guest asked for. */ + if (n > req_params_size) + return false; + + *out_params_size = n; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: KGR cmd=0x%x served from the real-GSP snapshot (%u bytes)", + cmd, n); + return true; +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -3089,6 +3197,19 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, return; } + /* + * KGR static-data snapshots (mode-35 boot capture): the live GSP's own + * replies to the real client's KGR query set are served byte-identically + * for the same commands, so the guest's GL/3D stack sees the device's + * true GR static values. Falls through to the emulated switch when no + * snapshot exists (backend not READY, other boot mode, cmd not captured). + */ + if (gsp_rm_kgr_snap_serve(ctx, req.cmd, req.paramsSize, + reply + GSP_RM_CTRL_HDR_SIZE, + reply_data_len - GSP_RM_CTRL_HDR_SIZE, + &out_params_size)) + goto rm_control_reply; + switch (req.cmd) { case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: gsp_rm_handle_fifo_device_table(ctx, sequence, &req, @@ -3409,6 +3530,27 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + /* + * r487/r489 THE GL/VULKAN VASPACE WALL (Xorg/EGL/GZDOOM, every + * session to date): the reply must follow the GSP's own large-reply + * framing. r487 clamped the echo to fit one 64 KB record; the guest + * then refused the record ("Incorrect message length 65528") and, + * when it fit, waited for CONTINUATION records that never came + * ("rpcRecvPoll timedout for fn 71 continuation record + * (remainingSize=0x940)") - rpc.c's _issueRpcLarge consumes the + * first frame up to pRpc->maxRpcSize (65080) and then requires + * NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD frames for the rest + * (the GSP's own _rpcWriteContinuationEntries). The size is left + * unclamped here; gsp_rm_send_rm_control() frames it per that + * protocol. Root evidence: cmd 0x2080a0a4 (PERF_VPSTATES_INFO, + * paramsSize 67392, FINN-serialized) sent during Xorg VASpace + * construction; its failure made vmmCreateVaspace return ffff and + * killed GL - and, after the length was fixed, the 3D-engine + * selection ("Failed to select a 3D engine") until the framing was + * correct. + */ + +rm_control_reply: /* Build the reply header */ memcpy(reply + 0, &req.hClient, 4); memcpy(reply + 4, &req.hObject, 4); @@ -3428,8 +3570,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, out_status, dt_us); } - if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, - reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) + if (gsp_rm_send_rm_control(ctx, sequence, reply, + GSP_RM_CTRL_HDR_SIZE + out_params_size)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply send failed cmd=0x%x", req.cmd); else @@ -3685,14 +3827,18 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) return; case GSPRM_RPC_FUNC_CONTINUATION_RECORD: - /* CONTINUATION_RECORD: driver expects a reply. Send SUCCESS. */ - nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> SUCCESS reply", - gsp_rm_rpc_name(function), function, sequence, - hdr->length); - if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) - nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); - else - rm->rpcs_replied++; + /* + * r489: a CONTINUATION_RECORD from the driver is ONE-WAY data + * - the tail of a multi-frame request (its _issueRpcLarge + * sends frame 1 then continuation records). The old reply + * here was a guess ("driver expects a reply"); it injected an + * extra fn-71 record into the reply stream that the driver + * drained as an event ("Unexpected RPC event from GPU0: 0x47") + * and it corrupted the framing of our own large replies + * (which use fn-71 continuations the other way). Consume + * silently: the driver waits only for the reply to frame 1. + */ + rm->rpcs_replied++; return; case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bcd8ac3..04bffc5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -160,6 +160,15 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -420,6 +429,39 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -486,6 +528,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -819,6 +867,7 @@ struct sema_svc { u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ u64 dmatrace_log; /* fail4: denied-transfer trace */ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ u64 snap_gate_log; /* fail4 r311: capture-gate witness */ @@ -1599,12 +1648,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1932,6 +2009,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -4069,6 +4166,23 @@ struct svc_wait_op { }; static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; @@ -5997,14 +6111,18 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total, } /* - * r444: drop every snapshot still owned by one entry. A snapshot that was - * not consumed by the entry's own copy execution (the copy ran via the - * forwarded hardware path, or a post-execution re-decode re-captured the - * unchanged staging) has no future consumer once the entry's release is - * delivered - leaving it behind let a LATER entry whose copy shares the - * (dst,src,total) triple execute from the stale bytes. + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. */ -static void svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) { u32 i; @@ -7323,6 +7441,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -7363,7 +7483,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, struct svc_wait_hold *gh; bool hold = false; u32 wi; - /* * Kill switch: a wait that never satisfies (a counter our * emulation does not advance the way the driver expects) @@ -7448,9 +7567,22 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (hold) { svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; svc.op_tr_pending = 1; return; } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } svc_wait_pass++; if (gh) gh->active = false; @@ -7468,7 +7600,8 @@ svc_wait_done: * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; /* * r456: this op already ran in an earlier pass. Any @@ -8914,6 +9047,87 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + */ +unsigned gsp_sema_push_raw; +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_n; + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + if (len < 512 || len > 8192) + return; + if (push_va == svc_pushraw_last_va) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -8978,6 +9192,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -9903,6 +10124,23 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; break; /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ case CE_M_LINE_COUNT: @@ -10210,6 +10448,39 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->spitch = spitch; } } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -12758,6 +13029,77 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + if (!eva || + eva >= ch->gpfifo_va || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = hp; + *get = hg; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n", + ch->h_object, ch->gr_index, off, hp, hg); + return 0; + } + } + } return -ENOENT; } @@ -13793,6 +14135,72 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + return false; +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -14069,6 +14477,85 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + u64 win_va = ch->gpfifo_va - SVC_GR_RAM_WIN / 2; + u64 win_pa = hit_pa - SVC_GR_RAM_WIN / 2; + u64 off; + + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + for (off = 0; off < SVC_GR_RAM_WIN; + off += SEMA_PAGE_SIZE) { + svc_pin_store_win(win_va + off, win_pa + off, + SVC_APERTURE_SYS); + svc_lat_store(win_va + off, win_pa + off, + SVC_APERTURE_SYS); + } + ch->gr_ram_logged = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -14076,8 +14563,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -14988,6 +15474,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -15907,28 +16412,28 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].payload : 0u); } /* - * r444: the entry is being consumed; no retry can need its - * staging snapshot any more (see svc_copy_snap_free_owner). - * - * r456: EXCEPT on the APPLY path. The apply runs at the head of a - * re-decode pass of an entry whose copy is still held for its - * operand pins; the pass's own op execution (which consumes the - * snapshot on success) follows the apply. Freeing here destroyed - * the pipeline-order snapshot before it could ever be read and - * the next pass's guard re-capture then read the driver's - * REFILLED ping-pong staging - the r455 operand-shift: the a1 H2D - * executed ~3 s after its push with a2's bytes (owner - * 0x5c00005c/0/6076682fa76b8792, snapshot first4 flipped from - * 00000000 3f000000... to 00000000 3e000000... across retries; - * dst16_after=000000000000003e0000803e0000c03e proves the a2 - * pattern landed in da1's page). The snapshot now survives the - * retries and is consumed (and freed) by the copy that finally - * executes; svc_copy_snap_free_owner still runs on every genuine - * consume path (FINISH/FLUSH/DROP/STALE/COUNTED/SWEEP/STOP) and - * the TTL covers the rest. + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. */ - if (e->active && caller != SVC_OPDEF_CLR_APPLY) - svc_copy_snap_free_owner(e->h_object, e->entry_idx); memset(e, 0, sizeof(*e)); } @@ -21358,10 +21863,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && + (svc_gr_qmd_seen > 0 || + all_zero) && !entry_replay && !(svc_gr_cand.valid && svc_gr_cand.h_object == @@ -21413,6 +21989,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len); } + } /* * A CE push that should carry this * channel's next tracking release but @@ -22122,6 +22699,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -22148,6 +22732,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 3765cd2..305a027 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -313,6 +313,9 @@ extern bool gsp_sema_wait_gate; /* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the * emulated launch completion latency for copy-after-launch ordering. */ extern unsigned gsp_sema_wait_hold_ms; +/* r481: verbatim method-stream dump for the first N large GR pushes + * (default 0 off). Diagnostic only; no behaviour change. */ +extern unsigned gsp_sema_push_raw; /* * r384 (guest boot-corruption lottery): fill-destination guards (default diff --git a/kernel/main.c b/kernel/main.c index 0110372..3ee5c23 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2832,6 +2832,8 @@ module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)"); module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644); MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)"); +module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644); +MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)"); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); /* |