diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 249 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 244 | ||||
| -rwxr-xr-x | tools/cuda-ab/run/tgemm_probe | bin | 829304 -> 829192 bytes |
4 files changed, 470 insertions, 24 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index eddfab5..90556f2 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -1169,6 +1169,7 @@ static const char *gsp_rm_rpc_name(u32 function) case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER: return "UNLOADING_GUEST_DRIVER"; case GSPRM_RPC_FUNC_UPDATE_BAR_PDE: return "UPDATE_BAR_PDE"; + case GSPRM_RPC_FUNC_DUP_OBJECT: return "DUP_OBJECT"; default: return "UNKNOWN"; } } @@ -2955,6 +2956,115 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx, } /* + * r519 experiment switch, independent of the P3 whitelist: forward ONLY the + * BINAPI reserved-interface query 0x20810107 to the live GSP session. The P3 + * whitelist may be off (its mixed real/emulated device answers perturb the + * boot); this one command is what the GL surface init needs answered. + */ +static bool gsp_rm_binapi07_fwd_param; +module_param_named(gsp_rm_binapi07_fwd, gsp_rm_binapi07_fwd_param, bool, 0644); +MODULE_PARM_DESC(gsp_rm_binapi07_fwd, + "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); + +/* + * The live session's oracle client/object for the 0x20810107 experiment, + * created on demand; 0 until the first query builds it. One device per module + * load, so file-statics are enough; reset by unload. + */ +static u32 gsp_rm_binapi07_be_client; +static u32 gsp_rm_binapi07_be_object; + +/* + * One GSP_RM_ALLOC (fn 103) against the live backend, built exactly like the + * rm-call ladder's helper (rpc_gsp_rm_alloc_v03_00: hClient, hParent, + * hObject, hClass, status, paramsSize, flags, reserved[4] + params). + * object_is_out selects the reply read: a ROOT alloc returns the + * server-chosen client handle in params[0] (payload +32), anything else + * echoes the client-chosen hObject at +8; status is at +16 in both cases. + */ +static int gsp_rm_binapi07_backend_alloc(u32 h_client, u32 h_parent, + u32 h_object, u32 h_class, + const void *params, u32 params_len, + bool object_is_out, + u32 *out_object, u32 *out_status) +{ + u8 *areq; + void *arsp = NULL; + u32 arsp_len = 0, st = 0xffffffff, ho = 0; + int ret; + + areq = kzalloc(32 + params_len, GFP_KERNEL); + if (!areq) + return -ENOMEM; + memcpy(areq + 0, &h_client, 4); + memcpy(areq + 4, &h_parent, 4); + memcpy(areq + 8, &h_object, 4); + memcpy(areq + 12, &h_class, 4); + memcpy(areq + 20, ¶ms_len, 4); + if (params_len) + memcpy(areq + 32, params, params_len); + + ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_ALLOC, areq, + 32 + params_len, &arsp, &arsp_len, 500); + kfree(areq); + if (!ret && arsp) { + if (arsp_len >= 20) + memcpy(&st, (u8 *)arsp + 16, 4); + if (object_is_out) { + if (arsp_len >= 36) + memcpy(&ho, (u8 *)arsp + 32, 4); + } else if (arsp_len >= 12) { + memcpy(&ho, (u8 *)arsp + 8, 4); + } + } + kfree(arsp); + if (out_object) + *out_object = ho; + if (out_status) + *out_status = st; + return ret; +} + +/* + * Build the oracle object on the live session: a class-0x2081 child of the + * session's internal subdevice, since the server dispatches the method from + * the OBJECT's class (the subdevice attempt alone answered + * NV_ERR_NOT_SUPPORTED, 0x56, and a fresh root client at runtime answered + * NV_ERR_INVALID_CLIENT, 0x23 - the ladder "adopts" the server's own + * internal client instead of creating one). A small candidate list of + * likely-free handles in that client's namespace is tried; the first + * success wins. Returns 0 with *out_object set. + */ +static int gsp_rm_binapi07_backend_setup(struct gsp_emu_context *ctx, + struct gsp_phys_backend_static_config *snap, + u32 *out_object) +{ + static const u32 cand[] = { 0xabcd2081u, 0xabcd2001u, 0xabcd2a2au, + 0xabcd5f5fu, 0xabcd1234u }; + u8 p4[4] = { 0, 0, 0, 0 }; + u32 st = 0xffffffff, obj = 0; + int i, ret; + + for (i = 0; i < (int)ARRAY_SIZE(cand); i++) { + u32 want = cand[i]; + + obj = want; + ret = gsp_rm_binapi07_backend_alloc(snap->h_internal_client, + snap->h_internal_subdevice, + want, 0x2081, p4, 4, + false, &obj, &st); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT oracle BINAPI alloc attempt %d: hObject=0x%x ret=%d status=0x%x", + i, want, ret, st); + if (!ret && st == 0 && obj) { + *out_object = obj; + return 0; + } + } + return -EIO; +} + +/* * Bounded logging for the P3 forward counters: the first four fallbacks (so a * session that cannot forward is visible) and every 64th event. Never called * when the parameter is off. @@ -3003,12 +3113,37 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, u32 out_params_size; int ret; + bool guest_handles = false; + /* Wave 1 whitelist: compile-time side-effect-free queries only. */ switch (req->cmd) { case TU104_CTRL_BUS_GET_PCI_BAR_INFO: case TU104_CTRL_BUS_GET_INFO_V2: case TU104_CTRL_FB_GET_INFO_V2: case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO: + if (!gsp_phys_ctrl_forward_enabled()) + return false; + break; + case 0x20810107: /* NV2081_BINAPI reserved-interface method 7 */ + /* + * EXPERIMENT (r519): the guest GL core allocates a BINAPI + * object (class 0x2081) and sends this 2-byte query during + * 3D-surface init; our {0,0} generic reply leaves the + * driver's feature flag clear and the EGL surface init tears + * down (EGL_BAD_ALLOC at eglMakeCurrent). The guest-side + * handles (hClient 0xc1d..., a 0xbeef object) have no backend + * counterpart: allocate a real BINAPI object (class 0x2081) on + * the live session once, then send the query to it. The first + * attempt (subdevice translation) came back NV_ERR_NOT_SUPPORTED + * (0x56) - the server dispatches the method by the OBJECT's + * class, so a subdevice cannot answer a BINAPI method. The + * reply is logged either way; an error falls back to the + * emulated generic success until the semantics are known. + * Gated by its own parameter so the P3 whitelist can stay off. + */ + if (!gsp_rm_binapi07_fwd_param) + return false; + guest_handles = true; break; default: return false; @@ -3020,24 +3155,44 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE) goto fallback; + /* The experiment is scoped to the observed shape: 2-byte in/out. */ + if (guest_handles && req->paramsSize != 2) + goto fallback; + /* A valid snapshot implies a READY session and the adopted handles. */ if (gsp_phys_backend_static_get(&snap)) goto fallback; + if (guest_handles && !gsp_rm_binapi07_be_object) { + u32 obj = 0; + + if (gsp_rm_binapi07_backend_setup(ctx, &snap, &obj)) + goto fallback; + /* + * The object lives in the internal client's namespace; the + * control is sent as that client (snap.h_internal_client). + */ + gsp_rm_binapi07_be_client = snap.h_internal_client; + gsp_rm_binapi07_be_object = obj; + } + /* Translate the guest's emulated internal handles; any other handle * domain stays emulated. */ - if (req->hClient != TU104_H_INTERNAL_CLIENT || - (req->hObject != TU104_H_INTERNAL_DEVICE && - req->hObject != TU104_H_INTERNAL_SUBDEVICE)) + if (!guest_handles && + (req->hClient != TU104_H_INTERNAL_CLIENT || + (req->hObject != TU104_H_INTERNAL_DEVICE && + req->hObject != TU104_H_INTERNAL_SUBDEVICE))) goto fallback; fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL); if (!fwd_req) goto fallback; memset(&fwd_hdr, 0, sizeof(fwd_hdr)); - fwd_hdr.hClient = snap.h_internal_client; - fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ? - snap.h_internal_device : snap.h_internal_subdevice; + fwd_hdr.hClient = guest_handles ? gsp_rm_binapi07_be_client : + snap.h_internal_client; + fwd_hdr.hObject = guest_handles ? gsp_rm_binapi07_be_object : + (req->hObject == TU104_H_INTERNAL_DEVICE ? + snap.h_internal_device : snap.h_internal_subdevice); fwd_hdr.cmd = req->cmd; fwd_hdr.paramsSize = req->paramsSize; fwd_hdr.flags = req->flags; @@ -3057,6 +3212,27 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, } /* + * r519 experiment evidence: log the live GSP's own answer to the + * BINAPI query (status + first params bytes) before deciding. + */ + if (req->cmd == 0x20810107) { + u32 st = 0xffffffff, ps = 0xffffffff; + u8 b[8] = { 0 }; + + if (rsp && rsp_len >= GSP_RM_CTRL_HDR_SIZE) { + memcpy(&st, (u8 *)rsp + 12, 4); + memcpy(&ps, (u8 *)rsp + 16, 4); + if (ps <= 8 && ps <= rsp_len - GSP_RM_CTRL_HDR_SIZE) + memcpy(b, (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE, + ps); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT 0x20810107 forwarded to live GSP: rsp_len=%u status=0x%x outSize=%u bytes=%02x %02x %02x %02x %02x %02x %02x %02x", + rsp_len, st, ps, b[0], b[1], b[2], b[3], b[4], + b[5], b[6], b[7]); + } + + /* * The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize * at +16, echoed params at +24 - the same 24-byte prefix the emulated * handler builds below. The RPC-level rpc_result was already checked @@ -3231,7 +3407,7 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, * return means nothing was sent and the emulated switch below runs * byte-identically to the parameter-off path. */ - if (gsp_phys_ctrl_forward_enabled() && + if ((gsp_phys_ctrl_forward_enabled() || gsp_rm_binapi07_fwd_param) && gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE, reply, reply_data_len, sequence)) { kfree(reply); @@ -3623,6 +3799,61 @@ rm_control_reply: } /* + * r510gl: DUP_OBJECT (function 21) - NVOS55_PARAMETERS echo with status=NV_OK. + * + * The guest driver dups an object into a second client at EGL/GLX + * MakeCurrent time (memory and VASpace objects: the GL context's client + * shares the device's allocations), and the historical generic-SUCCESS + * default replied with an EMPTY payload - the driver's reply must carry the + * params struct back (rpc_dup_object_v03_00 = NVOS55_PARAMETERS + * {hClient, hParent, hObject, hClientSrc, hObjectSrc, flags, status}), so + * an empty reply failed the call and the driver tore the whole EGL session + * down (guest witness: 'RPC UNKNOWN(21) seq=0 len=60 -> generic SUCCESS', + * immediately followed by the session's FREE cascade and + * eglMakeCurrent -> EGL_BAD_ALLOC (0x3003), 2026-10-23 r510). + * + * The emulation does not model per-object state for these classes (controls + * and allocs are answered from the request), so the echo with NV_OK is the + * complete answer; the dup'd handle is used by later calls exactly like the + * source handle. + */ +static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const struct gsprm_rpc_header *hdr = + (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr); + u8 params[28]; /* NVOS55_PARAMETERS_v03_00 */ + u32 hDstClient = 0, hDstObject = 0, hSrcClient = 0, hSrcObject = 0; + + if (hdr->length >= sizeof(*hdr) + sizeof(params)) + memcpy(params, req, sizeof(params)); + else + memset(params, 0, sizeof(params)); + + memcpy(&hDstClient, params + 0, 4); + memcpy(&hDstObject, params + 8, 4); + memcpy(&hSrcClient, params + 12, 4); + memcpy(&hSrcObject, params + 16, 4); + memset(params + 24, 0, 4); /* status = NV_OK */ + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo", + hDstClient, hDstObject, hSrcClient, hSrcObject, sequence); + + /* Small delay before the reply: same reply-matching race mitigation as + * FREE/UNLOADING. */ + udelay(2000); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_DUP_OBJECT, sequence, 0, + params, sizeof(params))) + nv_vgpu_err(ctx->vgpu, "vgsp-rm: DUP_OBJECT reply send failed"); + else + rm->rpcs_replied++; +} + +/* * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. * * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the @@ -3903,6 +4134,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) gsp_rm_handle_free(ctx, elem, sequence); return; + case GSPRM_RPC_FUNC_DUP_OBJECT: + gsp_rm_handle_dup_object(ctx, elem, sequence); + return; + case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length); return; diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index 96ea16a..45fcc64 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -45,6 +45,7 @@ struct gsp_emu_context; #define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76 #define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103 #define GSPRM_RPC_FUNC_FREE 10 +#define GSPRM_RPC_FUNC_DUP_OBJECT 21 #define GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER 47 #define GSPRM_RPC_FUNC_UPDATE_BAR_PDE 70 #define GSPRM_RPC_EVENT_POST_EVENT 0x1003 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1e9f981..bc21e95 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -394,6 +394,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -2248,8 +2258,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -7527,6 +7552,79 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) return free_slot; } +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (10000 ms, + * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the + * latch (used by svc_run_ops) precedes that define. + */ +#define SVC_OPFAIL_WINDOW_MS 10000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, u32 entry_idx, u64 pass_hash) @@ -7784,9 +7882,11 @@ svc_wait_done: */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + } else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; @@ -7817,7 +7917,8 @@ svc_wait_done: if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || - (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR))) + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -8777,7 +8878,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -8799,10 +8910,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -19065,6 +19186,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) * forever (the same identity bug class as the opdef=955/13/0 * streams2 wedge). The first hold arms; others ride it; a fresh * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). */ if (!ch->gr_hold_dead_va || time_after_eq(jiffies, ch->gr_hold_deadline)) { @@ -19072,6 +19205,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; } /* * r420: pause the per-launch consumption bound while the backend @@ -19094,7 +19229,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va); return false; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; + } + return false; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -20729,8 +20875,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) + local[n++] = svc.chan[i]; + } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + !svc.chan[i].engine_type) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -21263,10 +21429,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -21720,7 +21893,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -21732,6 +21905,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -21967,10 +22168,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; /* * r351: the bound firing used to @@ -23040,6 +23246,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; /* r361: the opdef absolute-bound record is * snapshot state too (the GRFIX lesson) - * without the write-back the deadline would diff --git a/tools/cuda-ab/run/tgemm_probe b/tools/cuda-ab/run/tgemm_probe Binary files differindex 00f3539..d9c9aa3 100755 --- a/tools/cuda-ab/run/tgemm_probe +++ b/tools/cuda-ab/run/tgemm_probe |