diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 116 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 19 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 422 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 6 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1610 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 13 | ||||
| -rw-r--r-- | kernel/main.c | 150 |
8 files changed, 2272 insertions, 65 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 7ba74ce..6fae2f2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16744,6 +16744,61 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -16776,6 +16831,25 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the @@ -19368,6 +19442,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; + int rel_map_ret = 0; + if (!rel_mod && req->rel_hpa && one_word && !red_en && n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { u64 sh_va = 0; @@ -19387,6 +19463,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, 0xffu); qmd[25] = req->rel_payload; } else { + rel_map_ret = ret; ret = 0; } } @@ -19415,6 +19492,23 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * completed on the guest word (the r414 saxpy hang). */ qmd_rel_redirected = true; + /* + * r633 diag: name the exact redirect cause. rel_hpa==0 + * means the caller could not resolve the release page + * (capture resolution or PA->HPA); map_ret!=0 means the + * page was resolved but its shadow mapping failed. + */ + { + static u32 qmdrel_redir_log; + + if (qmdrel_redir_log < 32) { + qmdrel_redir_log++; + pr_info("vgsp-gr: qmd-rel redirect: rel_hpa=0x%llx map_ret=%d one_word=%u red_en=%u rel_mod=%d n_shadow=%u pay=%u\n", + req->rel_hpa, rel_map_ret, + one_word, red_en, rel_mod ? 1 : 0, + n_shadow, req->rel_payload); + } + } } } @@ -20544,6 +20638,28 @@ gr_page_unmapped: gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, qmd_rel, want_ch, poll_ms, qmdwin_note); + { + /* + * r596 diag: the raw-prelude geometry the verbatim gate tests. + * The verbatim replay needs + * raw && raw_words >= hdr + 67 && + * raw_qmd_end == hdr + 67 && + * INIT_WORDS + hdr + 67 + TAIL_WORDS <= METHOD_WORDS + * Bounded to the first eight launches so the geometry is on + * record without flooding a battery run. + */ + static u32 prelude_geom_log; + + if (gsp_phys_rmcall_gr_prelude_value() && prelude_geom_log < 8) { + prelude_geom_log++; + pr_info("vgsp-gr: prelude geom raw=%u raw_words=%u hdr=%u end=%u want_end=%u v=%d\n", + req->raw ? 1u : 0u, req->raw_words, + req->raw_qmd_hdr, req->raw_qmd_end, + req->raw_qmd_hdr + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS, + gsp_phys_rmcall_gr_prelude_value()); + } + } if (qmd_win_diag) pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 74062dd..e9c0c5a 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -617,6 +617,25 @@ int gsp_phys_rmcall_gr_prelude_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value); /* + * r597: emit the SEM_EXECUTE ACQUIRE groups found in the captured guest + * prelude ahead of the rebuilt inline-QMD window (default 0). The verbatim + * prelude replay breaks the launch (qmd_rel=0 after an 8.9 s stall, r599), + * so the acquire cannot be taken by replaying the prelude; this gate takes + * only the wait method groups and leaves the proven window in place. + */ +static int gsp_phys_rmcall_gr_acq_param; +module_param_named(gsp_phys_rmcall_gr_acq, gsp_phys_rmcall_gr_acq_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_acq, + "Emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt GR window (default 0)"); + +int gsp_phys_rmcall_gr_acq_value(void) +{ + return gsp_phys_rmcall_gr_acq_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_acq_value); + +/* * QMD virtualization control gate (default 0 after m26 falsified the * hypothesis: on Turing the context init value is 0, native CUDA never * writes 0x29c, and the nonzero enable alone does not schedule). When set, diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 36c2220..3993c6e 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -295,6 +295,7 @@ int gsp_phys_rmcall_gr_promote_value(void); * init+window+PCAS+sem shape); 1 replays the verbatim prelude. */ int gsp_phys_rmcall_gr_prelude_value(void); +int gsp_phys_rmcall_gr_acq_value(void); /* * gsp_phys_rmcall_gr_qmdvirt (default 1): emit diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 2a957b3..2e47f98 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -926,6 +926,48 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) } /* + * r623 experiment: deliver a synthetic OS_ERROR_LOG (0x1006) record to the + * guest's RM queue - the error report a real GSP sends into the client's + * message queue when an engine faults. r618's synth-fault writes the channel + * notifier and posts a generic event; after it, the NEXT client stuck + * pre-submission (r620 atomics; r616b saxpy/math). If the guest's fault + * handling marks the device recoverable only when the 0x1006 report arrives, + * this record is the missing half. Payload mirrors the GSP ABI decode in + * gsp_phys_booter.c (GSP_PHYS_OSERR_OFF_*): exceptType u32, runlistId u32, + * chid u32, errString[256], prevXid u32. Manual/debug only. + */ +int gsp_rm_send_os_error_log(struct nv_vgpu_device *vgpu, u32 except_type, + u32 runlist_id, u32 chid, const char *engine) +{ + struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL; + struct gsp_rm_emu *rm; + u8 payload[272] = {0}; + int rc; + + if (!ctx) + return -ENODEV; + rm = ctx->rm_emu; + if (!rm || !rm->parsed) + return -EAGAIN; + + memcpy(payload + 0, &except_type, 4); + memcpy(payload + 4, &runlist_id, 4); + memcpy(payload + 8, &chid, 4); + snprintf((char *)payload + 12, 255, "ENGINE %s", + engine && *engine ? engine : "GR"); + /* prevXid stays 0 */ + + rc = gsp_rm_send(ctx, 0x1006, (u32)rm->stat_seq + 1, 0, payload, + sizeof(payload)); + if (!rc) + nv_vgpu_info(vgpu, + "vgsp-rm: OS_ERROR_LOG sent (synthetic) except=0x%x rl=%u chid=%u engine=%s\n", + except_type, runlist_id, chid, + engine && *engine ? engine : "GR"); + return rc; +} + +/* * Debug: post the event record for one registered event (or all of them with * idx 0xffffffff). Used to test the guest's event path on demand without * waiting for a channel submission. @@ -1871,7 +1913,20 @@ static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx, * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4) * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20) * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32). - * Report a minimal headless display: 1 head, 1 (core) channel. + * + * r651: this reply is also the gate for the WINDOW-channel alloc path. + * Live gdb on the guest nvidia.ko pinned the display path 0x56 to + * kdispGetChannelNum_v03_00: for dispChnClass_Win it requires + * pStaticInfo->windowPresentMask & NVBIT32(instance) and returns + * NV_ERR_NOT_SUPPORTED (0x56) when the bit is clear; the Curs class reads + * HEAD_EXISTS bits from feHwSysCap the same way (0x49+instance channel num). + * The previous reply left both masks zero, so nvRmApiAlloc(0xc57e) died in + * the guest's local resource construct (clientAllocResource -> + * resservResourceFactory -> dispchnConstruct -> kdispGetChannelNum) before + * any GSP-RM RPC was issued. Report the same one-head / 8-window model the + * NV0073 shim and the caps notifier expose (feHwSysCap head-0 exists, + * windowPresentMask bits 0..7, numDispChannels covering the cursor channel + * numbers 73..80, i.e. 81 entries). */ static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx, u32 sequence, @@ -1883,15 +1938,205 @@ static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx, memset(out_params, 0, 36); + /* + * feHwSysCap: NV_PDISP_FE_HW_SYS_CAP_HEAD_EXISTS(i) occupies bit i + * (dev_disp.h v03_00); head 0 exists on the emulated board. + */ + val32 = 0x1; + memcpy(out_params + 0, &val32, 4); + + /* + * windowPresentMask: bit i = window instance i present. Mirrors the + * caps notifier's SYS_CAPB=0xff and the NV0073 window assignment + * (windows 0/1 flexible) - eight usable windows. + */ + val32 = 0xff; + memcpy(out_params + 4, &val32, 4); + val32 = 1; /* numHeads */ memcpy(out_params + 12, &val32, 4); - val32 = 1; /* numDispChannels (core channel) */ + + /* + * numDispChannels sizes pClientChannelTable[], indexed by the packed + * channel number kdispGetChannelNum returns: CORE=0, WIN(i)=1+i, + * WINIM(i)=33+i, CURS(i)=73+i. Cover CURS(7)=80 -> 81 entries so any + * instance the guest allocates stays in bounds. + */ + val32 = 81; memcpy(out_params + 32, &val32, 4); *out_params_size = 36; *out_status = 0; - nv_vgpu_info(ctx->vgpu, "vgsp-rm: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1"); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=81 winMask=0xff feHwSysCap=0x1"); +} + +/* + * INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES (0x20800a5d) + * + * dispswConstruct_IMPL (disp_sw.c) asks for {displayMask, numHeads} and + * rejects every logicalHeadId >= numHeads with "invalid logical head + * number: N" + NV_ERR_INSUFFICIENT_RESOURCES; the generic all-zero reply + * made numHeads = 0, so nvkms' DISPLAY_SW objects could not be constructed + * on any head (observed r653: heads 0..3 all rejected -> vkCreateDevice -3). + * Report the same one-head model as DISPLAY_GET_STATIC_INFO: display id 0 + * active, one head. + * + * Params ABI (NV2080_CTRL_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES_PARAMS, + * 8 bytes): displayMask(0) numHeads(4). + */ +static void gsp_rm_handle_display_active_devices(struct gsp_emu_context *ctx, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 displayMask = 0x1; /* display id 0 */ + u32 numHeads = 1; + + memcpy(out_params + 0, &displayMask, 4); + memcpy(out_params + 4, &numHeads, 4); + + *out_params_size = 8; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DISPLAY_GET_ACTIVE_DISPLAY_DEVICES -> displayMask=0x1 numHeads=1"); +} + +/* + * NVC370_CTRL_CMD_GET_CHANNEL_INFO (0xc3700104) + * + * nvRMSyncEvoChannel() (nvkms-rm.c) polls this control through the display + * object after every EVO channel kickoff and waits up to 2 s for the channel + * to report NVC370_CTRL_GET_CHANNEL_INFO_STATE_IDLE, otherwise it logs + * "Idling display engine timed out: <class>:<instance>:<sd>:<token>" and + * fails the whole modeset init (observed on the r652 window path: 0xc57d and + * 0xc57e instances 0/1). The emulated display engine has no pushbuffer + * consumer, so the state would never leave BUSY; report IDLE immediately. + * + * Params ABI (NVC370_CTRL_CMD_GET_CHANNEL_INFO_PARAMS, 20 bytes): + * subdeviceIndex(0) channelClass(4) channelInstance(8) + * IsChannelInDebugMode(12) channelState(16 = NVBIT(0) IDLE). + * Echo the request so class/instance round-trip and set the state word. + */ +static void gsp_rm_handle_display_channel_info(struct gsp_emu_context *ctx, + const u8 *in_params, u32 in_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 idle = 1; /* NVC370_CTRL_GET_CHANNEL_INFO_STATE_IDLE */ + u32 class_id = 0, instance = 0; + u32 size = in_size; + + if (size > 64) + size = 64; /* defensive cap; the real params are 20 bytes */ + if (size >= 20) { + memcpy(out_params, in_params, size); + memcpy(&class_id, in_params + 4, 4); + memcpy(&instance, in_params + 8, 4); + memcpy(out_params + 16, &idle, 4); + } else { + memset(out_params, 0, 20); + memcpy(out_params + 16, &idle, 4); + size = 20; + } + + *out_params_size = size; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: GET_CHANNEL_INFO class=0x%x instance=%u -> IDLE", + class_id, instance); +} + +/* + * NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES (0x801707) + * + * The Vulkan/GL client queries the device for the size and alignment of an + * engine's context save region while building its GR channel context + * (observed: engineId 0x8 = GRAPHICS_ZCULL, one query at the end of every + * vkCreateDevice attempt). The generic all-zero reply handed back size = 0; + * the client's context-buffer allocation then failed (memdesc of size 0) + * and device creation aborted with VK_ERROR_INITIALIZATION_FAILED + * (observed r661: four create attempts, each ending at this control, then + * teardown and VKFAIL -3). + * + * The sizes below mirror the real 570.144 GSP-RM geometry captured on this + * TU104 (rm-call golden runs); entries marked "fill" were not covered by + * the capture and use nonzero page-aligned placeholders - revisit them when + * a real capture for the remaining engine ids exists. + * + * Params ABI (NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS, + * 12 bytes): engineId(0, in) alignment(4, out) size(8, out). + */ +static const struct gsp_rm_engine_ctx_geom { + u32 engine_id; + u32 alignment; + u32 size; +} gsp_rm_engine_ctx_geoms[] = { + { 0x00, 0x1000, 0xf2d00 }, /* GRAPHICS (captured) */ + { 0x08, 0x1000, 0x10000 }, /* GRAPHICS_ZCULL (fill) */ + { 0x09, 0x1000, 0xd100 }, /* GRAPHICS_PM (captured) */ + { 0x0a, 0x1000, 0x1000 }, /* COMPUTE_PREEMPT (fill) */ + { 0x0b, 0x1000, 0x1000 }, /* GRAPHICS_PREEMPT (fill) */ + { 0x0c, 0x1000, 0x1000 }, /* GRAPHICS_SPILL (fill) */ + { 0x0d, 0x100, 0x20000 }, /* GRAPHICS_PAGEPOOL (fill) */ + { 0x0e, 0x100, 0x8000 }, /* GRAPHICS_BETACB (fill) */ + { 0x0f, 0x100, 0x80000 }, /* GRAPHICS_RTV (fill) */ + { 0x10, 0x1000, 0x5000 }, /* GRAPHICS_PATCH (captured) */ + { 0x11, 0x100, 0x3000 }, /* GRAPHICS_BUNDLE_CB (captured) */ + { 0x12, 0x100, 0x20000 }, /* GRAPHICS_PAGEPOOL_GLOBAL (captured) */ + { 0x13, 0x1000, 0x7d7400 }, /* GRAPHICS_ATTRIBUTE_CB (captured) */ + { 0x14, 0x100, 0x80000 }, /* GRAPHICS_RTV_CB_GLOBAL (captured) */ + { 0x15, 0x100, 0x20000 }, /* GRAPHICS_GFXP_POOL (fill) */ + { 0x16, 0x100, 0x1000 }, /* GRAPHICS_GFXP_CTRL_BLK (fill) */ + { 0x17, 0x1000, 0x10000 }, /* GRAPHICS_FECS_EVENT (captured) */ + { 0x18, 0x1000, 0x80000 }, /* GRAPHICS_PRIV_ACCESS_MAP (captured) */ + { 0x19, 0x100, 0x1000 }, /* GRAPHICS_SETUP (fill) */ +}; + +static void gsp_rm_handle_fifo_engine_ctx_props(struct gsp_emu_context *ctx, + const u8 *in_params, u32 in_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 engine_id = ~0u; + u32 alignment = 0x1000, size = 0x1000; + u32 i; + bool found = false; + + if (in_size >= 4) + memcpy(&engine_id, in_params + 0, 4); + + for (i = 0; i < ARRAY_SIZE(gsp_rm_engine_ctx_geoms); i++) { + if (gsp_rm_engine_ctx_geoms[i].engine_id == engine_id) { + alignment = gsp_rm_engine_ctx_geoms[i].alignment; + size = gsp_rm_engine_ctx_geoms[i].size; + found = true; + break; + } + } + + if (engine_id > 0x19) { + /* Same rejection the real RM's VF handler uses for ids + * outside the engine context properties range. */ + *out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */ + *out_params_size = 0; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: FIFO_GET_ENGINE_CONTEXT_PROPERTIES engineId=0x%x -> NOT_SUPPORTED", + engine_id); + return; + } + + memcpy(out_params + 0, &engine_id, 4); + memcpy(out_params + 4, &alignment, 4); + memcpy(out_params + 8, &size, 4); + *out_params_size = 12; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: FIFO_GET_ENGINE_CONTEXT_PROPERTIES engineId=0x%x -> alignment=0x%x size=0x%x%s", + engine_id, alignment, size, found ? "" : " (default)"); } /* @@ -1917,6 +2162,103 @@ static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx, } /* + * NV0073 display-common heads + one static HDMI connector + * (0x730102 / 0x730287 / 0x7302ad / 0x730107 / 0x73028b / 0x730250 / + * 0x731140) + * + * nvidia-modeset's nvkms-rm.c ProbeHeadCountAndWindowAssignment asks the + * display-common object for the head count (NV0073_CTRL_CMD_SYSTEM_ + * GET_NUM_HEADS = 0x730102) and a zero reply makes it log "No heads found on + * board!" and fail -- which kills Xorg's ScreenInit and every Vulkan + * vkCreateDevice (measured: the emulated device replies all-zero through the + * generic path, so nvkms never opens). The kernel driver already sees 1 head + * (DISPLAY_GET_STATIC_INFO reports numHeads=1); report the same 1 head here, + * a single-head mask for GET_ALL_HEAD_MASK (0x730287), and a fully-flexible + * window assignment for windows 0/1 (0xFF = FLEXIBLE; windows above the head + * count stay 0 = not present). + * + * The next wall after the heads (measured r643): nvRmAllocDisplays fails on + * NoValidDisplays() - nvkms requires at least one valid display. Report one + * static TMDS/HDMI connector on display id 0: GET_SUPPORTED (0x730107) + * displayMask=1|DDC=1, SPECIFIC_OR_GET_INFO (0x73028b) SOR / + * SOR_SINGLE_TMDS_A (TMDS avoids the whole DP-AUX lib), SPECIFIC_ + * GET_CONNECTOR_DATA (0x730250) one HDMI_A entry, DFP_GET_INFO (0x731140) + * TMDS flags. Still a headless setup: no connect state or EDID is claimed + * (those are modeset-time queries). + */ +static void gsp_rm_handle_display_heads(struct gsp_emu_context *ctx, u32 cmd, + u32 req_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 one = 1; + u32 sz = req_size ? req_size : 72; + u32 v; + + memset(out_params, 0, sz); + switch (cmd) { + case 0x730102: /* SYSTEM_GET_NUM_HEADS: {subDev, flags, numHeads} */ + if (sz >= 12) + memcpy(out_params + 8, &one, 4); + break; + case 0x730287: /* SPECIFIC_GET_ALL_HEAD_MASK: {subDev, headMask} */ + if (sz >= 8) + memcpy(out_params + 4, &one, 4); + break; + case 0x7302ad: /* SPECIFIC_GET_VALID_HEAD_WINDOW_ASSIGNMENT: + * {subDev, windowHeadMask[32]} - windows 0 and 1 + * flexible (0xFF), the rest absent (0). */ + if (sz >= 36) { + out_params[4] = 0xff; + out_params[5] = 0xff; + } + break; + case 0x730107: /* SYSTEM_GET_SUPPORTED: {subDev, displayMask, + * displayMaskDDC} - display id 0 supported + DDC. */ + if (sz >= 12) { + memcpy(out_params + 4, &one, 4); + memcpy(out_params + 8, &one, 4); + } + break; + case 0x73028b: /* SPECIFIC_OR_GET_INFO: {subDev, displayId, index, + * type, protocol, ditherType, ditherAlgo, location, + * rootPortId, dcbIndex, vbiosAddress, bIsLitByVbios, + * bIsDispDynamic} */ + if (sz >= 52) { + v = 2; /* NV0073_CTRL_SPECIFIC_OR_TYPE_SOR */ + memcpy(out_params + 12, &v, 4); + v = 1; /* SOR_SINGLE_TMDS_A (no DP aux) */ + memcpy(out_params + 16, &v, 4); + /* location=0 (chip), remaining out-fields stay 0: + * not lit by vbios, not dynamic. */ + } + break; + case 0x730250: /* SPECIFIC_GET_CONNECTOR_DATA: {subDev, + * displayId, flags, DDCPartners, count, + * data[4]{index,type,location}, platform} */ + if (sz >= 72) { + v = 1; /* CONNECTOR_DATA_FLAGS_PRESENT_YES */ + memcpy(out_params + 8, &v, 4); + v = 1; /* count */ + memcpy(out_params + 16, &v, 4); + /* data[0].index = 0 */ + v = 0x61; /* CONNECTOR_DATA_TYPE_HDMI_A */ + memcpy(out_params + 24, &v, 4); + /* data[0].location = 0 */ + } + break; + case 0x731140: /* DFP_GET_INFO: {subDev, displayId, flags, + * UHBRSupportedByDfp} - flags signal bits = TMDS(0) */ + break; /* zeros are the TMDS signal value */ + } + *out_params_size = sz; + *out_status = 0; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DISPLAY shim cmd=0x%x -> 1 head / 1 HDMI connector (size %u)", + cmd, sz); +} + +/* * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08) * * kchangrpInit allocates one fault method buffer per runqueue of this @@ -3638,6 +3980,38 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; + case 0xc3700104: /* NVC370_CTRL_CMD_GET_CHANNEL_INFO */ + gsp_rm_handle_display_channel_info(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x20800a5d: /* NV2080_CTRL_CMD_INTERNAL_DISPLAY_GET_ACTIVE_DISPLAY_DEVICES */ + gsp_rm_handle_display_active_devices(ctx, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x801707: /* NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES */ + gsp_rm_handle_fifo_engine_ctx_props(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x730102: /* NV0073_CTRL_CMD_SYSTEM_GET_NUM_HEADS */ + case 0x730287: /* NV0073_CTRL_CMD_SPECIFIC_GET_ALL_HEAD_MASK */ + case 0x7302ad: /* NV0073_CTRL_CMD_SPECIFIC_GET_VALID_HEAD_WINDOW_ASSIGNMENT */ + case 0x730107: /* NV0073_CTRL_CMD_SYSTEM_GET_SUPPORTED */ + case 0x73028b: /* NV0073_CTRL_CMD_SPECIFIC_OR_GET_INFO */ + case 0x730250: /* NV0073_CTRL_CMD_SPECIFIC_GET_CONNECTOR_DATA */ + case 0x731140: /* NV0073_CTRL_CMD_DFP_GET_INFO */ + gsp_rm_handle_display_heads(ctx, req.cmd, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE: gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence, &out_status, @@ -4282,6 +4656,23 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) * For vGPU, we just acknowledge success. */ nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC UPDATE_BAR_PDE(%u) seq=%u len=%u -> SUCCESS", function, sequence, hdr->length); + { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: bar-pde payload: %s", dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: UPDATE_BAR_PDE reply send failed"); else @@ -4310,6 +4701,31 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) nv_vgpu_info(ctx->vgpu, "vgsp-rm: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)", gsp_rm_rpc_name(function), function, sequence, hdr->length, rm->unknown_rpcs); + /* + * Bounded payload dump for the first unknown RPCs: the vGPU + * guest maps its client VA spaces through some RPC; the + * mapping data (va pages / PAs / memdesc handles) must be + * visible here to be captured. Never echo it back (pure + * queries would eat their own stack garbage). + */ + if (rm->unknown_rpcs <= 96 && hdr->length) { + const u8 *pl = (const u8 *)hdr + sizeof(*hdr); + u32 n = min_t(u32, hdr->length, 64); + char dump[256]; + int o = 0; + u32 i; + + for (i = 0; i < n; i += 4) { + u32 v = 0; + + memcpy(&v, pl + i, 4); + o += snprintf(dump + o, sizeof(dump) - o, + "%08x ", v); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: unknown fn=%u seq=%u len=%u payload: %s", + function, sequence, hdr->length, dump); + } if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); else diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index a0bd4c7..a4b1afd 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -230,6 +230,12 @@ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); void gsp_rm_emu_fini(struct gsp_emu_context *ctx); int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src); int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx); +/* + * r623: deliver a synthetic OS_ERROR_LOG (0x1006) record to the guest's RM + * queue (the engine-fault report a real GSP sends). Debug/experiment only. + */ +int gsp_rm_send_os_error_log(struct nv_vgpu_device *vgpu, u32 except_type, + u32 runlist_id, u32 chid, const char *engine); int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu); int gsp_rm_dump_state(struct nv_vgpu_device *vgpu); int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index a0cdb2b..b6fa436 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -318,6 +318,21 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + bool ever_ops; /* r611: this channel has decoded at least one + * transfer op (copy/fill), i.e. it carries the + * client's stream-ordered memory work. The r607 + * stream-order watermark considers only such + * channels: a sibling GR LAUNCH channel's ring + * legitimately holds future launches, and + * deferring on those creates mutual deferral + * between the client's own launch channels + * (streams2: launches on h=0x5c00001f/23 each + * saw the other's unconsumed entry, burned the + * 15 s cap and were refused fail-closed - the + * r610c stream class red). Measured r606/r609/ + * r610c: op-decoding channels are exactly the + * copy ranks (0x5c000059 family) and the CE + * ranks; launch channels decode no ops. */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity @@ -1117,11 +1132,22 @@ struct sema_svc { u64 fwd_gr_argdefer; /* r543: held for a partial * argument set (unresolved arg) */ u32 fwd_gr_argdefer_log; + u64 fwd_gr_argfw; /* r612: non-degenerate fill + * forwarded at the cap (stalled + * candidates are decoys) */ + u32 fwd_gr_argfw_log; u64 fwd_gr_cewm; /* r590: held on the CE ring's * live submission watermark */ u64 fwd_gr_cewm_cap; /* r590: watermark still * backlogged at the defer cap */ u32 fwd_gr_cewm_log; + u64 ce_mutual_skips; /* r638: watermark + * skipped a channel whose + * head entry is wait-gate + * held (a consumer-side + * mutual citation, not a + * producer backlog) */ + u32 ce_mutual_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -1610,6 +1636,8 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* r633: per-session synthetic shadow budget reset (defined with the fill). */ +static void svc_gr_synth_ep_reset(void); /* * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): * small, reset per session where they are service-thread state, so a later @@ -2635,6 +2663,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap) struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; + /* + * r635: the page holds bytes a client/host WRITE actually landed + * (an H2D shadow-lane write, an inline write, a bridge write, an + * engine writeback). Synthetic pages minted by the GR fill serve + * zeros and must never be mistaken for the client's own content: + * only real content anchors a span or arms the launch-level anchor + * (without this flag the fill would re-anchor on its own minted + * pages and mint without bound across retry passes). + */ + u8 content; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; @@ -2670,6 +2708,14 @@ static bool svc_vas_has(u64 va) return svc_vas_find(va) != NULL; } +/* r635: the page holds REAL written content (not a synthetic zero page). */ +static bool svc_vas_has_content(u64 va) +{ + struct svc_vas_rec *r = svc_vas_find(va); + + return r && r->content; +} + /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects @@ -2677,8 +2723,23 @@ static bool svc_vas_has(u64 va) * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) * shadow any VA because their target is device memory by construction. */ +bool gsp_sema_no_shadow; /* r626 A/B: disable the shadow lane */ +module_param_named(sema_no_shadow, gsp_sema_no_shadow, bool, 0644); +MODULE_PARM_DESC(sema_no_shadow, + "Disable the VA shadow lane (copy-dst and fill fallbacks) for an A/B boot; unresolved device VAs hold instead (default 0)"); + static bool svc_vas_can_shadow(u64 va) { + /* + * r626 A/B: with the knob set, every shadow fallback is refused and + * the caller takes its -ENOENT/hold path. The r613c/r616/r623c + * corruption class (nvidia channel structures overwritten with + * PTE-like 0x444../0x4456.. patterns, fatal panic in + * kchannelGetGfid_IMPL via nvGpuOpsStopChannel) follows the + * shadow-lane era; this knob isolates the lane as a variable. + */ + if (gsp_sema_no_shadow) + return false; return va >= SVC_VAS_FLOOR; } @@ -2728,6 +2789,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len) if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + r->content = 1; /* r635: real bytes landed here */ svc.vas_writes += chunk; buf += chunk; va += chunk; @@ -3530,6 +3592,9 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_should_hold(u64 va); /* r609 */ +static void svc_dst_guard_age(void); /* r609 */ +static void svc_dst_guard_reset(void); /* r609 */ static bool svc_dst_guard_pending(u64 va); /* r231 */ static u32 svc_read_order_log; /* r231 budget */ static u32 svc_atrace_log; /* r234 A-TRACE budget */ @@ -4159,6 +4224,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) int wap, rc; bool verified = false; bool rb_ok = false; + static u32 sem_sw_dev_log; /* r619: non-pool device-VA write budget */ /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem @@ -4218,10 +4284,23 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * sysmem semaphore range (0x792..), with the resolver provenance and * the read-back, so the second session's translations can be * compared with the first session's for the same VA. Bounded. - */ - if (va >= 0x79240000000ULL && va < 0x79260000000ULL && - svc.sem_sw_log < 8192) { - svc.sem_sw_log++; + * + * r619 diag: the same proof for NON-pool device-VA writes (the VAS + * window >= 0x200000000; e.g. 0x20480fef0 push releases, 0x204827f7c + * QMD releases). The r618 hung math client read 0 at 0x20480fef0 + * while the host's deferred releases logged `cur` 97..105 on its own + * target - a write-target divergence that was invisible because this + * range never printed `pa`. r620: the first cut used the 1 TiB + * SVC_VAS_FLOOR and excluded this very window (no lines ever); + * bounded per boot (512). + */ + if (((va >= 0x79240000000ULL && va < 0x79260000000ULL && + svc.sem_sw_log < 8192) || + (va >= 0x200000000ULL && !is_pool && sem_sw_dev_log < 4096))) { + if (va >= 0x79240000000ULL && va < 0x79260000000ULL) + svc.sem_sw_log++; + else + sem_sw_dev_log++; pr_info("sema-sw: h=0x%x va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x rc=%d rb_ok=%d back=0x%x pool=%d\n", h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } @@ -4634,6 +4713,26 @@ static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; static u64 svc_wait_giveup; +static u64 svc_wait_extended; /* r613: give-up deferred while a + * GR forward is owed */ +static u32 svc_wait_ext_log; +/* + * r613: episode extension bound while a GR forward is owed. Comfortably + * above wait-hold (default 4000 ms) + the absolute launch defer cap + * (SVC_GR_DEFER_CAP_MS, 15000 ms) + margin, and far below every client-side + * timeout (>= 60 s). + * + * r640: raised 25 s -> 45 s. The r639 streams2 transcript measured the + * readback's episode give up at waited_ms=28378 while the launch whose + * release it waits for only forwarded (GR-STALL-backend, software release + * authored) at 419322.99 - ~9 s AFTER the give-up - because the launch's + * own path paid the watermark defer (15 s cap) + arg-partial defer (8 s) + + * backend poll before it moved. The readback then ran out of order and the + * client read zeros (bad=65534). 45 s still leaves >15 s of headroom under + * the clients' >= 60 s timeouts, and only an episode with an unoffered + * candidate ever reaches it. + */ +#define SVC_WAIT_EXT_CAP_MS 45000u static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ @@ -4656,6 +4755,10 @@ static bool svc_wait_rel_have; struct svc_wait_hold { bool active; u32 h; + u32 idx; /* r638: GPFIFO entry index this hold owns + * (0xffffffff when unset); the CE stream-order + * watermark uses it to detect a mutual citation + * (a held consumer entry, not a producer backlog) */ unsigned long since; }; #define SVC_WAIT_HOLD_MAX 16 @@ -5881,6 +5984,13 @@ struct svc_gr_data { u64 pa; /* guest PA, page aligned (0 = unresolved) */ u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ + /* + * r632: the page is served from the VA shadow lane instead of a guest + * page (no resolver and no landing answered for it). The submit build + * hands the shadow page's host address to the bounce machinery - see + * the fill's shadow-serve branch and the r632 note there. + */ + u8 shadow; }; /* @@ -5946,6 +6056,29 @@ struct svc_gr_cand_rec { static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) + +/* + * r613: is a GR forward owed for this device - a captured candidate that has + * not been offered (submitted or refused) yet? The wait gate uses this to + * distinguish "the acquire's release can never be installed by firmware + * alone" from "the release is pending behind OUR OWN launch deferral": the + * r612 streams2 capture gave up a D2H readback's acquire (0x20480ffe0 pay=4, + * cur=2) at waited_ms=10333 while the vadd launches that author that + * semaphore sat deferred by the CE watermark for the full 15 s cap - the + * readback then ran BEFORE the kernels and the client read zeros + * (bad=65534). While a forward is owed, the episode extends (bounded by + * wait-hold + defer cap + margin) instead of failing open. + */ +static bool svc_gr_forward_pending(void) +{ + u32 i; + + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered) + return true; + return false; +} /* * r383: push-VA history of recent captures, beyond the (four-slot) * candidate ring's lifetime. A replayed decode of a complete QMD asks @@ -6299,6 +6432,106 @@ static u8 svc_fwd_rel_n; static bool svc_fwd_defer_armed; /* + * r633: PARKED push-level release records for a GR launch whose entry was + * HELD before it could be forwarded. The records above are per-decode state + * (reset at the top of every svc_decode_stream); the two GR hold paths + * (busy-defer hold, m150 data-stalled hold) break out of the entry loop + * BEFORE the release block runs, so the entry's records would be lost at + * the next decode - and the eventual forward (queued from the capture ring + * in a later pass, which does not re-decode the entry) would engage the + * release hold with whatever records that pass happens to carry (often + * none, or another entry's), leaving the push's tracking release unapplied + * and the client spinning in cudaDeviceSynchronize forever (the r632 mail + * MALLOC hang: push release 0x20480fff0 pay=112 never landed while the + * client's word stayed at 111). Park keyed by (vgpu, channel, push VA); + * restored into svc_fwd_rel[] by svc_gr_request when the launch is finally + * queued, so the pump's hold-engage holds exactly the queued push's + * records; delivered inline by the paths that consume a capture without a + * forward. Service-thread only. + */ +#define SVC_GRPARK_SLOTS 8 +static struct svc_grpark { + struct nv_vgpu_device *vgpu; + u32 h_object; + u64 push_va; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +} svc_grpark[SVC_GRPARK_SLOTS]; +static u32 svc_grpark_pos; + +static void svc_grpark_put(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, const struct svc_fwd_rel *rel, u8 n) +{ + u32 i, slot = ~0u; + + if (!vgpu || !push_va || !rel || !n) + return; + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + slot = i; + break; + } + if (slot == ~0u && !svc_grpark[i].push_va) + slot = i; + } + if (slot == ~0u) + slot = svc_grpark_pos++ % SVC_GRPARK_SLOTS; + svc_grpark[slot].vgpu = vgpu; + svc_grpark[slot].h_object = h_object; + svc_grpark[slot].push_va = push_va; + svc_grpark[slot].n = n; + memcpy(svc_grpark[slot].rel, rel, + sizeof(svc_grpark[slot].rel)); +} + +static bool svc_grpark_get(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, struct svc_fwd_rel *out, u8 *n) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + if (out) + memcpy(out, svc_grpark[i].rel, + sizeof(svc_grpark[i].rel)); + if (n) + *n = svc_grpark[i].n; + return true; + } + } + return false; +} + +static void svc_grpark_drop(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + memset(&svc_grpark[i], 0, sizeof(svc_grpark[i])); + return; + } +} + +static void svc_grpark_reset(void) +{ + memset(svc_grpark, 0, sizeof(svc_grpark)); + svc_grpark_pos = 0; +} + +/* r633: bounded evidence that a parked set was restored/delivered. */ +static u32 svc_grpark_log; + +/* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). @@ -6748,6 +6981,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot bool audit = total >= SVC_COPY_AUDIT_MIN; u8 *snap = NULL; static u64 bigcopy_log; + static u32 shadow_dst_log; /* r614 diag: shadow-lane dst witness */ bool trace64 = (total == 65536u); const char *dir = "?"; @@ -6864,6 +7098,26 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot } else { dst_shadow = true; svc.vas_shadowed++; + /* + * r614 diag: the r613b red boot's matmul + * wedge began here - the H2D into the + * client's cudaMalloc'd buffer found no + * ledger/pin/walk answer, took the shadow + * silently, and the later GR fill could not + * consume the shadowed bytes (probe=none for + * every arg base; the launch held 15 s, then + * the degenerate refusal consumed it and the + * client spun). Name the first shadow-lane + * device-dst copies of a boot (bounded), so + * the next red boot is greppable where it + * starts instead of only at the launch. + */ + if (shadow_dst_log < 64 && + dst_orig >= SVC_VAS_FLOOR) { + shadow_dst_log++; + pr_info("sema-shadow-dst: h=0x%x dst=0x%llx total=%zu (device VA with no ledger/pin/walk answer; copy runs into the shadow, a GR fill cannot consume it)\n", + owner_ch, dst_orig, total); + } } } else { dst = pa; @@ -6890,7 +7144,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot if (dst_virt && dst_aperture == CE_APERTURE_VID && dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && gsp_sema_copy_dst_pin_guard && - svc_dst_guard_hold(dst_orig)) { + svc_dst_guard_should_hold(dst_orig)) { svc.copy_dst_hold++; if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", @@ -7697,6 +7951,22 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * kept the stale pre-pin pages while the copy landed on the pinned * page (streams2 r254: the b1 rec range carried the a2 pages and * the vadd read a2's pattern as its second input). + * + * r627: NEVER overwrite a per-page landing the loop above already + * stored. That loop stores each page's true resolved PA; this + * pass's `dst + po` is a contiguity guess valid only while the VA + * range maps contiguously. For a scattered UVM host buffer (r623c + * saxpy: page 0 at pa 0x115781000, page 0x7f000 at pa 0x12e700000) + * the guess clobbered the true entries - + * `sema-ledger: rewrite va=0x70e64e77f000 old=0x12e700000 + * new=0x115800000` - and the NEXT copy of the same VA then + * followed the poisoned ledger: 504x `sema-ledger: DIVERGE write + * va=0x70e64e7xxxxx prev=0x1158xxxxx cur=0x12e7xxxxx`; the + * non-pin branch keeps the ledger, so ~2 MiB of client c-data + * landed in guest pages the kernel had since re-used and the + * guest died in kchannelGetGfid_IMPL (r623c panic). The loop's + * per-page entries are the authority; this pass only seeds pages + * the loop did not reach. */ if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) { @@ -7704,8 +7974,14 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot SVC_APERTURE_FB : SVC_APERTURE_SYS; size_t po; - for (po = 0; po < total; po += SEMA_PAGE_SIZE) + for (po = 0; po < total; po += SEMA_PAGE_SIZE) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_orig + po, &lpa, &lap)) + continue; svc_ledger_store(dst_orig + po, dst + po, cap); + } } if (snap) svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, @@ -7887,6 +8163,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) } } +/* + * r638: is (h, idx) currently owned by the bounded wait gate? The CE + * stream-order watermark consults this to tell a producer backlog (a CE + * copy the client ordered before the launch - the reason the watermark + * exists) from a consumer-side wait the forward itself will unblock (the + * mutual citation measured on r612/r636: the deferred launch's release is + * the only value that can satisfy the held copy's acquire, and the copy's + * unconsumed entry is the only reason the launch is deferred). + */ +static bool svc_wait_held_entry(u32 h, u32 idx) +{ + int i; + + if (idx == 0xffffffffu) + return false; + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + const struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h && e->idx == idx) + return true; + } + return false; +} + /* r34x: the per-channel first-hold clock for the bounded wait. */ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) { @@ -7906,6 +8206,7 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) memset(free_slot, 0, sizeof(*free_slot)); free_slot->active = true; free_slot->h = h; + free_slot->idx = 0xffffffffu; /* r638: unset until a hold lands */ free_slot->since = jiffies; return free_slot; } @@ -8219,26 +8520,57 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, pend = svc_wait_pend_release(w->va); if (waited_ms >= gsp_sema_wait_hold_ms) { /* - * Bounded episode expired: fail open for the - * WHOLE entry (one row, then the ops run). A - * per-wait continue here would ping-pong - * with a second unsatisfied wait: the first - * gave up and reset the shared clock before - * the second was judged, and the pair would - * hold forever 100 ms at a time. + * r613: a deferred GR launch extends the + * episode. The release this acquire waits for + * may be authored by a launch we are holding + * through our own bounded deferral (the r612 + * streams2 capture: the D2H readback's + * acquire on 0x20480ffe0 was given up at + * waited_ms=10333 while the vadd launches + * that release it sat deferred by the CE + * watermark for 15 s; the readback then ran + * before the kernels and the client read + * zeros). While a captured candidate is + * unoffered, "the value cannot be installed + * by firmware alone" is not yet true - keep + * holding, bounded by the defer cap plus + * margin. The client-side timeouts are all + * >= 60 s, so a <= 24 s episode cannot + * strand it. */ - svc_wait_giveup++; - if (svc_wait_log < 2048) { - svc_wait_log++; - pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", - ch_h, w->va, w->op, w->payload, - cur, pend, waited_ms); + if (waited_ms < SVC_WAIT_EXT_CAP_MS && + svc_gr_forward_pending()) { + svc_wait_extended++; + if (svc_wait_ext_log < 32) { + svc_wait_ext_log++; + pr_info("sema-wait: extend ch=0x%x va=0x%llx op=%u pay=%u cur=%u waited_ms=%u -> hold (a GR forward is owed; the release is behind our own defer)\n", + ch_h, w->va, w->op, + w->payload, cur, + waited_ms); + } + } else { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; } - if (gh) - gh->active = false; - hold = false; - svc_wait_pass++; - goto svc_wait_done; } hold = true; if (svc_wait_log < 512) { @@ -8253,6 +8585,10 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc_wait_held++; svc_wait_hold_h = ch_h; svc_wait_hold_idx = entry_idx; + /* r638: record the held entry for the CE watermark's + * mutual-cycle check. */ + if (gh) + gh->idx = entry_idx; svc.op_tr_pending = 1; return; } @@ -11407,8 +11743,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * soon as the trust (or the operand translation) * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps * the pre-existing skip as the last resort. + * + * r610: a REPLAY pass must be exempt. A replay is the + * pump re-walking an entry whose ops were already + * recorded complete for this identity (entry_replay = + * same && slot_ops_complete): re-marking it as pending + * re-arms a 3 s opdef hold for a copy that HAS run, + * and the fresh pass that follows re-executes the op - + * a replay/fresh/replay cycle (r609 boot, idx=94: the + * d_b H2D re-marked and re-run every ~0.3 s from + * 368841 while the vadd launch waited; the r606 + * captured held chain is the same shape). Measured + * r604-r609: 100% of op_tr_pending marks (27/27, + * 71/71, 72/72) came from replay passes of already- + * executed copies - each one stalled the whole copy + * channel behind it, which is the held-chain class. + * The r308 rationale (pre-r595 the eng==0 copy + * channel's fresh passes never admitted ops, so the + * replay re-mark WAS the retry hatch) no longer + * applies: r595's cap_trust admission runs a plausible + * copy on its fresh pass, and a genuinely failed copy + * keeps its slot incomplete through op_owed / the + * rc!=0 pending rule. The whole accounting block is + * skipped on replay: no dstpend note (the write ran), + * no staging re-snapshot, no hold. A fresh + * (non-replay) denied pass marks pending exactly as + * before. */ - if (!allow_ops && trusted && + if (!allow_ops && trusted && !replay && !ch->engine_type && transfer != 0 && st.line_length && st.have_out && st.have_in && !DMA_REMAP_ENABLE(dma) && @@ -11531,6 +11893,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (n_ops != n_ops_before) { struct svc_memset_op *lm = &ops[n_ops - 1]; + /* r611: this channel carries transfer work + * (see ever_ops) - the stream-order + * watermark's scope. */ + ch->ever_ops = true; + /* * fail4 CONT.53: H2D-shaped copies (virtual * device destination, virtual host source) @@ -12265,6 +12632,9 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * exist; drop them instead of resolving them against the next session. */ static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ +/* r604: defined next to the per-VA defer tables; clears both on session end + * (they are keyed by push VA alone and the UVM pool reuses those VAs). */ +static void svc_gr_defer_reset(void); static void svc_session_flush(void) { @@ -12301,6 +12671,10 @@ static void svc_session_flush(void) } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ + svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_grpark_reset(); /* r633: per-session parked GR releases */ + svc_gr_synth_ep_reset(); /* r633: per-session synthetic budgets */ + svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; svc_pd_found = 0; @@ -14829,6 +15203,88 @@ bool gsp_sema_wait_gate = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r609: the m169 first-touch window is per PAGE, but the stale-table risk it + * covers is per CLIENT TABLE BUILD. The driver's H2D relocation burst on the + * eng==0 copy channel writes N distinct first-write pages back to back, and + * because the pump holds the FIRST held entry and the rest of the chain + * decodes only after it drains, each page paid its own full window, + * serialized (r606 red draw: `sema-pend: hold h=0x5c000059 idx=90` held + * 362167.7->362170.8, idx=92 held 362177.7->362182.9; the b H2D entry ran + * 362182.882, 326 ms AFTER the launch forward -> `VADD bad=32735 + * csum=8139264`, c = a + 0). The guard's bet - a fresh pin lands inside the + * window - is a property of the client's table state, not of one page: once + * ONE device-dst page of a channel has waited the full window with NO pin + * arriving (every observed release logs "without a pin"), the same bet for + * the next page of the same burst is already lost. Age the channel for + * gsp_sema_copy_dst_age_ms after such a release; further first-write + * device-dst pages of that channel inside the window skip the hold and run + * with the walk translation - the exact translation the expired guard would + * have used anyway. The first page of every burst still pays the full + * window, so the anti-stale bet is unchanged; only the N x window + * serialization is removed (r606 class: ~20 s of chain -> one window). + * Keyed by h_object like every channel table; the session flush clears it, + * so a recycled handle can never inherit the previous client's ageing + * (the r604 lifetime lesson). 0 disables (= per-page windows, pre-r609). + */ +unsigned gsp_sema_copy_dst_age_ms = 3000; +#define SVC_DST_AGE_SLOTS 16 +static u32 svc_dst_age_ch[SVC_DST_AGE_SLOTS]; +static unsigned long svc_dst_age_j[SVC_DST_AGE_SLOTS]; +static u32 svc_dst_age_pos; +static u32 svc_dst_age_log; + +/* r609: the current decode channel's guard window expired without a pin. */ +static void svc_dst_guard_age(void) +{ + u32 i; + + if (!gsp_sema_copy_dst_age_ms) + return; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch) { + svc_dst_age_j[i] = jiffies; + return; + } + i = svc_dst_age_pos++ % SVC_DST_AGE_SLOTS; + svc_dst_age_ch[i] = svc.cur_ch; + svc_dst_age_j[i] = jiffies; +} + +/* r609: is the current decode channel inside its post-expiry ageing window? */ +static bool svc_dst_guard_aged(void) +{ + unsigned long now = jiffies; + u32 i; + + if (!gsp_sema_copy_dst_age_ms || !svc.cur_ch) + return false; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch && + time_before(now, svc_dst_age_j[i] + + msecs_to_jiffies(gsp_sema_copy_dst_age_ms))) + return true; + return false; +} + +/* + * r609: decide whether a first-write device-dst page pays the m169 window. + * An aged channel skips it (the burst's first page already paid and no pin + * arrived); otherwise the normal per-page window runs. + */ +static bool svc_dst_guard_should_hold(u64 va) +{ + if (svc_dst_guard_aged()) { + if (svc_dst_age_log < 16) { + svc_dst_age_log++; + pr_info("sema-svc: device-dst guard age-skipped dst=0x%llx h=0x%x (same-channel burst after an un-pinned release; walk translation)\n", + va, svc.cur_ch); + } + return false; + } + return svc_dst_guard_hold(va); +} + +/* * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose * source VA has a PENDING device-dst write hold (the m169 guard window is * still open) reads the PRE-write content: the writer's bytes land only @@ -14877,6 +15333,7 @@ static bool svc_dst_guard_hold(u64 va) if (svc_dst_guard[i].first) { svc_dst_guard[i].first = 0; svc.copy_dst_relaxed++; + svc_dst_guard_age(); /* r609: burst ageing */ pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", page); } @@ -14893,6 +15350,24 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r609: clear the m169 page windows and the r609 ageing ring with the + * session. Both are keyed by VA page / h_object alone, and both would + * otherwise pre-age the next client's first touch (a page slot still inside + * its window makes the new client's first device-dst write skip the + * anti-stale wait; an ageing stamp makes the new channel's whole first + * burst skip it). The stale-table risk this guard covers is precisely the + * new-session condition, so the windows must start fresh. + */ +static void svc_dst_guard_reset(void) +{ + memset(svc_dst_guard, 0, sizeof(svc_dst_guard)); + svc_dst_guard_next = 0; + memset(svc_dst_age_ch, 0, sizeof(svc_dst_age_ch)); + memset(svc_dst_age_j, 0, sizeof(svc_dst_age_j)); + svc_dst_age_pos = 0; +} + +/* * r231 READ-ORDER guard: true while a held device-dst write for this VA * page is still inside its bounded window (the writer's bytes have not * landed yet). See the gsp_sema_read_order_guard declaration. @@ -18116,6 +18591,87 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, * applies the recorded releases AND the QMD word, bounded by the existing * hold timeout), so this check stands down for it. */ + +/* + * r618: synth-fault for a refused/failed GR launch. The r616 experiment + * (manual grfaultpoke on the live r616b matmul wedge - fill `0/5 high + * argument ptr(s)`, `GR submit refused: degenerate data set n_data=2 + * (min 16)`, sw release authored, client spun in cudaDeviceSynchronize + * forever) proved the edge: the channel error-notifier write + * (status=0xffff + channel event) ends the client's wait with + * `CUDA_ERROR unspecified launch failure` in seconds. Deliver the same + * edge automatically on every FAILED forward so a refused launch is an + * observable error, never an unbounded hang. One-line rollback: + * gsp_sema_refusal_fault=0. + */ +bool gsp_sema_refusal_fault = true; +module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); +MODULE_PARM_DESC(sema_refusal_fault, + "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); + +/* + * r638: break the CE-watermark/wait-gate mutual citation (see + * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose + * unconsumed head entry is wait-gate held is a consumer-side wait, not a + * producer backlog, and is skipped while the candidate's operands are fully + * resolved and no decoded device write is pending on a data page. Rollback + * to the r636 behaviour: sema_mutual_skip=0. + */ +bool gsp_sema_mutual_skip = true; +module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644); +MODULE_PARM_DESC(sema_mutual_skip, + "Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)"); + +static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception, as proven live */ + unsigned long flags; + struct svc_channel *ch = NULL; + u64 pa; + u32 as, eng; + int i; + + if (!gsp_sema_refusal_fault || !h_object) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object && + svc.chan[i].notifier_pa) { + ch = &svc.chan[i]; + break; + } + } + if (!ch) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return; + } + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + { + u16 info16 = (u16)eng; + + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + } + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info_ratelimited("vgsp-fault: SYNTH-FAULT notifier write failed ch=0x%x pa=0x%llx as=%u\n", + h_object, pa, as); + return; + } + pr_warn("vgsp-fault: SYNTH-FAULT ch=0x%x engine=%u pa=0x%llx status=0x%x except=0x%x (refused forwarded launch -> client-visible error)\n", + h_object, eng, pa, status, info32); + gsp_rm_post_channel_event(vgpu, h_object); +} static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) { bool done = smp_load_acquire(&svc_fwd_req.done); @@ -18149,10 +18705,18 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) + if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); - else + /* + * r618: the software release alone completes only launches the + * engine actually ran; a refused/failed one left the client in + * an unbounded cudaDeviceSynchronize. Deliver the synthetic + * engine-error edge (proven live by the r616 grfaultpoke test) + * so the failed launch is client-observable. + */ + svc_gr_fault_synth(vgpu, svc_fwd_req.gr_h_object); + } else pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited); pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", @@ -18228,6 +18792,19 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* + * r633 diag: the held record set, so a later release-delivery gap + * (the streams2 spin class) can be told apart from a hold that + * engaged empty / with another entry's records. Bounded. + */ + if (svc_grpark_log < 64) { + svc_grpark_log++; + pr_info("vgsp-work: hold engage h=0x%x entry=%u nrel=%u va0=0x%llx pay0=%u gr=%d\n", + ch->h_object, entry_idx, n, + n ? svc_fwd_hold.rel[0].va : 0ull, + n ? svc_fwd_hold.rel[0].payload : 0u, + svc_fwd_req.gr ? 1 : 0); + } /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ svc_fwd_hold.paused_until = 0; svc_fwd_hold.pause_logged = false; @@ -18850,6 +19427,26 @@ static void svc_fwd_gr_work(void) rel_unobserved = true; } svc_fwd_req.gr_rel_unobserved = rel_unobserved; + /* + * r633 diag: the release0 resolution chain, so a redirect + * ("release0 could not target the guest page") names its cause + * (rel_pa unresolved at capture vs HPA unresolved here vs the + * forward-only zeroing). Bounded. + */ + { + static u32 gr_relhpa_log; + + if (gr_relhpa_log < 32) { + gr_relhpa_log++; + pr_info("vgsp-work: gr-rel0 va=0x%llx pa=0x%llx ap=%u hpa=0x%llx off=%u word=%08x pay=%u unobs=%d\n", + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_pa, + svc_fwd_req.gr_rel_ap, rel_hpa, roff, + rel_hpa ? READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) : 0, + svc_fwd_req.gr_rel_payload, + rel_unobserved ? 1 : 0); + } + } } gs.qmd = svc_fwd_req.gr_qmd; @@ -18875,11 +19472,34 @@ static void svc_fwd_gr_work(void) u64 dhpa = 0; u32 doff = 0; - if (!dg->pa) - continue; - if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0, - SEMA_PAGE_SIZE, &dhpa, &doff) || doff) - continue; + if (dg->shadow) { + /* + * r632: the page was served from the VA shadow lane + * (no resolver and no landing answered for it). The + * shadow page is the page's storage: hand its host + * address to the bounce machinery exactly like a + * resolved guest page. The copy-in stages the client's + * own bytes (H2D'd or zeroed) from it, and the + * copy-out writes the engine's results back into it - + * the same page the D2H copies read through the + * shadow-first rule. Module-owned memory throughout; + * no guest page is involved. + */ + struct svc_vas_rec *vr = svc_vas_find(dg->va); + + if (!vr || !vr->data) + continue; + dhpa = (u64)virt_to_phys(vr->data) & + ~(u64)(SEMA_PAGE_SIZE - 1); + dkind = SVC_APERTURE_SYS; + } else { + if (!dg->pa) + continue; + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, + dg->pa, 0, SEMA_PAGE_SIZE, + &dhpa, &doff) || doff) + continue; + } if (gs.n_data >= SVC_GR_DATA_MAX) break; data[gs.n_data].va = dg->va; @@ -19529,6 +20149,19 @@ static bool svc_gr_page_inline_covers(u64 page) static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static u32 svc_gr_shadow_fill_log; /* r632: bounded shadow-serve log */ +/* + * r632 (the red-draw resolution class): caps on SYNTHETIC shadow pages a + * launch may create for argument candidates whose VAs no resolver answers + * (the output-buffer class). Existing shadow content (the client's own H2D + * writes) serves without a quota. The per-candidate cap bounds how far a + * span may guess past a buffer's true end (the first page past the cap ends + * the span through the historical drop path); the per-launch cap keeps the + * bounce window (16384 pages) safe. The real output operands are argument + * candidates and are collected first. + */ +#define SVC_GR_SHADOW_SYNTH_PER_CAND 1024u +#define SVC_GR_SHADOW_SYNTH_MAX 4096u static bool svc_gr_data_stalled; /* m150: high data args unresolved */ /* * m156 (R1): the GR request was refused because a forward was already @@ -19770,6 +20403,16 @@ bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) if (!va || !hpa) return true; + /* + * r632: a VA with a shadow page uses the shadow as its storage (the + * copy paths resolve it there before any other source, and the launch + * path serves its pages from it). The engine's writeback targets + * exactly that page, so a later pin or discovery answer must not skip + * the write - skipping it would strand the engine's output in the + * bounce while the D2H copies still read the shadow. + */ + if (svc_vas_has(va)) + return true; if (!svc_ipin_lookup(page, &ipa, &iap)) { u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -20032,11 +20675,268 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) return false; } +/* + * r632 (the red-draw resolution class): serve a data page from the VA shadow + * lane when the resolution chain (ipin > resolver > ledger) misses. + * + * The red draw's chain is: a client session whose device VAs no source + * answers (the FB-aperture directory walk returns rc=-2 for every published + * VA space, the pin stream delivers no pair, the discovery re-validation + * fails) - the client's H2D copies then land in the shadow lane (the copy's + * documented fallback), and the launch fill drops every operand, the hold + * expires, the data set is degenerate and the refusal faults the client. + * + * Serving the page from the shadow instead keeps one story across the whole + * pipeline: the copy paths already read/write a shadowed VA in the shadow + * (svc_vas_has is checked before every resolver there), so the fill hands the + * engine the client's own H2D bytes (or a zeroed page for an output the + * client never wrote), and the submit build passes the shadow page's host + * address to the bounce machinery. The engine's writeback lands back in that + * same page, which is exactly what the D2H copies read. The shadow is + * module-owned memory: no guest page is ever touched on this path, and the + * pages are freed with the session like every other shadow page. + * + * Existing shadow content serves unconditionally (the client wrote it). + * A synthetic page (an output buffer no copy has touched) is created only + * for ANCHORED argument candidates - candidates whose span already has at + * least one page the resolvers answered - under both a per-candidate cap + * (the span has no other way to know where the buffer ends - the cap bounds + * the guess and the first page failing the cap ends the span through the + * historical drop path) and a per-launch cap (the bounce window must + * survive). With sema_no_shadow=1 every fallback is refused and the caller + * keeps its historical -ENOENT/drop path. + * + * r633 (the r632 cap validation): the r632 caps were per-FILL-CALL, and the + * fill re-runs on every retry pass of a held launch - one probe=none junk + * candidate could therefore mint another 1024 synthetic pages per pass and + * the 8192-page shadow table filled within seconds (r632 mail: va-shadow + * 8192/8192 with all pages synthetic, from the two 16 MB guessed spans of + * candidates no resolver ever answered). Two changes bound this: + * - the ANCHOR GATE: synthetic pages exist only to extend a span that + * real evidence has already anchored (>= 1 page resolved through + * ipin/resolve/ledger). A candidate whose every page is answerless is + * the historical drop path - its guessed span is not a buffer; + * - the budget is PERSISTED PER LAUNCH EPISODE, keyed by (push VA, span + * base), so the caps hold across the episode's passes, not per call. + * The anchor gate is what keeps the genuinely useful case working: the + * red-draw chain's H2D-shadowed operand pages serve through the CONTENT + * path (unconditionally), and a real buffer's unresolved tail is anchored + * by its resolved head. + */ +#define SVC_GR_SYNTH_EP_SLOTS 16 +static struct svc_gr_synth_ep { + u64 push_va; + u64 base; /* span base; 0 = the launch-level total record */ + u32 n; + /* + * r635: the cross-mint counter (launch-level anchor, see + * svc_gr_shadow_fill_serve). Tracked separately from n so a cross + * mint can never consume the anchored budgets and vice versa. + */ + u32 cross; +} svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; +static u32 svc_gr_synth_ep_pos; + +static struct svc_gr_synth_ep *svc_gr_synth_ep_get(u64 push_va, u64 base, + bool create) +{ + u32 i, slot = ~0u; + + if (!push_va) + return NULL; + for (i = 0; i < SVC_GR_SYNTH_EP_SLOTS; i++) { + if (svc_gr_synth_ep[i].push_va == push_va && + svc_gr_synth_ep[i].base == base) + return &svc_gr_synth_ep[i]; + if (slot == ~0u && !svc_gr_synth_ep[i].push_va) + slot = i; + } + if (!create) + return NULL; + if (slot == ~0u) + slot = svc_gr_synth_ep_pos++ % SVC_GR_SYNTH_EP_SLOTS; + memset(&svc_gr_synth_ep[slot], 0, sizeof(svc_gr_synth_ep[slot])); + svc_gr_synth_ep[slot].push_va = push_va; + svc_gr_synth_ep[slot].base = base; + return &svc_gr_synth_ep[slot]; +} + +static u32 svc_gr_synth_ep_launch_n(u64 push_va) +{ + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, false); + + return lep ? lep->n : 0; +} + +static void svc_gr_synth_ep_reset(void) +{ + memset(svc_gr_synth_ep, 0, sizeof(svc_gr_synth_ep)); + svc_gr_synth_ep_pos = 0; +} + +/* + * r635 (host-side operand resolution without the scanner): the r634 red + * draws measured the remaining shape - a client whose INPUT buffers are + * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane) + * while its OUTPUT buffer (an argument candidate the client never wrote) + * still failed closed, because the r633 anchor gate only counted a page + * answered through ipin/resolve/ledger and every page of the output span + * was answerless. Two additions, both host-side (no scanner/pin needed): + * + * - a CONTENT-served page anchors its span exactly like a trusted answer: + * the client's own H2D bytes are first-hand truth, and the output tail + * of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers) + * then mints synthetic pages through the unchanged per-candidate / + * per-launch budgets instead of dropping; + * - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch + * is served from content or trust, an answerless ARG candidate whose + * span holds nothing at all (a non-contiguous output allocation) may + * still mint a strictly smaller cross-budget page set. This is what + * makes output buffers resolvable regardless of candidate order or + * allocation placement. + * + * Guards kept from r632/r633: only argument candidates mint, budgets are + * per launch episode, and every synthetic mint additionally respects + * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is + * three-quarters full, no guess may displace the headroom real client + * content (H2D writes) still needs; content serves unconditionally. + */ +#define SVC_GR_SHADOW_NONE 0 +#define SVC_GR_SHADOW_CONTENT 1 +#define SVC_GR_SHADOW_SYNTH 2 +#define SVC_GR_SHADOW_SYNTH_CROSS 3 +/* + * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per + * launch). They apply to CONTENT-anchored spans; the flood class they + * must never re-open (r632: 8192/8192 synthetic pages from answerless + * junk spans) is excluded by the r635 content gate - a green boot has + * no written shadow content at all, so the cross mint is off there, and + * an answerless junk span on a red boot has no head to anchor it. + */ +#define SVC_GR_SHADOW_CROSS_PER_CAND 1024u +#define SVC_GR_SHADOW_CROSS_MAX 4096u +#define SVC_GR_SHADOW_TABLE_GUARD (SVC_VAS_SLOTS * 3u / 4u) + +static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted, + bool span_content, bool launch_content, + u64 push_va, u64 base, bool *real_content) +{ + struct svc_gr_synth_ep *ep = NULL; + bool have; + + if (real_content) + *real_content = false; + if (!svc_vas_can_shadow(va)) + return SVC_GR_SHADOW_NONE; + have = svc_vas_has(va); + if (!have) { + if (!is_arg) + return SVC_GR_SHADOW_NONE; + if (trusted) { + /* + * r633 path: a trusted answer (ipin/resolver/ledger) + * anchors the span; the large r633 budgets apply. + */ + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return SVC_GR_SHADOW_NONE; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return SVC_GR_SHADOW_NONE; + } else if (span_content || launch_content) { + /* + * r635 path: the anchor is the client's own written + * content (its H2D took the shadow lane), not a + * resolver answer. The cross budgets apply - smaller, + * because this serves the output tail of a red-draw + * client, not a trusted span. + */ + struct svc_gr_synth_ep *lep; + + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND) + return SVC_GR_SHADOW_NONE; + lep = svc_gr_synth_ep_get(push_va, 0, false); + if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX) + return SVC_GR_SHADOW_NONE; + } else { + return SVC_GR_SHADOW_NONE; + } + } + { + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return SVC_GR_SHADOW_NONE; + if (have) { + /* + * r635: real_content tells the caller whether these + * bytes are the client's own WRITES (anchoring + * material) or a synthetic zero page minted by an + * earlier pass (servable, but never an anchor - the + * guard against the self-feeding mint loop). + */ + if (real_content) + *real_content = r->content != 0; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n", + va, r->content ? "" : ", synthetic"); + } + return SVC_GR_SHADOW_CONTENT; + } + } + if (trusted) { + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, true); + + ep->n++; + if (lep) + lep->n++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic; cand=%u launch=%u)\n", + va, ep->n, + svc_gr_synth_ep_launch_n(push_va)); + } + return SVC_GR_SHADOW_SYNTH; + } + { + struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, + true); + + if (ep) + ep->cross++; + if (lep) + lep->cross++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n", + va, ep ? ep->cross : 0, lep ? lep->cross : 0); + } + } + return SVC_GR_SHADOW_SYNTH_CROSS; +} + static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 wins[SVC_GR_CAND_MAX]; bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + /* + * r635: launch-level CONTENT anchor for the host-side operand + * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed + * only by the client's own WRITTEN shadow content (H2D bytes that + * took the shadow lane) - a resolver/pin/ledger answer is NOT + * enough, so a green boot cannot enable the cross mint; consumed + * by the cross-mint for answerless argument candidates. + */ + bool launch_arg_content = false; u32 n_cand; u32 off = 0, i, w; u8 kind; @@ -20261,6 +21161,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* + * r635: the launch-level CONTENT anchor for the host-side fallback, + * computed BEFORE the span loop so candidate order cannot decide. + * An argument candidate whose span already holds the client's own + * WRITTEN shadow bytes (r->content; a synthetic zero page does not + * count) marks the launch's operands as host-visible; an answerless + * ARG candidate (the output buffer the client never wrote, possibly + * not contiguous with the inputs) may then mint a strictly bounded + * cross-budget page set instead of failing the whole launch closed. + * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are + * consulted - a real H2D'd buffer always has its head written, and + * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger) + * do NOT arm this: on a green boot the cross mint stays off. + */ +#define SVC_GR_CROSS_ANCHOR_PROBE 64u + { + bool found = false; + + for (w = 0; w < n_cand && !found; w++) { + u32 p; + if (!is_arg[w] || wins[w] < 0x40000000000ULL) + continue; + for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) { + if (svc_vas_has_content(wins[w] + + (u64)p * SEMA_PAGE_SIZE)) { + found = true; + break; + } + } + } + launch_arg_content = found; + } /* m110/B-2: one contiguous run of pages per candidate pointer, * relocated into the active engine-visible data region (grown region * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the @@ -20270,6 +21202,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + bool cand_anchored = false; /* r633: >= 1 page of this span + * resolved through a trusted + * source (ipin/resolve/ledger); + * the r633 large synthetic + * budgets only apply here */ + bool cand_content = false; /* r635: >= 1 page of this span + * holds the client's own + * WRITTEN shadow bytes; the + * cross budgets extend it */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -20343,6 +21284,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 pa = 0; u32 ap = 0, x; struct svc_gr_data *rec; + bool shadow_serve = false; + int serve = SVC_GR_SHADOW_NONE; + bool real_content = false; /* r635 */ if (va >= (1ULL << 47)) break; @@ -20367,6 +21311,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; + cand_anchored = true; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* * C1 (stall/data class): the page may @@ -20385,6 +21330,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ledger_lookup(va, &lpa, &lap)) { pa = lpa; ap = lap; + cand_anchored = true; + } else if ((serve = svc_gr_shadow_fill_serve( + va, is_arg[w], + cand_anchored, + cand_content, + launch_arg_content, + svc_gr_cand.push_va, + wins[w], + &real_content)) != + SVC_GR_SHADOW_NONE) { + /* + * r632/r633: no resolver, no + * landing - serve the page + * from the shadow lane + * instead of dropping it. + * Existing H2D shadow + * content serves always; a + * SYNTHETIC page only when + * the span is anchored by a + * resolved page and the + * per-episode budgets hold. + * The rec is created with + * pa=0 + shadow=1 below; + * the submit build hands + * the shadow page to the + * bounce machinery. Never + * breaks the span: a + * shadow-served page is a + * resolved page. + */ + shadow_serve = true; + /* + * r635: REAL content (the + * client's own written H2D + * bytes, r->content) anchors + * the span for the cross + * budgets and arms the + * launch-level content + * anchor. A synthetic zero + * page served from an + * earlier pass is NOT an + * anchor (the self-feeding + * mint guard). + */ + if (serve == + SVC_GR_SHADOW_CONTENT && + real_content) { + cand_content = true; + if (is_arg[w]) + launch_arg_content = + true; + } } else { /* * m170 (R4a): the very first @@ -20457,6 +21454,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, dropped_pages++; break; } + } else { + /* r633: the trusted resolver answered + * this page - the span is anchored + * (the r633 large budgets apply; + * the r635 cross anchor stays + * content-only). */ + cand_anchored = true; } } /* @@ -20528,10 +21532,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; - rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + rec->pa = shadow_serve ? 0 : + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)); rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); - rec->ap = (u8)svc_ap_of(ap); + rec->ap = shadow_serve ? SVC_APERTURE_SYS : + (u8)svc_ap_of(ap); + rec->shadow = shadow_serve ? 1 : 0; svc_fwd_req.gr_n_data++; /* * r349: record the launch's own resolution as the @@ -20545,8 +21552,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * said SYS 0x7c00000; the D2H read followed the * ledger and returned zeros while the correct sum * sat on the written page). + * + * r632: a shadow-served rec has no guest page to + * record - the shadow is its storage, and the copy + * paths already prefer the shadow for these VAs. */ - if (va >= SVC_VAS_FLOOR) + if (!shadow_serve && va >= SVC_VAS_FLOOR) svc_ledger_store(va, rec->pa, rec->ap); /* * m142: bounded per-page attribution log. A wrong data @@ -20556,8 +21567,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ if (svc_gr_data_log < 192) { svc_gr_data_log++; - pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", - va, rec->pa, rec->ap); + pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u%s\n", + va, rec->pa, rec->ap, + shadow_serve ? " (shadow lane)" : ""); } } (void)base; @@ -20595,7 +21607,14 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (is_arg[w] && svc_fwd_req.gr_n_data == before && svc_gr_src_log < 64) { int vi; - char vb[240]; + /* r614 diag: the r613b red-boot post-mortem could not + * tell which registered pdb would have answered an + * unresolved arg base (the 240-byte buffer truncated + * the walk list at three entries, and only the red + * boot's misses were ever printed). A larger buffer + * with a clamped append keeps every registered + * VASpace's walk verdict visible. */ + char vb[512]; int o = 0; svc_gr_src_log++; @@ -20610,11 +21629,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, gsp_sema_walk_trace != 0); - o += snprintf(vb + o, sizeof(vb) - o, - " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", - vi, svc.vas[vi].pdb, - svc.vas[vi].entries, - svc.vas[vi].ap, wrc, wa); + if (o < (int)sizeof(vb) - 1) + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); } pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", base, pba_src, n_cand, svc_pd_found, @@ -20991,6 +22011,40 @@ static bool svc_gr_defer_expired(u64 push_va) } /* + * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed + * by push VA ALONE, and the UVM pool reuses the same launch push VAs for + * every client of a boot - the exact lifetime bug the session flush already + * fixes for the r565 exactly-once memories (svc_gr_cap_hist, svc_gr_seen, + * svc_inline_once: "matches on the VA ALONE ... a previous client's + * capture could classify a later client's fresh launch"). Carried across + * sessions, the r556 stamp arms once - at the previous client's first + * sighting - so a new client's first decision for a reused VA already reads + * defer_expired()=true: every partial-argument defer fails closed (r586 + * path), the engine never launches, the QMD release is authored in + * software, and the client reads stale/zero buffers. Witnessed r603 family + * boot: the first mail client (fresh VAs 0x20043be94/0x20043c364) deferred + * and launched (`GUEST GR LAUNCH OK`), while every later mail client (VAs + * 0x20043beac/0x20043c37c reused from earlier boot runs) saw defer_exp=1 + * at its FIRST decision and `GR forward REFUSED at defer cap ... argument + * pages still unresolved` for both launches; its "green" was the previous + * client's results still sitting in the reused device pages. + * + * Reset both tables with the session: within a session the original + * bounds stand unchanged (one deferred sighting per VA in the arg window, + * one absolute 15 s cap per VA lifetime); a new session starts with fresh + * push generations and must not inherit either stamp. + */ +static void svc_gr_defer_reset(void) +{ + memset(svc_gr_arg_defer_va, 0, sizeof(svc_gr_arg_defer_va)); + memset(svc_gr_arg_defer_j, 0, sizeof(svc_gr_arg_defer_j)); + svc_gr_arg_defer_pos = 0; + memset(svc_gr_defer_cap_va, 0, sizeof(svc_gr_defer_cap_va)); + memset(svc_gr_defer_cap_j, 0, sizeof(svc_gr_defer_cap_j)); + svc_gr_defer_cap_pos = 0; +} + +/* * r548: is a forward still OWED for this push VA - a captured, never * offered candidate sitting in the candidate ring? The r547 saxpy stall: * both launches were deferred by the r541/r543 guards, the candidate ring @@ -21114,15 +22168,52 @@ static struct svc_channel *svc_pass_local; static u32 svc_pass_local_n; /* - * r590: does any CE ring of this device still hold entries this pass's pump - * has not consumed? Reads the ring's live GP_PUT from its own USERD and - * compares it (modulo the ring size) with the snapshot's last_get. A - * mismatch means the client submitted copies/fills the pump has not decoded - * yet - exactly the stream-order backlog the r541/r568 guards cannot see. + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted copies/fills the pump has not decoded yet - exactly + * the stream-order backlog the r541/r568 guards cannot see. + * + * r607 (r606 red-draw capture, final form of r590b): the engine_type==0 + * filter skipped the channel the guard exists for. The driver's H2D + * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 - + * the r595 finding), and the captured red draw shows the launch forwarding + * while that channel held three published, unconsumed entries behind an + * opdef operand hold: + * 362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0 + * copypend=0 ceback=0 + * 362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer + * retry) -> VADD bad=32735 csum=8139264 (c = a + 0) + * 362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward + * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand + * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the + * only blindness was the class filter - ceback=0 while the copy channel + * was three entries behind. Consider every other channel of the device; + * the launching channel itself is excluded (its own ring legitimately runs + * ahead with future entries). r590's validated part is unchanged: same + * pass snapshot, same bounded busy-defer, historical forward at the + * absolute cap (this guard itself never refuses; the r586 fail-closed + * path is a separate branch that fires only when argument pages never + * resolve). r595's GR-class ops admission has since cut the hold + * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap. + * * Channels whose USERD is unreadable are skipped (their work cannot be * ordered by this mechanism anyway). - */ -static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) + * + * r611: only channels that carry transfer ops are considered (ever_ops). + * The r607 blanket "every other ring" caught sibling GR LAUNCH channels + * whose rings hold the client's own future launches: streams2's two stream + * channels (h=0x5c00001f/23) each deferred on the other's published entry + * for the full 15 s cap, then the r586 fail-closed refused both kernels + * (measured r610c: every streams2 launch REFUSED, bad=65534). A launch + * channel decodes no ops (measured r606/r609/r610c: ops appear only on the + * 0x5c000059 copy family and the CE ranks), so the flag separates them + * cleanly while keeping the r606 red-draw copy channel in scope. + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) { u32 k; @@ -21132,8 +22223,11 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) const struct svc_channel *c = &svc_pass_local[k]; u32 put = 0; - if (!c->valid || (vgpu && c->vgpu != vgpu) || - !c->engine_type) + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->ever_ops) continue; if (!c->gpfifo_entries || !c->userd_pa) continue; @@ -21142,13 +22236,138 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu) c->userd_as, &put, sizeof(put))) continue; put %= c->gpfifo_entries; - if (put != ((u32)c->last_get % c->gpfifo_entries)) + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* + * r638 mutual-cycle gate. The r612/r636 captures + * showed a provable deadlock: a launch L is deferred + * by this watermark because channel X holds an + * unconsumed entry; X's head entry is held by the + * bounded wait gate because its acquire's release is + * authored by L itself, which cannot forward until + * this watermark clears - both halves cite each + * other, and only the 15 s defer cap / the wait's + * fail-open break it (the copy runs ~20 s late; the + * launch finds its operand pages empty). Such a head + * is a consumer-side wait, not a producer backlog: + * skip it and let the forward install the release. + * Bounded by the same guards the watermark's + * consumers already trust: skip only while the + * candidate's operands are fully resolved (no + * stalled fill pages - a launch that still needs + * producer data stays protected) and no decoded + * device write is pending on a data page. Rollback: + * sema_mutual_skip=0. + */ + if (gsp_sema_mutual_skip && + svc_gr_forward_pending() && + !svc_gr_data_stalled && + !svc_gr_dstpend_overlap() && + svc_wait_held_entry(c->h_object, + (u32)c->last_get % c->gpfifo_entries)) { + svc.ce_mutual_skips++; + if (svc.ce_mutual_log < 32) { + svc.ce_mutual_log++; + pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n", + c->h_object, + (u32)c->last_get % + c->gpfifo_entries); + } + continue; + } return true; + } } return false; } /* + * r598 diag: the driver's ring WRITE pointer, not the GPFIFO put. The r568 + * draw needs the copy entries that are already written into the ring but + * whose GP_PUT has not been published yet. When the put says the ring is + * drained, peek at the slots immediately after the consumption position and + * log their {va,len} plus whether the slot already executed that exact + * entry. Bounded per launch, gated by gsp_phys_gr_peek. + */ +u32 gsp_phys_gr_peek; + +static void svc_gr_peek_lookahead(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k, logged = 0, backlog_logged = 0; + + if (!gsp_phys_gr_peek || !svc_pass_local) + return; + /* + * r604 diag: 8 was too small. svc_pass_local carries the six CE ranks + * (engine_type 11/12/13) FIRST and the GR-class channels after, so the + * cap was always spent before the copy-carrying GR-class channels + * (0x5c000059 family - the r595 finding: "the driver's relocation / + * H2D copies live on a compute channel") could be logged. Session 48 + * concluded "the copy channel's lookahead read empty" from captures + * where that channel was never in the logged set at all. 24 covers + * every drained channel of a live client with margin; the lookahead + * depth 6 covers the copy behind the launch's own small pushes. + * + * r604: also name every channel the drained-only filter SKIPS because + * its live GP_PUT reads ahead of the pass consumption position. Those + * are exactly the channels the r590 CE watermark watches - but that + * guard skips engine_type==0, so a published backlog on a GR-class + * copy channel is invisible to every existing guard; the capture must + * show it so a red decision can be attributed to write-ahead (put == + * last, entries logged below) vs published-backlog (put != last, + * logged here) without guessing. + */ + for (k = 0; k < svc_pass_local_n && logged < 24; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0, j; + + if (!c->valid || c->h_object == self->h_object || + !c->gpfifo_va || !c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu, c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* real published backlog */ + if (backlog_logged < 12) { + backlog_logged++; + pr_info("vgsp-gr: peek-backlog h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, + (u32)c->last_get); + } + continue; + } + pr_info("vgsp-gr: peek h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, (u32)c->last_get); + for (j = 1; j <= 6; j++) { + u32 idx = ((u32)c->last_get + j) % c->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len; + u64 va; + + if (svc_read_va(vgpu, + c->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || !len || len > SVC_PUSH_MAX) + break; + pr_info("vgsp-gr: peek +%u idx=%u va=0x%llx len=%u ident_match=%d\n", + j, idx, va, len, + c->slot_ops_ident[idx] == + ((u64)e0 | ((u64)e1 << 32))); + } + logged++; + } +} + +/* * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget * per boot). Logged from svc_gr_request right after the fill, for launches * whose fill saw dropped pages / a partial argument set or whose decision @@ -21172,7 +22391,7 @@ static void svc_gr_dec_dump(const struct svc_channel *ch) dstpend = svc_gr_dstpend_overlap(); copypend = svc_copy_pending(); - ceback = svc_ce_stream_backlog(NULL); + ceback = svc_ce_stream_backlog(NULL, ch); pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, @@ -21318,7 +22537,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (gsp_phys_gr_dec_log_max && (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || svc_gr_data_stalled || svc_gr_dstpend_overlap() || - svc_copy_pending() || svc_ce_stream_backlog(vgpu))) + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) svc_gr_dec_dump(ch); /* * r541: launch-ordering guard. The fill's data pages are what the @@ -21366,6 +22585,24 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* + * r633: deliver the parked push-level release records too - a + * fail-closed refusal still completes the client's tracking + * word (forward-only; a record already applied is skipped). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -21401,7 +22638,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * historical forward stands (witnessed) - the guard delays a launch, * it never refuses one, so it cannot recreate r588's mail refusal. */ - if (svc_ce_stream_backlog(vgpu)) { + svc_gr_peek_lookahead(vgpu, ch); + if (svc_ce_stream_backlog(vgpu, ch)) { if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); @@ -21480,9 +22718,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * no-write). Refuse the forward, author the completion in software, * consume; the dropped pages were already demanded through the pin * channel (r585) so the retry path had its chance first. + * + * r612: refuse only a DEGENERATE fill - one whose dropped set is at + * least as large as the resolved set (r584's 2 resolved / 18 + * dropped). A launch with a full resolved window (969 resolved / + * 9 dropped) whose unresolved candidates are still unresolved at + * the absolute cap has them stalled in EVERY source for the whole + * window; they are the param page's decoy/foreign pointers (the + * r611 streams2 first-client capture: candidates 0x783299000000 / + * 0x783297000000, `stalled VA` from the first fill, no pin, no + * walk, no ledger, never resolving in the run) - waiting longer + * cannot help, and refusing converts a launch whose real operands + * all resolved into a guaranteed no-write client failure (streams2 + * bad=65534). At the cap the copy-related guards (dstpend, CE + * watermark) have already cleared or the refusal branches above + * win first, so a non-degenerate fill forwards historically, + * loudly witnessed. */ if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && - svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_fwd_req.gr_n_data < svc_fwd_req.gr_dropped) { svc.fwd_gr_refused++; svc.fwd_gr_closed++; if (svc_gr_cand.rel_va) { @@ -21491,6 +22746,20 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* r633: deliver the parked push-level release records too. */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -21505,6 +22774,21 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * candidate is owned and sw-completed */ } /* + * r612: see the refusal note above - a non-degenerate fill at the cap + * forwards historically (witnessed once per budget slot). + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_argfw++; + if (svc.fwd_gr_argfw_log < 32) { + svc.fwd_gr_argfw_log++; + pr_info("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (partial argument set: %u dropped of %u resolved; stalled candidates never resolve, forwarding historically)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_n_data); + } + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -21568,6 +22852,38 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; /* candidate kept: the pump retries the entry */ } /* + * r633: this launch may have been held in an earlier pass whose decode + * recorded its push-level release records - records that are per-decode + * state and were parked at the hold's break. Restore them now so the + * caller's hold-engage holds THIS push's records (a queued launch from + * the capture ring is not accompanied by a re-decode, so the current + * svc_fwd_rel[] can be empty or belong to another entry; the r632 mail + * hang: the vadd push's tracking release never landed and the client + * spun in cudaDeviceSynchronize). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, &parked_n)) { + if (parked_n > SVC_FWD_REL_MAX) + parked_n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_rel, parked, sizeof(svc_fwd_rel)); + svc_fwd_rel_n = parked_n; + svc_fwd_defer_armed = true; + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + if (svc_grpark_log < 32) { + svc_grpark_log++; + pr_info("vgsp-work: GR parked releases restored h=0x%x push=0x%llx n=%u va0=0x%llx pay0=%u\n", + ch->h_object, svc_gr_cand.push_va, + parked_n, parked[0].va, + parked[0].payload); + } + } + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -22209,10 +23525,60 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[i].engine_type) local[n++] = svc.chan[i]; } - for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && - !svc.chan[i].engine_type) - local[n++] = svc.chan[i]; + /* + * r605 stream-order, the eng==0 half of the r507o fix. The same + * argument that made CE channels walk before GR ones applies WITHIN + * the GR class: the driver's relocation/H2D copies ride a + * later-registered compute-class channel (h=0x5c000059 family, + * eng=0 - the r595 finding), while the launch channel registers + * early (0x5c000019). The stable partition walked the launch first, + * so at a launch decision the copy channel's PUBLISHED backlog was + * unconsumed and invisible to every guard: the r590 watermark skips + * engine_type==0, and the r541/r568 gates only see DECODED copies. + * r604 capture (fresh boot, peek armed): at the vadd's second + * decision `peek-backlog h=0x5c000059 eng=0 put=95 last=94` - the + * b-copy entry was already in the ring with GP_PUT ahead of the + * pump, executed 8 ms after the decision and 320 ms before the + * launch's copy-in, and the run was green only because the + * partial-argument defer had stretched the window; pins landing one + * pass earlier would have forwarded against the pre-copy page + * (c = a + 0, the r568 red draw). Newest-registered first + * (descending h_object) walks the copy / push-library channels + * before the compute channels, so every launch decision sees the + * copy ring consumed to its live write pointer (GP_PUT): the + * write-pointer drain, by ordering. No deferral, no cap + * interaction, no speculative execution - the pump still consumes + * each entry exactly once through its normal per-slot identity + * bookkeeping (slot_ops_ident/done/complete/hash), so nothing needs + * mirroring. A client whose compute channel registers last simply + * keeps the historical order (no worse). + */ + { + u32 prev_h = 0xffffffffu; + int picked; + + for (picked = 0; picked < SVC_MAX_CHANNELS; picked++) { + int best = -1; + u32 best_h = 0; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || + c->engine_type) + continue; + if (c->h_object >= prev_h) + continue; + if (best < 0 || c->h_object > best_h) { + best = i; + best_h = c->h_object; + } + } + if (best < 0) + break; + local[n++] = svc.chan[best]; + prev_h = best_h; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -23124,9 +24490,30 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc.op_tr_pending = 0; svc.op_owed = 0; + /* + * r595: a GR-class channel + * (engine_type 0) carries the + * driver's own relocation / + * H2D copies as well as its + * launches, but ops were only + * allowed for CE channels (or + * an injected push page). The + * copies then never executed + * and their releases were + * applied by the hold path - + * exactly the r568 b-copy + * class. Admit a GR push's + * ops when its content is a + * plausible method stream + * (cap_trust is that same + * content arbiter, set just + * above), never on a replay + * or a rehold. + */ svc_decode_stream(vgpu, pb, len, 0, - (allow_ops || injected) && + (allow_ops || injected || + cap_trust) && !entry_replay && !rehold, ch, @@ -23421,6 +24808,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_busy_defer_log < 64) { svc_busy_defer_log++; @@ -23498,6 +24900,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them (the r632 + * mail hang: the vadd push's + * tracking release was lost + * here and the client spun + * at cudaDeviceSynchronize). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; @@ -26408,3 +27829,68 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } + +/* + * r616 diag: manual fault poke for the refused-launch hang experiment. A GR + * push that the fill cannot resolve is refused (degenerate data set) and the + * entry is consumed with a software release; the authored release lands in + * the guest page (measured: kcore read 0x6 == pay on r613d) yet the guest + * driver never completes cudaDeviceSynchronize for a launch that never ran. + * This writes the channel's error notifier in the exact shape the relayed- + * fault edge uses (info32=exceptType, info16=engine, status=0xffff, latched + * by notifier_err) and posts the channel event, so a hung client's kernel + * can be poked into observing an engine error and ending its wait. Debug + * only; harmless when no client is hung (the pump's completion clear resumes + * after SVC_NOTIFIER_ERR_HOLD). h_object 0 = every valid channel. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception */ + int i, done = 0; + + if (!vgpu) + return -ENODEV; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + unsigned long flags; + u64 pa; + u32 as, eng, h; + u16 info16; + + spin_lock_irqsave(&svc.chan_lock, flags); + if (!ch->valid || ch->vgpu != vgpu || !ch->notifier_pa || + (h_object && ch->h_object != h_object)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + continue; + } + h = ch->h_object; + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + /* Latch exactly like the relayed-fault edge so the pump's + * completion clear cannot wipe the poke before the guest + * reads it. */ + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + info16 = (u16)eng; + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info("vgsp-fault: POKE ch=0x%x notifier write failed pa=0x%llx as=%u\n", + h, pa, as); + continue; + } + pr_info("vgsp-fault: POKE ch=0x%x engine=%u pa=0x%llx as=%u status=0x%x except=0x%x (manual experiment)\n", + h, eng, pa, as, status, info32); + gsp_rm_post_channel_event(vgpu, h); + done++; + } + return done ? done : -ENOENT; +} diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 31512ef..fbc7e4d 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -44,6 +44,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); /* + * r616 diag: manually write a channel's error notifier (engine-exception + * shape) and post its channel event, to test whether a refused launch's + * hung client can be ended by a fault-conversion edge. h_object 0 = every + * valid channel of the device. Debug only. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object); + +/* * Release every piece of service state that belongs to one mdev before its * guest session disappears: drop the mdev's channels (per-vgpu session end), * clear a release hold that names it (never dereference a removed device), @@ -259,6 +267,7 @@ extern u64 gsp_sema_copy_dst2; /* CE pending-entry grace window in ms before the release fallback fires. */ extern unsigned gsp_sema_ce_grace_ms; +extern u32 gsp_phys_gr_peek; /* * Bounded work-before-release hold (ms) for an unreadable CE entry whose @@ -328,6 +337,10 @@ extern bool gsp_sema_pool_hold_fresh; * walk/RAM scan until a pin confirms (default on). */ extern bool gsp_sema_copy_dst_pin_guard; +/* r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms). 0 restores the per-page windows (pre-r609). */ +extern unsigned gsp_sema_copy_dst_age_ms; extern bool gsp_sema_wait_gate; /* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the * emulated launch completion latency for copy-after-launch ordering. */ diff --git a/kernel/main.c b/kernel/main.c index d821762..dd11d12 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2752,6 +2752,10 @@ MODULE_PARM_DESC(sema_copy_dst2, "Second destination window for sema_copy_dst (d * the release the client synchronizes on; the translation usually lands within * a service pass. */ +module_param_named(gsp_phys_gr_peek, gsp_phys_gr_peek, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_peek, + "Peek the ring slots past GP_PUT at each GR launch decision (diagnostic, default 0)"); + module_param_named(sema_ce_grace_ms, gsp_sema_ce_grace_ms, uint, 0644); MODULE_PARM_DESC(sema_ce_grace_ms, "CE pending-entry grace window in ms (default 25000)"); @@ -2872,6 +2876,14 @@ MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644); MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)"); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); +/* + * r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms) - the r606 held-chain serialization fix. 0 restores the + * per-page windows (pre-r609). + */ +module_param_named(sema_copy_dst_age_ms, gsp_sema_copy_dst_age_ms, uint, 0644); +MODULE_PARM_DESC(sema_copy_dst_age_ms, "device-dst guard burst-ageing window after an un-pinned release, ms (r609, default 3000; 0 off)"); /* * r384 (guest boot-corruption lottery): fill-destination guards. A large @@ -3881,5 +3893,143 @@ static const struct kernel_param_ops nv_vgpu_rmpost_ops = { module_param_cb(rmpost, &nv_vgpu_rmpost_ops, NULL, 0200); MODULE_PARM_DESC(rmpost, "Post a GSP-RM event to the guest by event index (debug)"); +/* + * Debug: poke a channel's error notifier with an engine-exception shape and + * post its channel event (the relayed-fault delivery edge, manually): + * echo 0x5c000019 > .../parameters/grfaultpoke + * echo 0 > .../parameters/grfaultpoke (every valid channel) + * Experiment: a refused/degenerate GR launch ends its client's + * cudaDeviceSynchronize spin if a synthetic engine error completes the wait + * (r616; see gsp_sema_channel_fault_poke). + */ +struct nv_vgpu_faultpoke_req { + u32 h_object; + int rc; + struct completion done; +}; + +static int nv_vgpu_faultpoke_thread(void *arg) +{ + struct nv_vgpu_faultpoke_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + req->rc = gsp_sema_channel_fault_poke(vgpu, req->h_object); + pr_info("vgsp: grfaultpoke h=0x%x -> %d\n", + req->h_object, req->rc); + } + mutex_unlock(&nv_vgpu_driver.lock); + + complete(&req->done); + return 0; +} + +static int nv_vgpu_faultpoke_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_faultpoke_req req; + struct task_struct *task; + + memset(&req, 0, sizeof(req)); + req.rc = -ENODEV; + if (kstrtou32(val, 0, &req.h_object)) + return -EINVAL; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_faultpoke_thread, &req, "vgpu-faultpoke"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_faultpoke_ops = { + .set = nv_vgpu_faultpoke_set, +}; + +module_param_cb(grfaultpoke, &nv_vgpu_faultpoke_ops, NULL, 0200); +MODULE_PARM_DESC(grfaultpoke, "Write a channel error notifier + post its event (r616 experiment)"); + +/* + * Debug r623: deliver a synthetic OS_ERROR_LOG (0x1006) engine-fault report + * to the guest's RM queue: + * echo 1 > .../parameters/oserrlog + * echo 0x1,0,0 > .../parameters/oserrlog (exceptType,runlistId,chid) + * Experiment: after a synth-fault, the next client sticks pre-submission; if + * the guest's fault handling completes only when the 0x1006 report arrives, + * this record is the missing recovery half. + */ +struct nv_vgpu_oserrlog_req { + u32 except_type, runlist_id, chid; + int rc; + struct completion done; +}; + +static int nv_vgpu_oserrlog_thread(void *arg) +{ + struct nv_vgpu_oserrlog_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + req->rc = gsp_rm_send_os_error_log(vgpu, req->except_type, + req->runlist_id, req->chid, + "GR"); + pr_info("vgsp: oserrlog except=0x%x rl=%u chid=%u -> %d\n", + req->except_type, req->runlist_id, req->chid, req->rc); + } + mutex_unlock(&nv_vgpu_driver.lock); + + complete(&req->done); + return 0; +} + +static int nv_vgpu_oserrlog_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_oserrlog_req req; + struct task_struct *task; + char buf[64]; + char *c1, *c2; + + memset(&req, 0, sizeof(req)); + req.rc = -ENODEV; + strscpy(buf, val, sizeof(buf)); + req.except_type = 1; /* generic engine exception, as the synth-fault */ + { + c1 = strchr(buf, ','); + if (c1) { + *c1++ = '\0'; + c2 = strchr(c1, ','); + if (c2) { + *c2++ = '\0'; + if (kstrtou32(c2, 0, &req.chid)) + return -EINVAL; + } + if (kstrtou32(c1, 0, &req.runlist_id)) + return -EINVAL; + } + if (kstrtou32(buf, 0, &req.except_type)) + return -EINVAL; + } + + init_completion(&req.done); + task = kthread_run(nv_vgpu_oserrlog_thread, &req, "vgpu-oserrlog"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_oserrlog_ops = { + .set = nv_vgpu_oserrlog_set, +}; + +module_param_cb(oserrlog, &nv_vgpu_oserrlog_ops, NULL, 0200); +MODULE_PARM_DESC(oserrlog, "Send a synthetic OS_ERROR_LOG (0x1006) RM record to the guest (r623 experiment)"); + module_init(nv_vgpu_init); module_exit(nv_vgpu_exit); |