diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 629 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 76 | ||||
| -rw-r--r-- | kernel/gsp_phys_fwsec.c | 22 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 324 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 27 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 223 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 6 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 180 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 11 | ||||
| -rw-r--r-- | kernel/main.c | 24 | ||||
| -rwxr-xr-x | tools/guest-diag/phys_claim_cycle.sh | 52 |
11 files changed, 1440 insertions, 134 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 255d79e..f9c0947 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -955,6 +955,25 @@ static struct gsp_phys_boot_stage { struct gsp_phys_boot_chunk logrm; /* LOGRM task log */ } boot_stage; +/* + * Live-session accessor for the P1 fail-open check: the real GSP-RM session + * is usable only after stage 3e saw the init-done event and before the + * UNLOADING_GUEST_DRIVER RPC has been published in this claim. + */ +bool gsp_phys_boot_rm_ready(void) +{ + return boot_stage.rm_init_done && !boot_stage.rm_unload_submitted; +} +EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_ready); + +/* Companion detail for the fail-open log line (see gsp_phys_backend_start_work). */ +void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted) +{ + *init_done = boot_stage.rm_init_done; + *unload_submitted = boot_stage.rm_unload_submitted; +} +EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags); + static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { @@ -4163,6 +4182,15 @@ static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p, } /* + * Decoded function-65 snapshot cache (P2). Filled by gsp_phys_sci_decode() + * under gsp_phys_rm_lock, served by gsp_phys_backend_static_get() only while + * the cache generation still matches the backend session's and the session is + * READY, and dropped by gsp_phys_backend_static_invalidate() when the backend + * leaves READY. + */ +static struct gsp_phys_backend_static_config gsp_phys_backend_static; + +/* * Decode the GspStaticConfigInfo fields this module needs. Every field is * bounds-checked against the reply payload length, so a short or older reply * logs only what it carries. The struct itself never lands on the stack. @@ -4243,6 +4271,84 @@ static void gsp_phys_sci_decode(const u8 *p, u32 len) boot_stage.rm_probe_internal_valid = true; } } + + /* + * Fill the backend static-config snapshot from this decode. Every + * decoded field is copied (fb geometry/PDE bases are kept for later + * phases), and valid mirrors the all-three-handles-nonzero test above + * so the guest overlay and the rm-call stage-5 adoption agree. The + * generation key ties the cache to the backend session; the getter + * refuses it once the session leaves READY. + */ + gsp_phys_backend_static.valid = false; + gsp_phys_backend_static.generation = gsp_phys_backend_generation(); + gsp_phys_backend_static.fw_crc32 = gsp_phys_gpu_fw_crc32(); + gsp_phys_backend_static.payload_len = len; + gsp_phys_backend_static.h_internal_client = + boot_stage.rm_probe_h_internal_client; + gsp_phys_backend_static.h_internal_device = + boot_stage.rm_probe_h_internal_device; + gsp_phys_backend_static.h_internal_subdevice = + boot_stage.rm_probe_h_internal_subdevice; + gsp_phys_backend_static.bar1_pde_base = + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF); + gsp_phys_backend_static.bar2_pde_base = + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF); + gsp_phys_backend_static.fb_length = + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF); + gsp_phys_backend_static.fbio_mask = + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF); + gsp_phys_backend_static.fbp_mask = + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF); + gsp_phys_backend_static.fb_bus_width = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF); + gsp_phys_backend_static.fb_ram_type = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF); + gsp_phys_backend_static.l2_cache_size = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF); + gsp_phys_backend_static.engine_caps[0] = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF); + gsp_phys_backend_static.engine_caps[1] = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4); + gsp_phys_backend_static.engine_caps[2] = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8); + strscpy(gsp_phys_backend_static.gpu_name, name, + sizeof(gsp_phys_backend_static.gpu_name)); + strscpy(gsp_phys_backend_static.gpu_short_name, short_name, + sizeof(gsp_phys_backend_static.gpu_short_name)); + gsp_phys_backend_static.sriov_max_gfid = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF); + gsp_phys_backend_static.vbios_sub_vendor = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF); + gsp_phys_backend_static.vbios_sub_device = + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF); + gsp_phys_backend_static.b_vbios_valid = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF); + gsp_phys_backend_static.b_page_retirement = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF); + gsp_phys_backend_static.b_split_vas = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF); + gsp_phys_backend_static.b_self_hosted = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF); + gsp_phys_backend_static.b_ats_supported = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF); + gsp_phys_backend_static.b_poison_fuse = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF); + gsp_phys_backend_static.b_gpu_uefi = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF); + gsp_phys_backend_static.b_efi_init = + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF); + gsp_phys_backend_static.valid = boot_stage.rm_probe_internal_valid; + + pr_info("gsp-phys-boot: rm-probe: static snapshot cached: generation=%u fw_crc32=0x%08x payload=%u name='%s' handles=0x%08x/0x%08x/0x%08x valid=%u\n", + gsp_phys_backend_static.generation, + gsp_phys_backend_static.fw_crc32, + gsp_phys_backend_static.payload_len, + gsp_phys_backend_static.gpu_name, + gsp_phys_backend_static.h_internal_client, + gsp_phys_backend_static.h_internal_device, + gsp_phys_backend_static.h_internal_subdevice, + gsp_phys_backend_static.valid); } /* @@ -4447,8 +4553,6 @@ static int gsp_phys_rm_probe(void __iomem *bar0) * Backend session transport API (P0 skeleton, inert) * ============================================================ */ -static struct gsp_phys_backend_static_config gsp_phys_backend_static; - int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, void **reply, u32 *reply_len, u32 timeout_ms) { @@ -4549,7 +4653,77 @@ out: return ret; } -int gsp_phys_backend_static_snapshot(struct gsp_phys_backend_static_config *out) +/* + * Bounded wait for the snapshot cache to become valid (P2). Polls in 10 ms + * sleeps and never holds gsp_phys_rm_lock across a sleep (each poll takes and + * releases it), so it is safe from the emulation thread. Fails fast with + * -EAGAIN when no session can produce a snapshot (OFF/STOPPING); returns + * -ENODATA when the session is live but the cache did not arrive in time. + * The caller fails open to the emulated model. + */ +#define GSP_PHYS_STATIC_WAIT_POLL_MS 10 + +int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, + u32 timeout_ms) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + int state; + int ret; + + for (;;) { + ret = gsp_phys_backend_static_get(out); + if (!ret) + return 0; + + state = gsp_phys_backend_state(); + if (state != GSP_PHYS_BACKEND_READY && + state != GSP_PHYS_BACKEND_CLAIMING && + state != GSP_PHYS_BACKEND_BOOTING) + return -EAGAIN; + + if (time_after_eq(jiffies, deadline)) + return -ENODATA; + msleep(GSP_PHYS_STATIC_WAIT_POLL_MS); + } +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); + +/* + * Drop the cached static-config snapshot. Called when the backend session + * leaves READY (or a new claim starts): the cache generation key is bumped + * past the current backend generation so a concurrent getter cannot serve the + * previous session's decode; the next successful fn65 decode re-fills and + * re-keys it. Takes gsp_phys_rm_lock to serialise against the decode. + */ +void gsp_phys_backend_static_invalidate(void) +{ + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_backend_static.valid = false; + gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + mutex_unlock(&gsp_phys_rm_lock); +} + +/* + * Run one fn1+fn65 probe on a READY backend to fill the static snapshot + * cache. Takes gsp_phys_rm_lock inside gsp_phys_rm_probe(), so the caller + * must not hold it. Non-fatal: a failed probe leaves the cache as it was and + * the caller keeps the emulation path. + */ +int gsp_phys_backend_static_probe(void) +{ + void __iomem *bar0; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) + return -ENODEV; + + return gsp_phys_rm_probe(bar0); +} + +int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) { u32 generation; @@ -4566,15 +4740,18 @@ int gsp_phys_backend_static_snapshot(struct gsp_phys_backend_static_config *out) mutex_unlock(&gsp_phys_rm_lock); return -EAGAIN; } - if (!gsp_phys_backend_static.valid) { + if (!gsp_phys_backend_static.valid || + gsp_phys_backend_static.generation != generation) { mutex_unlock(&gsp_phys_rm_lock); - pr_debug("gsp-phys-boot: backend static snapshot: cache not populated yet\n"); + pr_debug("gsp-phys-boot: backend static snapshot: no valid cache for generation %u\n", + generation); return -ENODATA; } *out = gsp_phys_backend_static; mutex_unlock(&gsp_phys_rm_lock); return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call @@ -5780,18 +5957,20 @@ static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data) } /* - * CE method block for one 4 KB sysmem-to-sysmem pitch copy with a release - * semaphore, in the all-virtual user-channel shape: the semaphore methods - * always carry a GPU VA (no mode bit; uvm_pascal_ce.c:42-63) and the src/dst - * VAs translate through the channel PDB, so no SET_SRC/DST_PHYS_MODE pair is - * pushed (uvm_maxwell_ce.c:179-205). The method order mirrors + * CE method block for one bytes-long sysmem-to-sysmem pitch copy with a + * release semaphore, in the all-virtual user-channel shape: the semaphore + * methods always carry a GPU VA (no mode bit; uvm_pascal_ce.c:42-63) and the + * src/dst VAs translate through the channel PDB, so no SET_SRC/DST_PHYS_MODE + * pair is pushed (uvm_maxwell_ce.c:179-205). The method order mirrors * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/ * mem_utils_gm107.c:1968-2118) and the semaphore payload block * (channel_utils.c:812-820). Returns the block size in dwords (13 method - * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). + * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder + * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes + * the caller's size (both within one 4 KB ladder page). */ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, - u64 dst_va) + u64 dst_va, u32 bytes) { u32 *p = pb; @@ -5807,16 +5986,13 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, (u32)(dst_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER, (u32)dst_va); - p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT, - GSP_PHYS_RMCALL_COPY_SIZE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT, bytes); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER, (u32)(src_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER, (u32)src_va); - p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN, - GSP_PHYS_RMCALL_COPY_SIZE); - p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN, - GSP_PHYS_RMCALL_COPY_SIZE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN, bytes); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN, bytes); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA, GSP_PHYS_RMCALL_LAUNCH_DMA); @@ -5824,6 +6000,326 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, } /* + * Result of one ladder CE submit, valid only when the submit helper returned 0. + * Kept for the caller's log line and the GP_GET/GPPut context. + */ +struct gsp_phys_rmcall_ce_submit { + u32 words; + u32 slot; + u32 gpput; + u32 gpget; + u32 e0; + u32 e1; + u32 doorbell; + u32 runlist_id; + u32 ch_id; + u64 pb_dma; + u64 sem_dma; + u64 src_dma; + u64 dst_dma; +}; + +/* + * One CE copy block on the ladder channel: clear the module semaphore, build + * the methods for bytes into the module pushbuffer, publish the GPFIFO entry + * and USERD GP_PUT and ring the doorbell. The caller must hold + * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical + * lock-across-poll; the P5 work API polls without the lock). Every guard of + * the original stage-4 submission body is preserved here: server-memory + * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the + * token check. The semaphore is cleared before the doorbell so a stale release + * from a previous run cannot be observed as this submission's completion. + */ +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + struct gsp_phys_rmcall_ce_submit *out) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + struct gsp_phys_boot_chunk *ring = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + struct gsp_phys_boot_chunk *userd_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + resource_size_t bar0_len; + u32 *pb_va, *userd, *ring_va; + u32 words, gpput, gpget, slot, e0, e1; + u32 runlist_id, ch_id, doorbell; + u64 pb_gpu_va; + + if (boot_stage.rmcall_channel_server_mem) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_gpu_va = boot_stage.rmcall_va_pb; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + + pb_va = gsp_phys_chunk_page_va(pb, 0); + ring_va = gsp_phys_chunk_page_va(ring, 0); + userd = gsp_phys_chunk_page_va(userd_c, 0); + if (!pb_va || !ring_va || !userd || + !gsp_phys_chunk_page_va(sem, 0) || + !gsp_phys_chunk_page_va(src, 0) || + !gsp_phys_chunk_page_va(dst, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, + bytes); + + gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + return -EIO; + } + slot = gpput; + + /* + * e0: bits 31:2 hold the GET field, which itself is (fetch address + * >> 2); the engine reconstructs (field << 2) = (e0 & ~3), so for a + * 4 KB-aligned pushbuffer VA the raw word is LO32(VA) with bits 1:0 + * clear. Storing LO32(VA) >> 2 here shifts a second time in the + * engine (the 0x40000 fetch for pb VA 0x101000). FETCH/NO_CONTEXT_ + * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte + * and LENGTH 30:10 in dwords, exactly the reference RM's own submission + * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get) + * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN). + * The engine translates this VA through the channel PDB built in + * gsp_phys_rmcall_pdb_build. + */ + e0 = (u32)pb_gpu_va & ~3u; + if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) + pr_err("gsp-phys-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + e0, (u64)(e0 & ~3u), pb_gpu_va); + e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); + put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); + put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); + dma_wmb(); + /* + * GP_PUT advances on the client sysmem USERD chunk, which is the + * channel's hardware USERD because the params carry that same chunk; + * the host write is therefore what the engine polls (cycle 21). + */ + put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + /* + * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | + * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated + * token already is that value; log its runlist/ChID decomposition and + * write the formula's result (identical unless the token carries stray + * high bits, which is logged). + */ + runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; + ch_id = boot_stage.rmcall_token & 0xfffu; + doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); + if (doorbell != boot_stage.rmcall_token) + pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + boot_stage.rmcall_token, doorbell); + wmb(); + writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); + + if (out) { + out->words = words; + out->slot = slot; + out->gpput = gpput; + out->gpget = gpget; + out->e0 = e0; + out->e1 = e1; + out->doorbell = doorbell; + out->runlist_id = runlist_id; + out->ch_id = ch_id; + out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); + out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); + out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); + out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + } + return 0; +} + +/* + * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain + * increments from the service thread and the debugfs paths; only logging and + * the debugfs state file read them. + */ +#define GSP_PHYS_BACKEND_WORK_POLL_MAX_MS 200u + +static u64 gsp_phys_work_submitted; +static u64 gsp_phys_work_released; +static u64 gsp_phys_work_timed_out; +static u64 gsp_phys_work_refused; + +bool gsp_phys_backend_work_ready(void) +{ + if (!gsp_phys_work_forward_enabled()) + return false; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return false; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return false; + if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || + !boot_stage.rmcall_token) + return false; + if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo || + !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_dst || + !boot_stage.rmcall_va_sem || !boot_stage.rmcall_pdb_root) + return false; + return true; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready); + +bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, + u32 *bytes) +{ + if (!gsp_phys_backend_work_ready()) + return false; + if (src_va) + *src_va = boot_stage.rmcall_va_src; + if (dst_va) + *dst_va = boot_stage.rmcall_va_dst; + if (sem_va) + *sem_va = boot_stage.rmcall_va_sem; + if (bytes) + *bytes = GSP_PHYS_RMCALL_COPY_SIZE; + return true; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_work_operands); + +int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 poll_ms) +{ + struct gsp_phys_rmcall_ce_submit sub; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + void __iomem *bar0; + unsigned long deadline; + u32 poll; + int ret; + + if (!gsp_phys_backend_work_ready()) { + gsp_phys_work_refused++; + return -EAGAIN; + } + if (!bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) { + gsp_phys_work_refused++; + return -EINVAL; + } + if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) + poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; + + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + gsp_phys_work_refused++; + return -EAGAIN; + } + + gsp_phys_work_submitted++; + mutex_lock(&gsp_phys_rm_lock); + /* + * Re-check under the lock: the release may have committed between the + * readiness gate and the lock, and the ladder state may have changed. + * No device write happens on any of these paths. + */ + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || + boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_work_refused++; + if ((gsp_phys_work_refused & 63) == 1) + pr_info("gsp-phys-work: shadow submit refused before the doorbell (backend changed) refused=%llu\n", + gsp_phys_work_refused); + return -EAGAIN; + } + ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, + bytes, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + gsp_phys_work_refused++; + if ((gsp_phys_work_refused & 63) == 1) + pr_info("gsp-phys-work: shadow submit refused (ret %d) refused=%llu\n", + ret, gsp_phys_work_refused); + return ret; + } + + /* + * Poll the module semaphore WITHOUT gsp_phys_rm_lock: the release is + * written by the engine, never by a host path, so no lock is needed + * and holding it across the poll would stall the RPC path (plan ยง1.2). + */ + deadline = jiffies + msecs_to_jiffies(poll_ms); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + gsp_phys_work_timed_out++; + if ((gsp_phys_work_timed_out & 63) == 1) + pr_info("gsp-phys-work: shadow copy timeout after %u ms (sem=0x%08x want=0x%08x) timeouts=%llu\n", + poll_ms, poll, + GSP_PHYS_RMCALL_SEM_PAYLOAD, + gsp_phys_work_timed_out); + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + /* + * Only the ladder operands are verified: the engine copied within the + * module-owned src/dst pages, so a mismatch is a real engine/encode + * failure. A caller with different VAs (none in wave 1) is not + * verified against the ladder pages. + */ + if (src_va == boot_stage.rmcall_va_src && + dst_va == boot_stage.rmcall_va_dst && + memcmp(gsp_phys_chunk_page_va(src, 0), + gsp_phys_chunk_page_va(dst, 0), bytes) != 0) { + gsp_phys_work_refused++; + pr_info("gsp-phys-work: semaphore observed but the destination does not match the source after %u bytes (refused=%llu)\n", + bytes, gsp_phys_work_refused); + return -EIO; + } + + gsp_phys_work_released++; + return 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_ce_copy_submit); + +void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, + u64 *timed_out, u64 *refused) +{ + if (submitted) + *submitted = gsp_phys_work_submitted; + if (released) + *released = gsp_phys_work_released; + if (timed_out) + *timed_out = gsp_phys_work_timed_out; + if (refused) + *refused = gsp_phys_work_refused; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters); + +/* * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method @@ -5848,20 +6344,18 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, */ static int gsp_phys_rmcall_stage4(void __iomem *bar0) { - struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; - struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + struct gsp_phys_rmcall_ce_submit sub; u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; /* NVA06F engineType */ u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; /* NVC36F index */ u8 sched_params[2] = { 1, 0 }; /* NVA06F bEnable=1, bSkipSubmit=0 */ resource_size_t bar0_len; - u32 *pb_va, *userd, *ring_va; - u32 words, gpput, gpget, slot, e0, e1, poll; - u32 token, rpc_result, status, runlist_id, ch_id, doorbell; - u64 pb_dma, sem_dma, src_dma, dst_dma; + u32 *userd; + u32 gpput, gpget, poll; + u32 token, rpc_result, status; unsigned long deadline; int ret; @@ -5992,79 +6486,32 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) } boot_stage.rmcall_token = token; - pb_va = gsp_phys_chunk_page_va(pb, 0); - ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); - pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); - sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); - src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); - dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); - words = gsp_phys_rmcall_build_methods(pb_va, GSP_PHYS_RMCALL_VA_SEM, - GSP_PHYS_RMCALL_VA_SRC, - GSP_PHYS_RMCALL_VA_DST); - - gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); - if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", - gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); - return -EIO; - } - slot = gpput; - - /* - * e0: bits 31:2 hold the GET field, which itself is (fetch address - * >> 2); the engine reconstructs (field << 2) = (e0 & ~3), so for a - * 4 KB-aligned pushbuffer VA the raw word is LO32(VA) with bits 1:0 - * clear. Storing LO32(VA) >> 2 here shifts a second time in the - * engine (the 0x40000 fetch for pb VA 0x101000). FETCH/NO_CONTEXT_ - * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte - * and LENGTH 30:10 in dwords, exactly the reference RM's own submission - * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get) - * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN). - * The engine translates this VA through the channel PDB built in - * gsp_phys_rmcall_pdb_build. - */ - BUILD_BUG_ON(GSP_PHYS_RMCALL_VA_PB & 3); - e0 = (u32)GSP_PHYS_RMCALL_VA_PB & ~3u; - if (unlikely(e0 != (u32)((GSP_PHYS_RMCALL_VA_PB >> 2) << 2))) - pr_err("gsp-phys-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", - e0, (u64)(e0 & ~3u), (u64)GSP_PHYS_RMCALL_VA_PB); - e1 = (u32)((GSP_PHYS_RMCALL_VA_PB >> 32) & 0xffu) | (words << 10); - put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); - put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); - dma_wmb(); /* - * GP_PUT advances on the client sysmem USERD chunk, which is the - * channel's hardware USERD because the params carry that same chunk; - * the host write is therefore what the engine polls (cycle 21). + * The submission body is shared with the P5 shadow API + * (gsp_phys_rmcall_ce_submit_locked): method block, GPFIFO entry, + * USERD GP_PUT and doorbell, with the same guards. Stage 4 keeps its + * historical lock-across-poll: gsp_phys_rmcall_run() holds + * gsp_phys_rm_lock for the whole stage. */ - put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, - (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); - /* - * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | - * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated - * token already is that value; log its runlist/ChID decomposition and - * write the formula's result (identical unless the token carries stray - * high bits, which is logged). - */ - runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; - ch_id = boot_stage.rmcall_token & 0xfffu; - doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); - if (doorbell != boot_stage.rmcall_token) - pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", - boot_stage.rmcall_token, doorbell); - wmb(); - writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); + ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sub); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", + ret); + return ret; + } pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", - pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, - words, slot, gpget, gpput, e0, e1, - boot_stage.rmcall_token, runlist_id, ch_id, doorbell, - GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, + sub.words, sub.slot, sub.gpget, sub.gpput, sub.e0, sub.e1, + boot_stage.rmcall_token, sub.runlist_id, sub.ch_id, sub.doorbell, + GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma, GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS); @@ -6103,9 +6550,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", - poll, src_dma, dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, + poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), - words, boot_stage.rmcall_token); + sub.words, boot_stage.rmcall_token); return 0; } diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 15068ff..2c24524 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -26,6 +26,15 @@ bool gsp_phys_boot_dryrun_enabled(void); /* True when either boot parameter is set (forces the stage-3b layout). */ bool gsp_phys_boot_requested(void); +/* + * Live-session accessor: true only when the real GSP-RM session is usable + * (stage 3e saw init-done and the UNLOADING_GUEST_DRIVER RPC has not been + * submitted in this claim). The backend marks itself READY only when this is + * true; gsp_phys_boot_rm_flags() exposes the underlying bits for logging. + */ +bool gsp_phys_boot_rm_ready(void); +void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted); + /* Views into the staged GSP firmware ELF (set by gsp_phys_gpu.c). */ struct gsp_phys_boot_sources { const u8 *fw_image; /* .fwimage section of the GSP-RM ELF */ @@ -102,14 +111,22 @@ void gsp_phys_boot_staging_free(struct pci_dev *pdev); * not live. On success *reply owns a kmalloc'd copy of the reply payload * (*reply_len bytes, NULL/0 for an empty payload, caller frees). * - * gsp_phys_backend_static_snapshot() returns the cached static-config fields - * decoded from the function-65 reply; the cache is filled at READY in a - * later phase, so it currently reports -ENODATA. + * gsp_phys_backend_static_get() returns the cached static-config fields + * decoded from the function-65 reply; the cache is filled by the READY-time + * probe (gsp_phys_backend_static_probe() on the backend work item) and is + * invalidated whenever the backend session leaves READY. It returns -ENODATA + * while no valid cache exists and -EAGAIN when the session is not READY or + * the cache's generation key no longer matches the backend's. + * + * gsp_phys_backend_static_wait() polls the same getter in bounded sleeps + * (never holding gsp_phys_rm_lock) and fails open with -ENODATA/-EAGAIN so a + * caller can fall back to the emulated model. */ #define GSP_PHYS_BACKEND_GPU_NAME_LEN 64 struct gsp_phys_backend_static_config { bool valid; /* cache holds a decoded function-65 reply */ + u32 generation; /* backend session generation the fill saw */ u32 fw_crc32; /* cache key: loaded firmware image */ u32 payload_len; /* decoded reply payload bytes */ u32 h_internal_client; @@ -141,7 +158,58 @@ struct gsp_phys_backend_static_config { int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, void **reply, u32 *reply_len, u32 timeout_ms); -int gsp_phys_backend_static_snapshot(struct gsp_phys_backend_static_config *out); +int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out); +int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, + u32 timeout_ms); +void gsp_phys_backend_static_invalidate(void); + +/* + * Run one fn1+fn65 probe on a READY backend to fill the static snapshot + * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must + * run on a work item, never inline under a caller-held rm lock or on the mdev + * create path. Non-fatal: a failed probe leaves the cache as it was and the + * caller keeps the emulation path. + */ +int gsp_phys_backend_static_probe(void); + +/* + * P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1; default off). + * + * gsp_phys_backend_work_ready() is a lock-free readiness gate: the + * gsp_phys_work_forward parameter is set, the backend session is READY, the + * claim is live with GSP-RM init-done and no unload submitted, and the debugfs + * ladder has a usable client-memory channel (channel handle, work-submit + * token, non-server memory, PDB window and all ladder VAs present). It never + * waits on gsp_phys_rm_lock; every submit re-checks the same state under the + * lock before touching the device. + * + * gsp_phys_backend_work_operands() reports the ladder's module-owned CE + * operand VAs (src/dst/semaphore) and the fixed shadow copy size, or false + * while the ladder window is not usable. + * + * gsp_phys_backend_ce_copy_submit() submits one bytes-long CE copy on the + * ladder channel: it clears the module semaphore, encodes the method block, + * publishes the GPFIFO entry and USERD GP_PUT and rings the PF doorbell under + * gsp_phys_rm_lock, then releases the lock and polls the module semaphore for + * up to poll_ms (clamped to 200 ms) without the lock. Returns 0 when the + * release is observed, -ETIMEDOUT on the poll bound, -EAGAIN when the backend + * is not ready (including a release that committed before the lock), -EIO for + * a structural guard failure (BAR0/token/GP_PUT bounds/verify) and + * -EOPNOTSUPP for the server-allocated-memory channel variant. No guest + * memory or guest VA is ever touched; counters are maintained for all paths. + * + * gsp_phys_backend_work_counters() reads the shadow counters (attempts, + * observed releases, timeouts, refusals) for logging or the state file. + */ +#define GSP_PHYS_BACKEND_WORK_POLL_MS 50u /* submit clamps to 200 ms */ + +bool gsp_phys_backend_work_ready(void); +bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, + u32 *bytes); +int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 poll_ms); +void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, + u64 *timed_out, u64 *refused); /* * Command-queue TX: build a msgq record with the 570.144 element/rpc header diff --git a/kernel/gsp_phys_fwsec.c b/kernel/gsp_phys_fwsec.c index 83a68ed..c20758b 100644 --- a/kernel/gsp_phys_fwsec.c +++ b/kernel/gsp_phys_fwsec.c @@ -875,6 +875,28 @@ static int gsp_phys_fwsec_exec_with_loader(void __iomem *bar0, GSP_PHYS_SEC2_BL_CODE_SIZE, bl_dst, block_count, GSP_PHYS_SEC2_BL_START_TAG); + /* + * Read both staged regions back before the kick. A silent DMA failure + * leaves the falcon reading zeros, burns the full halt timeout and the + * boot only fails much later with no useful evidence; a mismatch here + * means the falcon is not accepting DMA, so fail open before STARTCPU. + */ + { + u32 dmem_rd[4], imem_rd[4]; + + gsp_phys_falcon_dmem_read(bar0, GSP_PHYS_GSP_BASE, 0, + dmem_rd, 4); + gsp_phys_falcon_imem_read(bar0, GSP_PHYS_GSP_BASE, bl_dst, + imem_rd, 4); + if (memcmp(dmem_rd, &bl, sizeof(dmem_rd)) || + memcmp(imem_rd, gsp_phys_sec2_bl_image, sizeof(imem_rd))) { + pr_err("gsp-phys-fwsec: staging not visible before kick (DMEM0 %08x/%08x/%08x/%08x IMEM %08x/%08x/%08x/%08x); the falcon is not accepting DMA - aborting before STARTCPU\n", + dmem_rd[0], dmem_rd[1], dmem_rd[2], dmem_rd[3], + imem_rd[0], imem_rd[1], imem_rd[2], imem_rd[3]); + return -EIO; + } + } + writel(GSP_PHYS_SEC2_BL_START_TAG << 8, bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC); cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 1000eaf..4d057d0 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -65,6 +65,7 @@ #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" #include "gsp_phys_fwsec.h" +#include "gsp_sema_service.h" #define GSP_PHYS_VENDOR 0x10de #define GSP_PHYS_DEVICE 0x1ed3 @@ -106,13 +107,83 @@ MODULE_PARM_DESC(gsp_phys_hazard_ack, * blocks) and the last mdev remove stops and releases it after the emulation * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every * failure is fail-open - the backend stays OFF and the mdev keeps working on - * the existing emulation path. + * the existing emulation path. READY is set only once the live GSP-RM session + * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is + * released fail-open. */ static bool gsp_phys_mdev_owned_param; module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_mdev_owned, "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)"); +/* + * P2 static-config identity overlay (default 0). When set, the guest's + * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity + * from the live backend snapshot (bounded 2 s wait, then fail-open to the + * emulated reply). FB geometry, masks, engine caps, PDE bases and handles + * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0. + */ +static bool gsp_phys_static_live_param; +module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_static_live, + "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)"); + +bool gsp_phys_static_live_enabled(void) +{ + return gsp_phys_static_live_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled); + +/* + * P3 read-only control forwarding (default 0). When set, a whitelist of + * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO, + * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to + * the live backend with the snapshot's adopted internal handles; every miss + * (no READY session, no valid snapshot, unmappable handle, backend + * error/timeout, malformed reply) falls back to the emulated handler. One-line + * rollback: gsp_phys_ctrl_forward=0. + */ +static bool gsp_phys_ctrl_forward_param; +module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_ctrl_forward, + "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)"); + +bool gsp_phys_ctrl_forward_enabled(void) +{ + return gsp_phys_ctrl_forward_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled); + +/* + * P5 wave-1 shadow work forwarding (default 0). When set, the sema service + * requests one real CE copy on the debugfs ladder's channel for each decoded + * guest CE push, using only the ladder's module-owned buffers; the software + * release path stays authoritative and unchanged. Refused/dropped/timeout + * submissions never touch guest memory and never block the pump. One-line + * rollback: gsp_phys_work_forward=0. + */ +static bool gsp_phys_work_forward_param; +module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_work_forward, + "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)"); + +bool gsp_phys_work_forward_enabled(void) +{ + return gsp_phys_work_forward_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); + +/* + * One-way power latch (default). Re-enabling runtime PM and D3cold after a + * claim lets the board power-gate and the host hard-freeze on resume + * (observed 2026-10-01); keep the pre-claim D0 policy instead and require + * gsp_phys_repower=1 with a reviewed recovery plan to restore it. + */ +static bool gsp_phys_repower; +module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644); +MODULE_PARM_DESC(gsp_phys_repower, + "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)"); + /* BAR0 registers sampled after claim as proof of live access. */ #define GSP_PHYS_BOOT_0 0x00000000 #define GSP_PHYS_BOOT_1 0x00000004 @@ -135,6 +206,12 @@ MODULE_PARM_DESC(gsp_phys_mdev_owned, /* Settle time after the distribution driver probes before releasing D0. */ #define GSP_PHYS_POWER_SETTLE_MS 1500 +/* Bounded post-rebind readiness gate before the D0 locks are released. */ +#define GSP_PHYS_REBIND_GATE_TRIES 200 /* 200 x 50 ms = 10 s */ +#define GSP_PHYS_REBIND_GATE_POLL_MS 50 +/* Bounded D0 settle check before the unbind. */ +#define GSP_PHYS_D0_SETTLE_TRIES 100 /* 100 x 20 ms = 2 s */ +#define GSP_PHYS_D0_SETTLE_POLL_MS 20 /* GPU + up to 7 sibling functions + upstream root port. */ #define GSP_PHYS_MAX_POWER_LOCKS 16 @@ -192,6 +269,16 @@ void __iomem *gsp_phys_gpu_bar0(void) EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0); /* + * CRC32 of the staged GSP firmware image (0 when no image is loaded). The + * backend static-config snapshot caches it as its firmware key. + */ +u32 gsp_phys_gpu_fw_crc32(void) +{ + return phys.fw_crc32; +} +EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); + +/* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can @@ -571,13 +658,24 @@ static void gsp_phys_power_unlock(void) struct gsp_phys_power_lock *lock = &phys.locks[--phys.nr_locks]; - if (lock->was_runtime_auto) - pm_runtime_allow(&lock->pdev->dev); - if (lock->was_d3cold_enabled) - pci_d3cold_enable(lock->pdev); - - pr_info("gsp-phys: power domain released: %s\n", - pci_name(lock->pdev)); + /* + * Default is a one-way latch: re-enabling runtime PM and D3cold + * after a claim can power-gate the board and hard-freeze the + * host on resume. The entries are still dropped; only the + * device policy stays held until reboot. + */ + if (gsp_phys_repower) { + if (lock->was_runtime_auto) + pm_runtime_allow(&lock->pdev->dev); + if (lock->was_d3cold_enabled) + pci_d3cold_enable(lock->pdev); + + pr_info("gsp-phys: power domain released: %s\n", + pci_name(lock->pdev)); + } else { + pr_info("gsp-phys: power domain left LOCKED (gsp_phys_repower=0): %s\n", + pci_name(lock->pdev)); + } pci_dev_put(lock->pdev); lock->pdev = NULL; } @@ -600,6 +698,19 @@ static const char *gsp_phys_link_speed(u16 lnksta) } /* + * Quiet liveness read for the post-rebind gate: a valid vendor ID means the + * board still answers config reads. Unlike gsp_phys_board_check() this never + * WARNs (it is polled) and never touches the root-port link status. + */ +static bool gsp_phys_board_live(struct pci_dev *pdev) +{ + u32 id = 0; + + pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); + return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id); +} + +/* * Abort signal for the handoff. The board must answer config reads while * it is claimed and again just before it is handed back; an invalid ID * means the power domain was cut (or never came back) and neither the @@ -720,6 +831,36 @@ int gsp_phys_gpu_claim(void) return ret; } + /* + * The pm_runtime_forbid() in the lock should have resumed both + * devices synchronously, but confirm they really are in D0 before + * unbinding: unbinding a suspended GPU is the observed freeze path. + * Refuse (and recover the distribution driver) if D0 is not reached. + */ + { + struct pci_dev *root = pci_upstream_bridge(pdev); + int i; + + for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) { + if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && + (!root || + READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE)) + break; + msleep(GSP_PHYS_D0_SETTLE_POLL_MS); + } + if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE || + (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) { + pr_err("gsp-phys: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n", + pdev->dev.power.runtime_status, + root ? root->dev.power.runtime_status : -1); + ret = -EBUSY; + goto err_recover; + } + } + + /* Let the engine clocks settle after the D0 transition. */ + msleep(100); + if (pdev->dev.driver) { pr_info("gsp-phys: unbinding %s from %s\n", pdev->dev.driver->name, pci_name(pdev)); @@ -895,11 +1036,21 @@ void gsp_phys_gpu_release(void) struct pci_dev *pdev; struct device_driver *drv; int ret, wpr2_ret = 0; + int i; if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) return; /* + * P5 wave 1: a shadow CE copy may be polling the ladder's module + * semaphore without gsp_phys_rm_lock. Stop new requests and wait for + * the in-flight one (bounded by the submit's 200 ms poll cap) before + * any staging is freed, so no poll can touch freed memory. No-op when + * forwarding never queued anything. + */ + gsp_sema_service_work_quiesce(); + + /* * WPR2 teardown must run while this module still owns BAR0. A * successful stage 3c leaves WPR2 carved in FB; the distribution * driver refuses to boot against an unexpected WPR2, so it has to be @@ -1005,12 +1156,46 @@ void gsp_phys_gpu_release(void) pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); + + /* + * Post-rebind readiness gate: the distribution driver's resume + * path can fail silently; dropping the D0 locks into that state + * re-arms D3cold and power-gates the board (the 2026-10-01 hard + * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an + * answering config space before letting go. + */ + for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) { + if (pdev->dev.driver && + READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && + gsp_phys_board_live(pdev)) + break; + msleep(GSP_PHYS_REBIND_GATE_POLL_MS); + } + if (i == GSP_PHYS_REBIND_GATE_TRIES) { + pr_err("gsp-phys: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n", + pci_name(pdev), + pdev->dev.driver ? pdev->dev.driver->name : "none", + pdev->dev.power.runtime_status); + /* + * Intentional imbalance: keep the claim's runtime-PM + * reference held together with the D0 locks so a board + * that did not finish resuming cannot be power-gated + * again. It stays held until reboot. + */ + if (phys.runtime_ref_held) + pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (rebind gate timeout)\n", + pci_name(pdev)); + pci_dev_put(pdev); + return; + } + msleep(GSP_PHYS_POWER_SETTLE_MS); /* - * The distribution driver has rebound and settled: drop the - * claim's runtime-PM reference exactly once, before the D0 - * locks are released. + * The distribution driver has rebound, resumed and settled: + * drop the claim's runtime-PM reference exactly once, before + * the D0 locks are released (the unlock itself now latches + * when gsp_phys_repower=0). */ gsp_phys_runtime_ref_drop(pdev); pci_dev_put(pdev); @@ -1036,6 +1221,7 @@ static struct gsp_phys_backend_session { enum gsp_phys_backend_state state; u32 refcount; u32 generation; + u32 static_probe_generation; /* generation whose snapshot probe ran */ bool module_owned; bool releasing; } backend = { @@ -1062,6 +1248,51 @@ u32 gsp_phys_backend_generation(void) } /* + * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend + * generation, run on a work item because the probe takes gsp_phys_rm_lock and + * can wait seconds for the fn65 reply. The work item is coalesced by the + * workqueue and additionally guarded by static_probe_generation, so a second + * schedule for the same generation is a no-op. + */ +static void gsp_phys_backend_static_probe_work(struct work_struct *work); +static DECLARE_WORK(gsp_phys_static_probe_work_item, + gsp_phys_backend_static_probe_work); + +static void gsp_phys_backend_static_probe_work(struct work_struct *work) +{ + u32 generation; + int ret; + + mutex_lock(&gsp_phys_session_lock); + if (backend.state != GSP_PHYS_BACKEND_READY) { + mutex_unlock(&gsp_phys_session_lock); + return; + } + generation = backend.generation; + if (backend.static_probe_generation == generation) { + mutex_unlock(&gsp_phys_session_lock); + return; + } + backend.static_probe_generation = generation; + mutex_unlock(&gsp_phys_session_lock); + + ret = gsp_phys_backend_static_probe(); + if (ret) + pr_warn("gsp-phys: static-config probe failed: %d; the static overlay keeps the emulated model\n", + ret); +} + +/* Caller holds gsp_phys_session_lock. */ +static void gsp_phys_backend_static_probe_schedule(void) +{ + if (backend.state != GSP_PHYS_BACKEND_READY) + return; + if (backend.static_probe_generation == backend.generation) + return; + schedule_work(&gsp_phys_static_probe_work_item); +} + +/* * Claim the release of the session under the session lock. Returns true only * for the caller that must run the sleeping release sequence. A module-owned * claim is never released; a concurrent second stopper sees releasing=true @@ -1101,6 +1332,12 @@ static bool gsp_phys_backend_claim_release_locked(const char *who) static void gsp_phys_backend_release_run(void) { cancel_work_sync(&gsp_phys_backend_start_work_item); + /* + * A running static-config probe may be mid-RPC on the real GSP; wait + * for it to finish (bounded by the probe's own timeouts) before the + * release touches the device. + */ + cancel_work_sync(&gsp_phys_static_probe_work_item); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); @@ -1132,6 +1369,8 @@ int gsp_phys_backend_start(void) backend.state, backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); + /* A new/joined session re-enables P5 shadow requests. */ + gsp_sema_service_work_resume(); return 0; } @@ -1152,23 +1391,40 @@ int gsp_phys_backend_start(void) backend.module_owned = was_claimed; if (was_claimed) { + if (!gsp_phys_boot_rm_ready()) + pr_warn("gsp-phys: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n"); + /* + * A new READY session: bump the generation (so a cache left + * from an earlier session can never match) and fill the + * static-config snapshot once, asynchronously. + */ + backend.generation++; + gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); - pr_info("gsp-phys: backend start: joined the live module-owned claim (READY)\n"); + pr_info("gsp-phys: backend start: joined the live module-owned claim (READY, generation=%u)\n", + backend.generation); + gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); return 0; } + backend.generation++; + gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING); - pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1)\n"); + pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n", + backend.generation); schedule_work(&gsp_phys_backend_start_work_item); mutex_unlock(&gsp_phys_session_lock); + /* A new session re-enables P5 shadow requests (gated on READY). */ + gsp_sema_service_work_resume(); return 0; } /* * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle - * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev - * create and init_work never block on the 60 s init-done poll. A failure + * create and init_work never block on the 60 s init-done poll. Any failure, + * including a successful claim whose GSP-RM session never became usable, * leaves the backend OFF: the mdev keeps working on the existing emulation * path. */ @@ -1201,12 +1457,38 @@ static void gsp_phys_backend_start_work(struct work_struct *work) if (ret) { backend.refcount = 0; backend.module_owned = false; + backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); + gsp_phys_backend_static_invalidate(); pr_err("gsp-phys: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n", ret); return; } + /* + * A successful claim is not enough: only a live GSP-RM session may be + * exposed as READY (a failed stage-3d boot leaves rm_init_done false + * and must not claim otherwise). Fail open by releasing the claim the + * way the stopper would; the emulation path keeps serving the guest. + */ + if (!gsp_phys_boot_rm_ready()) { + u32 rm_init_done, rm_unload_submitted; + + gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted); + mutex_unlock(&gsp_phys_session_lock); + pr_err("gsp-phys: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n", + rm_init_done, rm_unload_submitted); + if (gsp_phys_gpu_claimed()) + gsp_phys_gpu_release(); + mutex_lock(&gsp_phys_session_lock); + backend.refcount = 0; + backend.module_owned = false; + backend.generation++; + WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); + mutex_unlock(&gsp_phys_session_lock); + gsp_phys_backend_static_invalidate(); + return; + } if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start: claim completed while STOPPING; release deferred to the stopper\n"); @@ -1215,6 +1497,12 @@ static void gsp_phys_backend_start_work(struct work_struct *work) WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("gsp-phys: backend start: READY (owner=mdev, refcount=%u)\n", backend.refcount); + /* + * Fill the static-config snapshot once for this generation before any + * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm + * lock itself and fails open. + */ + gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); } @@ -1253,6 +1541,14 @@ void gsp_phys_backend_stop(void) */ void gsp_phys_backend_begin_stop(void) { + /* + * Flush the static-config probe work item first (no session lock + * held): a running probe may be mid-RPC and must finish before the + * release touches the device. For a module-owned claim this is the + * only effect; the caller still releases the claim itself. + */ + cancel_work_sync(&gsp_phys_static_probe_work_item); + mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index c8beb1d..709b3a6 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -11,6 +11,7 @@ bool gsp_phys_gpu_claimed(void); int gsp_phys_gpu_claim(void); void gsp_phys_gpu_release(void); void __iomem *gsp_phys_gpu_bar0(void); +u32 gsp_phys_gpu_fw_crc32(void); /* * Read by the backend session to enforce the mdev-owned start gate; the @@ -61,4 +62,30 @@ u32 gsp_phys_backend_generation(void); */ bool gsp_phys_mdev_owned_enabled(void); +/* + * True when the gsp_phys_static_live module parameter is set (default 0: + * the guest GET_GSP_STATIC_INFO reply is the emulated model unchanged). When + * set, the handler overlays the live GPU name/short-name/VBIOS identity from + * the backend snapshot, failing open to the emulated reply on any miss. + */ +bool gsp_phys_static_live_enabled(void); + +/* + * True when the gsp_phys_ctrl_forward module parameter is set (default 0: + * guest GSP_RM_CONTROL replies are the emulated answers unchanged). When set, + * a whitelist of side-effect-free control queries is forwarded to the live + * backend with the snapshot's adopted internal handles, falling back to the + * emulated per-command handler on any miss. + */ +bool gsp_phys_ctrl_forward_enabled(void); + +/* + * True when the gsp_phys_work_forward module parameter is set (default 0: the + * guest's CE pushes stay software-emulated exactly as before). When set, the + * sema service shadows each decoded CE push as one copy on the debugfs ladder's + * real channel using only module-owned ladder buffers; the software release + * remains authoritative and is applied unchanged either way. + */ +bool gsp_phys_work_forward_enabled(void); + #endif /* GSP_PHYS_GPU_H */ diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index d6aef9e..b0ad98f 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -10,12 +10,15 @@ #include <linux/slab.h> #include <linux/vfio.h> #include <linux/delay.h> +#include <linux/string.h> #include "nvidia-vgpu-vfio.h" #include "gsp_emulation.h" #include "gsp_rm_emulation.h" #include "gsp_rm_tu104.h" #include "gsp_sema_service.h" +#include "gsp_phys_booter.h" +#include "gsp_phys_gpu.h" /* ============================================================ * Guest-memory access ("virtual DMA" on behalf of the device) @@ -1167,6 +1170,64 @@ static void gsp_rm_build_static_info(u8 *buf) memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8); } +/* + * P2 static-config identity overlay: replace the emulated identity fields + * (GPU name, short name, VBIOS valid/subvendor/subdevice) with the live + * decode from the backend snapshot. FB geometry, masks, engine caps, PDE + * bases and handles stay emulated in this phase. The wait is bounded and + * sleeps; any miss leaves the emulated reply intact (fail-open). + */ +#define GSPRM_STATIC_LIVE_WAIT_MS 2000 + +static void gsp_rm_apply_static_overlay(struct gsp_emu_context *ctx, u8 *buf) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + struct gsp_phys_backend_static_config snap; + u16 val16; + int ret; + + if (!gsp_phys_static_live_enabled()) + return; + + ret = gsp_phys_backend_static_wait(&snap, GSPRM_STATIC_LIVE_WAIT_MS); + if (ret) { + rm->static_fallback++; + if (ret == -ENODATA) + rm->static_timeout++; + return; + } + + /* Wire slots: name 64 B at 1260, short name 64 B at 1324. An empty + * live string never clears the emulated one. */ + if (snap.gpu_name[0]) + strscpy((char *)(buf + TU104_GSCI_OFF_gpuNameString), + snap.gpu_name, + TU104_GSCI_OFF_gpuShortNameString - + TU104_GSCI_OFF_gpuNameString); + if (snap.gpu_short_name[0]) + strscpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString), + snap.gpu_short_name, + TU104_GSCI_OFF_gpuNameString_Unicode - + TU104_GSCI_OFF_gpuShortNameString); + + /* Only overlay the VBIOS identity when the live decode carries a valid + * VBIOS; never clear an emulated valid flag. */ + if (snap.b_vbios_valid) { + buf[TU104_GSCI_OFF_bVbiosValid] = 1; + val16 = (u16)snap.vbios_sub_vendor; + memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val16, sizeof(val16)); + val16 = (u16)snap.vbios_sub_device; + memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val16, sizeof(val16)); + } + + rm->static_live++; + nv_vgpu_info(ctx->vgpu, + "GSPRM: static-info overlay applied (generation=%u): name='%s' short='%s' vbiosValid=%u subVendor=0x%04x subDevice=0x%04x", + snap.generation, snap.gpu_name, snap.gpu_short_name, + snap.b_vbios_valid, (u16)snap.vbios_sub_vendor, + (u16)snap.vbios_sub_device); +} + static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx, u32 sequence) { @@ -1180,6 +1241,7 @@ static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx, } gsp_rm_build_static_info(reply); + gsp_rm_apply_static_overlay(ctx, reply); nv_vgpu_info(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)", sequence, TU104_GSCI_SIZE, @@ -2608,6 +2670,154 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx, } } +/* + * Bounded logging for the P3 forward counters: the first four fallbacks (so a + * session that cannot forward is visible) and every 64th event. Never called + * when the parameter is off. + */ +static void gsp_rm_ctrl_forward_note(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u64 events = rm->ctrl_fwd_ok + rm->ctrl_fwd_fallback; + + if ((rm->ctrl_fwd_fallback && rm->ctrl_fwd_fallback <= 4) || + (events && (events % 64) == 0)) + nv_vgpu_info(ctx->vgpu, + "GSPRM: RM_CONTROL forward: ok=%llu fallback=%llu timeout=%llu", + rm->ctrl_fwd_ok, rm->ctrl_fwd_fallback, + rm->ctrl_fwd_timeout); +} + +/* + * P3 read-only control forwarding (gsp_phys_ctrl_forward=1; default off). + * + * A small whitelist of side-effect-free queries is sent to the live backend + * session with the guest's emulated internal handles translated to the + * snapshot's adopted server handles; the server's echoed params become the + * guest reply. Everything else - the vGPU-specific controls, anything + * mutating, unknown IDs, an unreadable/absent snapshot, unmappable handles, + * a multi-element request, a backend error/timeout or a malformed reply - + * returns false and the emulated per-command handler runs unchanged + * (fail-open). + * + * gsp_phys_backend_rpc() returns with gsp_phys_rm_lock released and the reply + * in a kmalloc'd copy; the status queue is only written afterwards by + * gsp_rm_send(), never while a backend wait is pending (doc 11 section 6.5). + */ +static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, + const struct gsp_rm_control_req *req, + const u8 *req_params, u8 *reply, + u32 reply_data_len, u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + struct gsp_phys_backend_static_config snap; + struct gsp_rm_control_req fwd_hdr; + u8 *fwd_req; + void *rsp = NULL; + u32 rsp_len = 0; + u32 out_status; + u32 out_params_size; + int ret; + + /* Wave 1 whitelist: compile-time side-effect-free queries only. */ + switch (req->cmd) { + case TU104_CTRL_BUS_GET_PCI_BAR_INFO: + case TU104_CTRL_BUS_GET_INFO_V2: + case TU104_CTRL_FB_GET_INFO_V2: + case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO: + break; + default: + return false; + } + + /* The capture hands us only the first queue element, so a request that + * does not fit one element (paramsSize+24 <= GSPRM_MAX_SINGLE_ELEM_DATA) + * cannot be reassembled and stays emulated. */ + if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE) + goto fallback; + + /* A valid snapshot implies a READY session and the adopted handles. */ + if (gsp_phys_backend_static_get(&snap)) + goto fallback; + + /* Translate the guest's emulated internal handles; any other handle + * domain stays emulated. */ + if (req->hClient != TU104_H_INTERNAL_CLIENT || + (req->hObject != TU104_H_INTERNAL_DEVICE && + req->hObject != TU104_H_INTERNAL_SUBDEVICE)) + goto fallback; + + fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL); + if (!fwd_req) + goto fallback; + memset(&fwd_hdr, 0, sizeof(fwd_hdr)); + fwd_hdr.hClient = snap.h_internal_client; + fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ? + snap.h_internal_device : snap.h_internal_subdevice; + fwd_hdr.cmd = req->cmd; + fwd_hdr.paramsSize = req->paramsSize; + fwd_hdr.flags = req->flags; + memcpy(fwd_req, &fwd_hdr, GSP_RM_CTRL_HDR_SIZE); + if (req->paramsSize) + memcpy(fwd_req + GSP_RM_CTRL_HDR_SIZE, req_params, + req->paramsSize); + + ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_CONTROL, fwd_req, + GSP_RM_CTRL_HDR_SIZE + req->paramsSize, + &rsp, &rsp_len, 500); + kfree(fwd_req); + if (ret) { + if (ret == -ETIMEDOUT) + rm->ctrl_fwd_timeout++; + goto fallback; + } + + /* + * The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize + * at +16, echoed params at +24 - the same 24-byte prefix the emulated + * handler builds below. The RPC-level rpc_result was already checked + * by gsp_phys_backend_rpc(). + */ + if (!rsp || rsp_len < GSP_RM_CTRL_HDR_SIZE) + goto fallback; + memcpy(&out_status, (u8 *)rsp + 12, 4); + memcpy(&out_params_size, (u8 *)rsp + 16, 4); + if (out_status != 0 || + out_params_size > rsp_len - GSP_RM_CTRL_HDR_SIZE || + out_params_size > reply_data_len - GSP_RM_CTRL_HDR_SIZE) + goto fallback; + + /* Build the guest reply exactly as the emulated handler does: the + * guest's own handles/cmd and the server's status/params. */ + memcpy(reply + 0, &req->hClient, 4); + memcpy(reply + 4, &req->hObject, 4); + memcpy(reply + 8, &req->cmd, 4); + memcpy(reply + 12, &out_status, 4); + memcpy(reply + 16, &out_params_size, 4); + memcpy(reply + 20, &req->flags, 4); + if (out_params_size) + memcpy(reply + GSP_RM_CTRL_HDR_SIZE, + (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE, out_params_size); + kfree(rsp); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, + reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) { + nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL forward reply send failed cmd=0x%x", + req->cmd); + rm->ctrl_fwd_fallback++; + } else { + rm->ctrl_fwd_ok++; + } + gsp_rm_ctrl_forward_note(ctx); + return true; + +fallback: + kfree(rsp); + rm->ctrl_fwd_fallback++; + gsp_rm_ctrl_forward_note(ctx); + return false; +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -2636,6 +2846,19 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, return; } + /* + * P3: when enabled, forward a whitelist of read-only queries to the + * live backend. The helper builds and sends the reply itself; a false + * return means nothing was sent and the emulated switch below runs + * byte-identically to the parameter-off path. + */ + if (gsp_phys_ctrl_forward_enabled() && + gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE, + reply, reply_data_len, sequence)) { + kfree(reply); + return; + } + switch (req.cmd) { case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: gsp_rm_handle_fifo_device_table(ctx, sequence, &req, diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index 2938c92..e464809 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -176,6 +176,12 @@ struct gsp_rm_emu { u32 alloc_dumps; /* raw GSP_RM_ALLOC dumps emitted */ u32 alloc_param_dumps; /* alloc param dumps emitted */ u32 ctrl_param_dumps; /* control param dumps emitted */ + u64 static_live; /* static-info replies carrying live identity */ + u64 static_fallback; /* overlay requested, no snapshot served */ + u64 static_timeout; /* fallbacks caused by the bounded wait */ + u64 ctrl_fwd_ok; /* whitelisted controls answered live */ + u64 ctrl_fwd_fallback; /* forward attempt used emulation */ + u64 ctrl_fwd_timeout; /* fallbacks caused by a backend timeout */ /* * NV01_EVENT objects (class 0x5, allocated via class 0x79 in the diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e97c2ff..25a3cdb 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -41,10 +41,13 @@ #include <linux/mutex.h> #include <linux/sort.h> #include <linux/bitmap.h> +#include <linux/workqueue.h> #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" +#include "gsp_phys_booter.h" +#include "gsp_phys_gpu.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 @@ -5016,6 +5019,145 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) test_bit(idx, ch->slot_done_epoch); } +/* ------------------------------------------------------------------ */ +/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1) */ +/* ------------------------------------------------------------------ */ + +/* + * The pump never waits on the backend: for an eligible CE push it only marks + * one shadow request in flight and queues a dedicated ordered work item. A + * request that arrives while one is in flight (or before the backend/ladder + * is ready) is dropped and counted. The work item runs the ladder's + * module-owned copy and observes the real semaphore; the software release + * path is unchanged and stays authoritative. No guest VA or guest PA ever + * reaches the engine in this wave. + */ +#define SVC_FWD_INFLIGHT 0 +#define SVC_FWD_LOG_MASK 63 + +static struct workqueue_struct *svc_fwd_wq; +static unsigned long svc_fwd_state; +static u64 svc_fwd_requests; +static u64 svc_fwd_dropped; +/* + * Pause gate shared by the pump's request path and the backend release path: + * quiesce takes this lock, sets paused and waits for the in-flight work item; + * a request either queues before quiesce (and is waited for) or observes + * paused and drops. Without it a request could be queued after quiesce + * returned and run while the ladder buffers are being freed. + */ +static DEFINE_MUTEX(svc_fwd_lock); +static bool svc_fwd_paused; + +static void svc_fwd_work_fn(struct work_struct *work) +{ + u64 src_va = 0, dst_va = 0, sem_va = 0; + u32 bytes = 0; + unsigned long start = jiffies; + int ret; + + /* + * Re-resolve the ladder operands on the work item: the backend may + * have gone away between the pump's gate and this run. A false return + * means no module buffer is usable; drop without touching anything. + */ + if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va, + &bytes)) { + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n", + svc_fwd_dropped); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + return; + } + + ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + if (!ret) + pr_info("gsp-phys-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n", + bytes, jiffies_to_msecs(jiffies - start), + svc_fwd_requests, svc_fwd_dropped); + /* Timeout/refusal counters and their rate-limited logs are in the + * submit; the software release has already run either way. */ + + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); +} + +static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); + +/* Called only from the service thread (the pump). */ +static void svc_fwd_request(const struct svc_channel *ch) +{ + if (!ch->engine_type) + return; + if (!gsp_phys_work_forward_enabled()) + return; + if (!gsp_phys_backend_work_ready()) { + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (backend not ready) dropped=%llu\n", + svc_fwd_dropped); + return; + } + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused) { + mutex_unlock(&svc_fwd_lock); + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (forwarding paused for release) dropped=%llu\n", + svc_fwd_dropped); + return; + } + if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (one in flight) dropped=%llu\n", + svc_fwd_dropped); + return; + } + svc_fwd_requests++; + if (!svc_fwd_wq) { + /* + * Lazy, on the service thread only: no workqueue exists at all + * while the parameter is off. + */ + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + return; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + } + mutex_unlock(&svc_fwd_lock); +} + +/* + * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new + * requests under svc_fwd_lock and waits for the in-flight one; the work item + * never takes that lock, so the wait cannot deadlock and is bounded by the + * submit's 200 ms poll cap. + */ +void gsp_sema_service_work_quiesce(void) +{ + mutex_lock(&svc_fwd_lock); + svc_fwd_paused = true; + cancel_work_sync(&svc_fwd_work); + mutex_unlock(&svc_fwd_lock); +} + +void gsp_sema_service_work_resume(void) +{ + mutex_lock(&svc_fwd_lock); + svc_fwd_paused = false; + mutex_unlock(&svc_fwd_lock); +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -5262,6 +5404,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); /* + * P5 wave 1: shadow this + * CE push on the ladder + * channel. The software + * release above is already + * applied and stays + * authoritative; the work + * item only observes the + * real semaphore. + */ + svc_fwd_request(ch); + /* * A CE push that should carry this * channel's next tracking release but * decoded none read from a @@ -6397,6 +6550,11 @@ int gsp_sema_service_start(void) init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); spin_lock_init(&svc.vas_lock); + /* + * The P5 shadow workqueue is created lazily by svc_fwd_request() only + * while gsp_phys_work_forward is enabled; the software path needs no + * workqueue at all. + */ svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); @@ -6411,14 +6569,22 @@ int gsp_sema_service_start(void) * Stop the service thread without kthread_stop(): the thread can also end on * its own if it ever faults, and joining a task that is already gone faults in * the caller (which wedges the whole module). The completion tells us the - * thread function has returned, whatever ended it. + * thread function has returned, whatever ended it. The shadow work item is + * cancelled after the thread stops (so no new request can be queued) and + * before the workqueue is destroyed; the submit's own poll is bounded to + * 200 ms, so this wait is bounded. */ void gsp_sema_service_stop(void) { - if (!svc.task) - return; - svc.stop = true; - if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) - pr_err("sema-svc: thread did not stop within 5s\n"); - svc.task = NULL; + if (svc.task) { + svc.stop = true; + if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) + pr_err("sema-svc: thread did not stop within 5s\n"); + svc.task = NULL; + } + cancel_work_sync(&svc_fwd_work); + if (svc_fwd_wq) { + destroy_workqueue(svc_fwd_wq); + svc_fwd_wq = NULL; + } } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 7778443..ae9b7da 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -66,6 +66,17 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa); void gsp_sema_service_doorbell(u32 token); /* + * P5 wave-1 shadow work lifetime hooks. The backend release path must call + * gsp_sema_service_work_quiesce() before it frees the ladder buffers: it + * pauses new shadow requests and waits (bounded by the submit's 200 ms poll + * cap) for the in-flight one to finish, so no poll ever touches freed memory. + * gsp_sema_service_work_resume() re-enables requests when a new backend + * session starts. + */ +void gsp_sema_service_work_quiesce(void); +void gsp_sema_service_work_resume(void); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). diff --git a/kernel/main.c b/kernel/main.c index f47ef69..3784a70 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -1452,9 +1452,23 @@ static int __init nv_vgpu_init(void) if (gsp_phys_gpu_param && !gsp_phys_mdev_owned_enabled()) { ret = gsp_phys_gpu_claim(); - if (ret) + if (ret) { pr_warn("gsp-resman-vfio: physical GPU claim failed: %d\n", ret); + } else { + /* + * G0.4 (P2 prep): expose the load-time module-owned + * claim as a READY backend session. The join path only + * marks the state and schedules the static-config + * snapshot probe (a work item); failure is non-fatal, + * module load keeps the emulation path. The claim + * itself is still released at rmmod. + */ + ret = gsp_phys_backend_start(); + if (ret) + pr_warn("gsp-resman-vfio: physical backend start failed: %d\n", + ret); + } } else if (gsp_phys_gpu_param) { pr_info("gsp-resman-vfio: gsp_phys_mdev_owned=1: physical GPU claim+boot deferred to the first mdev create\n"); } else if (gsp_phys_mdev_owned_enabled()) { @@ -1518,10 +1532,12 @@ static void __exit nv_vgpu_exit(void) gsp_phys_backend_begin_stop(); } else { /* - * Module-owned claim (gsp_phys_gpu=1, mdev_owned=0): keep - * today's position relative to the debugfs teardown and the - * emulation threads; module-owned sessions never forward live. + * Module-owned claim (gsp_phys_gpu=1, mdev_owned=0): flush the + * static-config probe work item (begin_stop's only effect for a + * module-owned session), then keep today's release position + * relative to the debugfs teardown and the emulation threads. */ + gsp_phys_backend_begin_stop(); gsp_phys_gpu_release(); } diff --git a/tools/guest-diag/phys_claim_cycle.sh b/tools/guest-diag/phys_claim_cycle.sh index 40577f9..0e00ae8 100755 --- a/tools/guest-diag/phys_claim_cycle.sh +++ b/tools/guest-diag/phys_claim_cycle.sh @@ -7,7 +7,8 @@ # start persistenced -> wait GPU+root port active -> power/control=on on # GPU+port -> dmesg -C -> (stop persistenced && insmod) as ONE command -> # verify lock/claim patterns -> hold -> verify while claimed -> -# rmmod -> verify release patterns -> restore auto + persistenced -> +# rmmod -> verify release patterns (power domain left LOCKED by default; +# with --restore-power: power/control=auto + persistenced restored) -> # health check. # # !!! HOST-FREEZE HAZARD !!! @@ -22,7 +23,7 @@ # # Usage: phys_claim_cycle.sh --yes [--hold SECONDS] [--cycles N] [--layout] # [--dryrun] [--boot-dryrun] [--boot] [--keep] -# [--ko PATH] [--fw NAME] [--no-restore] +# [--ko PATH] [--fw NAME] [--restore-power] # # --yes required: acknowledge the host-freeze hazard # --hold S seconds to keep the GPU claimed (default 30) @@ -68,8 +69,13 @@ # the "gsp-nocat:" prefix. The unique-record summary is # printed at the end of stage 3e (success or failure) with or # without this flag; per-record logging is verbose. -# --no-restore do not restore power/control=auto or restart persistenced -# at the end of each cycle (leaves the D0 hold in place) +# --restore-power restore power/control=auto and restart persistenced at +# the end of each cycle; expects 5 "power domain released" +# lines and d3cold_allowed=1 on the GPU. Default off: the +# module latches the D0 hold, power/control stays "on", +# persistenced stays stopped; expects 5 "power domain left +# LOCKED" lines and d3cold_allowed=0 on the GPU. +# --no-restore accepted as a no-op: the safe locked default is now built in # # --boot is mutually exclusive with --dryrun/--boot-dryrun. # @@ -99,7 +105,7 @@ BOOT=0 BOOT_DRYRUN=0 NOCAT=0 KEEP=0 -NO_RESTORE=0 +RESTORE_POWER=0 KO="$REPO_ROOT/kernel/open-iov-vfio.ko" FW="" @@ -137,7 +143,8 @@ EOF --ko) KO="${2:-}"; shift ;; --fw) FW="${2:-}"; shift ;; --nocat) NOCAT=1 ;; - --no-restore) NO_RESTORE=1 ;; + --restore-power) RESTORE_POWER=1 ;; + --no-restore) : ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;; esac @@ -531,10 +538,16 @@ EOF OK_TEARDOWN=0 grep -q "teardown complete: WPR2 down" "$DLOG" && OK_TEARDOWN=1 fi - NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)" + if [ "$RESTORE_POWER" -eq 1 ]; then + NR_RELEASED="$(grep -c "power domain released" "$DLOG" 2>/dev/null || true)" + RELEASE_KIND="released" + else + NR_RELEASED="$(grep -c "power domain left LOCKED" "$DLOG" 2>/dev/null || true)" + RELEASE_KIND="locked" + fi if [ "$OK_REBIND" -ne 1 ] || [ "$NR_RELEASED" -ne 5 ]; then ALL_GREEN=0 - warn "cycle $n: release verification failed (rebind=$OK_REBIND released=$NR_RELEASED); locks may still be held - do not retry" + warn "cycle $n: release verification failed (rebind=$OK_REBIND $RELEASE_KIND=$NR_RELEASED); locks may still be held - do not retry" save_dmesg "$DLOG" break fi @@ -551,23 +564,34 @@ EOF ALL_GREEN=0 fi - if [ "$NO_RESTORE" -eq 0 ]; then + if [ "$RESTORE_POWER" -eq 1 ]; then set_power_control "$GPU_BDF" auto || warn "cycle $n: failed to restore GPU power/control=auto" set_power_control "$PORT_BDF" auto || warn "cycle $n: failed to restore port power/control=auto" sudo systemctl start nvidia-persistenced 2>/dev/null || true wait_active || warn "cycle $n: GPU/port not active after persistenced restart" else - log "cycle $n: --no-restore: leaving power/control=on and persistenced stopped" + log "cycle $n: power domain left LOCKED (default): power/control=on retained, persistenced stopped" fi HEALTH_RED=0 nvidia-smi -L >/dev/null 2>&1 || HEALTH_RED=1 [ "$(pci_field "$GPU_BDF")" = "nvidia" ] || HEALTH_RED=1 - if [ "$NO_RESTORE" -eq 0 ]; then + if [ "$RESTORE_POWER" -eq 1 ]; then [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = auto ] || HEALTH_RED=1 + [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1 + else + # Latched mode: the module keeps dev->no_d3cold set, which is not + # visible in sysfs (d3cold_allowed is the user policy and stays 1), + # and the root port stays pinned in D0 even if the udev nvidia bind + # rule rewrites the GPU's power/control to auto (60-nvidia.rules). + # The latch is verified by the "power domain left LOCKED" count and + # the absence of an in-window rail cycle, so do not require + # control=on/d3cold_allowed=0 here. + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/control" 2>/dev/null)" = on ] || HEALTH_RED=1 + [ "$(cat "/sys/bus/pci/devices/$PORT_BDF/power/runtime_status" 2>/dev/null)" = active ] || HEALTH_RED=1 + log "cycle $n: power policy (diagnostic): GPU control=$(cat "/sys/bus/pci/devices/$GPU_BDF/power/control" 2>/dev/null) d3cold_allowed=$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null) (dev->no_d3cold is not visible in sysfs)" fi - [ "$(cat "/sys/bus/pci/devices/$GPU_BDF/d3cold_allowed" 2>/dev/null)" = 1 ] || HEALTH_RED=1 IN_RAIL="$(in_window_rail_count "$DLOG")" OUT_RAIL="$(( $(rail_count "$DLOG") - IN_RAIL ))" @@ -603,9 +627,9 @@ EOF fi if [ "$ALL_GREEN" -eq 1 ] && [ "$IN_RAIL" -eq 0 ] && [ "$HEALTH_RED" -eq 0 ]; then - log "cycle $n: GREEN (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)" + log "cycle $n: GREEN (locks=$NR_LOCKS, $RELEASE_KIND=$NR_RELEASED, in-window rail=0, outside rail=$OUT_RAIL, hold=${HOLD}s)" else - log "cycle $n: RED (locks=$NR_LOCKS, released=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))" + log "cycle $n: RED (locks=$NR_LOCKS, $RELEASE_KIND=$NR_RELEASED, in-window rail=$IN_RAIL, outside rail=$OUT_RAIL, health_ok=$((1-HEALTH_RED)))" if [ "$IN_RAIL" -gt 0 ]; then echo "ABORT: cycle $n: rail cycle inside the locked window; not retrying" >&2 save_dmesg "$DLOG" |