From c886760efda836b060fee03bfce852f16f04b5b8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 02:07:52 -0400 Subject: gsp-phys: freeze-safe release, boot-proof READY, and guest-wiring P2/P3/P5-wave1 Host-freeze fix bundle (validated 2/2 supervised cycles): - power unlock now latches D0/D3cold by default (new gsp_phys_repower, default 0); releasing the claim can no longer power-gate the board and hard-freeze the host on resume - bounded post-rebind readiness gate before dropping the locks; bounded D0 settle/verify before the unbind - READY means a live GSP-RM session: the async worker and the module-owned join only expose READY when init-done is seen and no unload was sent; otherwise the claim is released fail-open - FWSEC reads DMEM0/IMEM back before STARTCPU and fails fast when the staging is not visible - the claim harness leaves the board latched by default and takes --restore-power to opt back into the old restore Guest-wiring: - P2: generation-keyed fn65 static-config snapshot cache plus a default-off identity overlay (new gsp_phys_static_live) for name/short-name/VBIOS; the probe cycle runs once per backend generation at READY - P3: default-off read-only RM_CONTROL forwarding whitelist (new gsp_phys_ctrl_forward) with emulated-handle translation to the adopted server handles and full emulation fallback - P5 wave 1: default-off shadow execution of guest CE pushes on the ladder channel (new gsp_phys_work_forward) via a dedicated ordered work item; the ladder stage-4 submission is refactored into a shared bounded helper and the software release path stays authoritative Live: module-owned boot to init-done, guest smoke ALL OK 64/64 with P2/P3 enabled, 514 shadow copies observed on the physical CE; all cycles released cleanly with WPR2 down and no reboot. --- kernel/gsp_phys_gpu.c | 324 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 310 insertions(+), 14 deletions(-) (limited to 'kernel/gsp_phys_gpu.c') diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 1000eaf..4d057d0 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -65,6 +65,7 @@ #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" #include "gsp_phys_fwsec.h" +#include "gsp_sema_service.h" #define GSP_PHYS_VENDOR 0x10de #define GSP_PHYS_DEVICE 0x1ed3 @@ -106,13 +107,83 @@ MODULE_PARM_DESC(gsp_phys_hazard_ack, * blocks) and the last mdev remove stops and releases it after the emulation * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every * failure is fail-open - the backend stays OFF and the mdev keeps working on - * the existing emulation path. + * the existing emulation path. READY is set only once the live GSP-RM session + * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is + * released fail-open. */ static bool gsp_phys_mdev_owned_param; module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_mdev_owned, "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)"); +/* + * P2 static-config identity overlay (default 0). When set, the guest's + * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity + * from the live backend snapshot (bounded 2 s wait, then fail-open to the + * emulated reply). FB geometry, masks, engine caps, PDE bases and handles + * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0. + */ +static bool gsp_phys_static_live_param; +module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_static_live, + "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)"); + +bool gsp_phys_static_live_enabled(void) +{ + return gsp_phys_static_live_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled); + +/* + * P3 read-only control forwarding (default 0). When set, a whitelist of + * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO, + * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to + * the live backend with the snapshot's adopted internal handles; every miss + * (no READY session, no valid snapshot, unmappable handle, backend + * error/timeout, malformed reply) falls back to the emulated handler. One-line + * rollback: gsp_phys_ctrl_forward=0. + */ +static bool gsp_phys_ctrl_forward_param; +module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_ctrl_forward, + "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)"); + +bool gsp_phys_ctrl_forward_enabled(void) +{ + return gsp_phys_ctrl_forward_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled); + +/* + * P5 wave-1 shadow work forwarding (default 0). When set, the sema service + * requests one real CE copy on the debugfs ladder's channel for each decoded + * guest CE push, using only the ladder's module-owned buffers; the software + * release path stays authoritative and unchanged. Refused/dropped/timeout + * submissions never touch guest memory and never block the pump. One-line + * rollback: gsp_phys_work_forward=0. + */ +static bool gsp_phys_work_forward_param; +module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_work_forward, + "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)"); + +bool gsp_phys_work_forward_enabled(void) +{ + return gsp_phys_work_forward_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); + +/* + * One-way power latch (default). Re-enabling runtime PM and D3cold after a + * claim lets the board power-gate and the host hard-freeze on resume + * (observed 2026-10-01); keep the pre-claim D0 policy instead and require + * gsp_phys_repower=1 with a reviewed recovery plan to restore it. + */ +static bool gsp_phys_repower; +module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644); +MODULE_PARM_DESC(gsp_phys_repower, + "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)"); + /* BAR0 registers sampled after claim as proof of live access. */ #define GSP_PHYS_BOOT_0 0x00000000 #define GSP_PHYS_BOOT_1 0x00000004 @@ -135,6 +206,12 @@ MODULE_PARM_DESC(gsp_phys_mdev_owned, /* Settle time after the distribution driver probes before releasing D0. */ #define GSP_PHYS_POWER_SETTLE_MS 1500 +/* Bounded post-rebind readiness gate before the D0 locks are released. */ +#define GSP_PHYS_REBIND_GATE_TRIES 200 /* 200 x 50 ms = 10 s */ +#define GSP_PHYS_REBIND_GATE_POLL_MS 50 +/* Bounded D0 settle check before the unbind. */ +#define GSP_PHYS_D0_SETTLE_TRIES 100 /* 100 x 20 ms = 2 s */ +#define GSP_PHYS_D0_SETTLE_POLL_MS 20 /* GPU + up to 7 sibling functions + upstream root port. */ #define GSP_PHYS_MAX_POWER_LOCKS 16 @@ -191,6 +268,16 @@ void __iomem *gsp_phys_gpu_bar0(void) } EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0); +/* + * CRC32 of the staged GSP firmware image (0 when no image is loaded). The + * backend static-config snapshot caches it as its firmware key. + */ +u32 gsp_phys_gpu_fw_crc32(void) +{ + return phys.fw_crc32; +} +EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); + /* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind @@ -571,13 +658,24 @@ static void gsp_phys_power_unlock(void) struct gsp_phys_power_lock *lock = &phys.locks[--phys.nr_locks]; - if (lock->was_runtime_auto) - pm_runtime_allow(&lock->pdev->dev); - if (lock->was_d3cold_enabled) - pci_d3cold_enable(lock->pdev); - - pr_info("gsp-phys: power domain released: %s\n", - pci_name(lock->pdev)); + /* + * Default is a one-way latch: re-enabling runtime PM and D3cold + * after a claim can power-gate the board and hard-freeze the + * host on resume. The entries are still dropped; only the + * device policy stays held until reboot. + */ + if (gsp_phys_repower) { + if (lock->was_runtime_auto) + pm_runtime_allow(&lock->pdev->dev); + if (lock->was_d3cold_enabled) + pci_d3cold_enable(lock->pdev); + + pr_info("gsp-phys: power domain released: %s\n", + pci_name(lock->pdev)); + } else { + pr_info("gsp-phys: power domain left LOCKED (gsp_phys_repower=0): %s\n", + pci_name(lock->pdev)); + } pci_dev_put(lock->pdev); lock->pdev = NULL; } @@ -599,6 +697,19 @@ static const char *gsp_phys_link_speed(u16 lnksta) } } +/* + * Quiet liveness read for the post-rebind gate: a valid vendor ID means the + * board still answers config reads. Unlike gsp_phys_board_check() this never + * WARNs (it is polled) and never touches the root-port link status. + */ +static bool gsp_phys_board_live(struct pci_dev *pdev) +{ + u32 id = 0; + + pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); + return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id); +} + /* * Abort signal for the handoff. The board must answer config reads while * it is claimed and again just before it is handed back; an invalid ID @@ -720,6 +831,36 @@ int gsp_phys_gpu_claim(void) return ret; } + /* + * The pm_runtime_forbid() in the lock should have resumed both + * devices synchronously, but confirm they really are in D0 before + * unbinding: unbinding a suspended GPU is the observed freeze path. + * Refuse (and recover the distribution driver) if D0 is not reached. + */ + { + struct pci_dev *root = pci_upstream_bridge(pdev); + int i; + + for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) { + if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && + (!root || + READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE)) + break; + msleep(GSP_PHYS_D0_SETTLE_POLL_MS); + } + if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE || + (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) { + pr_err("gsp-phys: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n", + pdev->dev.power.runtime_status, + root ? root->dev.power.runtime_status : -1); + ret = -EBUSY; + goto err_recover; + } + } + + /* Let the engine clocks settle after the D0 transition. */ + msleep(100); + if (pdev->dev.driver) { pr_info("gsp-phys: unbinding %s from %s\n", pdev->dev.driver->name, pci_name(pdev)); @@ -895,10 +1036,20 @@ void gsp_phys_gpu_release(void) struct pci_dev *pdev; struct device_driver *drv; int ret, wpr2_ret = 0; + int i; if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) return; + /* + * P5 wave 1: a shadow CE copy may be polling the ladder's module + * semaphore without gsp_phys_rm_lock. Stop new requests and wait for + * the in-flight one (bounded by the submit's 200 ms poll cap) before + * any staging is freed, so no poll can touch freed memory. No-op when + * forwarding never queued anything. + */ + gsp_sema_service_work_quiesce(); + /* * WPR2 teardown must run while this module still owns BAR0. A * successful stage 3c leaves WPR2 carved in FB; the distribution @@ -1005,12 +1156,46 @@ void gsp_phys_gpu_release(void) pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); + + /* + * Post-rebind readiness gate: the distribution driver's resume + * path can fail silently; dropping the D0 locks into that state + * re-arms D3cold and power-gates the board (the 2026-10-01 hard + * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an + * answering config space before letting go. + */ + for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) { + if (pdev->dev.driver && + READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && + gsp_phys_board_live(pdev)) + break; + msleep(GSP_PHYS_REBIND_GATE_POLL_MS); + } + if (i == GSP_PHYS_REBIND_GATE_TRIES) { + pr_err("gsp-phys: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n", + pci_name(pdev), + pdev->dev.driver ? pdev->dev.driver->name : "none", + pdev->dev.power.runtime_status); + /* + * Intentional imbalance: keep the claim's runtime-PM + * reference held together with the D0 locks so a board + * that did not finish resuming cannot be power-gated + * again. It stays held until reboot. + */ + if (phys.runtime_ref_held) + pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (rebind gate timeout)\n", + pci_name(pdev)); + pci_dev_put(pdev); + return; + } + msleep(GSP_PHYS_POWER_SETTLE_MS); /* - * The distribution driver has rebound and settled: drop the - * claim's runtime-PM reference exactly once, before the D0 - * locks are released. + * The distribution driver has rebound, resumed and settled: + * drop the claim's runtime-PM reference exactly once, before + * the D0 locks are released (the unlock itself now latches + * when gsp_phys_repower=0). */ gsp_phys_runtime_ref_drop(pdev); pci_dev_put(pdev); @@ -1036,6 +1221,7 @@ static struct gsp_phys_backend_session { enum gsp_phys_backend_state state; u32 refcount; u32 generation; + u32 static_probe_generation; /* generation whose snapshot probe ran */ bool module_owned; bool releasing; } backend = { @@ -1061,6 +1247,51 @@ u32 gsp_phys_backend_generation(void) return READ_ONCE(backend.generation); } +/* + * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend + * generation, run on a work item because the probe takes gsp_phys_rm_lock and + * can wait seconds for the fn65 reply. The work item is coalesced by the + * workqueue and additionally guarded by static_probe_generation, so a second + * schedule for the same generation is a no-op. + */ +static void gsp_phys_backend_static_probe_work(struct work_struct *work); +static DECLARE_WORK(gsp_phys_static_probe_work_item, + gsp_phys_backend_static_probe_work); + +static void gsp_phys_backend_static_probe_work(struct work_struct *work) +{ + u32 generation; + int ret; + + mutex_lock(&gsp_phys_session_lock); + if (backend.state != GSP_PHYS_BACKEND_READY) { + mutex_unlock(&gsp_phys_session_lock); + return; + } + generation = backend.generation; + if (backend.static_probe_generation == generation) { + mutex_unlock(&gsp_phys_session_lock); + return; + } + backend.static_probe_generation = generation; + mutex_unlock(&gsp_phys_session_lock); + + ret = gsp_phys_backend_static_probe(); + if (ret) + pr_warn("gsp-phys: static-config probe failed: %d; the static overlay keeps the emulated model\n", + ret); +} + +/* Caller holds gsp_phys_session_lock. */ +static void gsp_phys_backend_static_probe_schedule(void) +{ + if (backend.state != GSP_PHYS_BACKEND_READY) + return; + if (backend.static_probe_generation == backend.generation) + return; + schedule_work(&gsp_phys_static_probe_work_item); +} + /* * Claim the release of the session under the session lock. Returns true only * for the caller that must run the sleeping release sequence. A module-owned @@ -1101,6 +1332,12 @@ static bool gsp_phys_backend_claim_release_locked(const char *who) static void gsp_phys_backend_release_run(void) { cancel_work_sync(&gsp_phys_backend_start_work_item); + /* + * A running static-config probe may be mid-RPC on the real GSP; wait + * for it to finish (bounded by the probe's own timeouts) before the + * release touches the device. + */ + cancel_work_sync(&gsp_phys_static_probe_work_item); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); @@ -1132,6 +1369,8 @@ int gsp_phys_backend_start(void) backend.state, backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); + /* A new/joined session re-enables P5 shadow requests. */ + gsp_sema_service_work_resume(); return 0; } @@ -1152,23 +1391,40 @@ int gsp_phys_backend_start(void) backend.module_owned = was_claimed; if (was_claimed) { + if (!gsp_phys_boot_rm_ready()) + pr_warn("gsp-phys: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n"); + /* + * A new READY session: bump the generation (so a cache left + * from an earlier session can never match) and fill the + * static-config snapshot once, asynchronously. + */ + backend.generation++; + gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); - pr_info("gsp-phys: backend start: joined the live module-owned claim (READY)\n"); + pr_info("gsp-phys: backend start: joined the live module-owned claim (READY, generation=%u)\n", + backend.generation); + gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); return 0; } + backend.generation++; + gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING); - pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1)\n"); + pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n", + backend.generation); schedule_work(&gsp_phys_backend_start_work_item); mutex_unlock(&gsp_phys_session_lock); + /* A new session re-enables P5 shadow requests (gated on READY). */ + gsp_sema_service_work_resume(); return 0; } /* * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle - * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev - * create and init_work never block on the 60 s init-done poll. A failure + * create and init_work never block on the 60 s init-done poll. Any failure, + * including a successful claim whose GSP-RM session never became usable, * leaves the backend OFF: the mdev keeps working on the existing emulation * path. */ @@ -1201,12 +1457,38 @@ static void gsp_phys_backend_start_work(struct work_struct *work) if (ret) { backend.refcount = 0; backend.module_owned = false; + backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); + gsp_phys_backend_static_invalidate(); pr_err("gsp-phys: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n", ret); return; } + /* + * A successful claim is not enough: only a live GSP-RM session may be + * exposed as READY (a failed stage-3d boot leaves rm_init_done false + * and must not claim otherwise). Fail open by releasing the claim the + * way the stopper would; the emulation path keeps serving the guest. + */ + if (!gsp_phys_boot_rm_ready()) { + u32 rm_init_done, rm_unload_submitted; + + gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted); + mutex_unlock(&gsp_phys_session_lock); + pr_err("gsp-phys: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n", + rm_init_done, rm_unload_submitted); + if (gsp_phys_gpu_claimed()) + gsp_phys_gpu_release(); + mutex_lock(&gsp_phys_session_lock); + backend.refcount = 0; + backend.module_owned = false; + backend.generation++; + WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); + mutex_unlock(&gsp_phys_session_lock); + gsp_phys_backend_static_invalidate(); + return; + } if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start: claim completed while STOPPING; release deferred to the stopper\n"); @@ -1215,6 +1497,12 @@ static void gsp_phys_backend_start_work(struct work_struct *work) WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("gsp-phys: backend start: READY (owner=mdev, refcount=%u)\n", backend.refcount); + /* + * Fill the static-config snapshot once for this generation before any + * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm + * lock itself and fails open. + */ + gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); } @@ -1253,6 +1541,14 @@ void gsp_phys_backend_stop(void) */ void gsp_phys_backend_begin_stop(void) { + /* + * Flush the static-config probe work item first (no session lock + * held): a running probe may be mid-RPC and must finish before the + * release touches the device. For a module-owned claim this is the + * only effect; the caller still releases the claim itself. + */ + cancel_work_sync(&gsp_phys_static_probe_work_item); + mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); -- cgit v1.2.3