/* SPDX-License-Identifier: GPL-2.0 */ #ifndef GSP_PHYS_BOOTER_H #define GSP_PHYS_BOOTER_H #include #include #include #include "gsp_phys_boot.h" /* * Real-GSP boot stages 3d/3e: Booter Load on SEC2, RISC-V reset and libos * arguments, FALCON_OS, RISC-V-active check (3d), then the status-queue poll * for the GSP-RM init event (3e). * * Stage 3d runs only after a successful FWSEC run (WPR2 up) in the same * claim. Its host-side staging - FMC copy, radix3 page tables for the GSP-RM * ELF, signature copy, message queues and the libos/GSP arguments - runs * first; gsp_phys_boot_dryrun prepares exactly that staging with no BAR0 * writes and no falcon kicks. */ bool gsp_phys_boot_enabled(void); bool gsp_phys_boot_dryrun_enabled(void); /* True when either boot parameter is set (forces the stage-3b layout). */ bool gsp_phys_boot_requested(void); /* * Live-session accessor: true only when the real GSP-RM session is usable * (stage 3e saw init-done and the UNLOADING_GUEST_DRIVER RPC has not been * submitted in this claim). The backend marks itself READY only when this is * true; gsp_phys_boot_rm_flags() exposes the underlying bits for logging. */ bool gsp_phys_boot_rm_ready(void); void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted); /* Views into the staged GSP firmware ELF (set by gsp_phys_gpu.c). */ struct gsp_phys_boot_sources { const u8 *fw_image; /* .fwimage section of the GSP-RM ELF */ u64 fw_image_size; const u8 *fw_signature; /* .fwsignature_tu10x section */ u64 fw_signature_size; }; /* * Stage 3d host-memory staging: allocate/copy every sysmem buffer the Booter * Load ucode DMAs from, fill the message queues / libos arguments / GSP * arguments, update the WPR metadata sysmem addresses and run the placement * self-checks. The only device access is the bounded read-only state log. * With dryrun=true nothing device-visible is written and no falcon is * touched. Staging stays allocated until gsp_phys_boot_quiesce() / * gsp_phys_boot_staging_free() on release. */ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0, const struct gsp_phys_boot_sources *src, bool dryrun); /* * Stage 3d device sequence: requires primed staging and WPR2 up. Resets the * GSP falcon into RISC-V, programs the libos args mailbox, runs Booter Load * on SEC2 with the WPR metadata address, writes FALCON_OS and verifies the * RISC-V core is active. */ int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0); /* * Stage 3e: bounded status-queue poll for the GSP-RM init event (function * 0x1001, rpc_result 0). When GSP-RM asks the host to execute an inline CPU * sequencer command buffer first (function 0x1002, * NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER), the buffer is executed against * the claimed BAR0 (register operations bounded and validated against * bar0_len) and the poll then continues. Non-fatal: a timeout or a sequencer * validation/execution failure leaves the successful 3d boot and the release * teardown valid. */ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len); /* * Release sequence, in this order (gsp_phys_gpu_release): * 0. gsp_phys_boot_rm_unload(): if stage 3d ran, the GSP-RM init-done * event (function 0x1001 with rpc_result 0) was observed, and the * UNLOADING_GUEST_DRIVER RPC has not been submitted yet, submit it once * (bounded) and wait for the GSP-RM processor suspend so the scratch14 * handoff bit the Booter Unload gate needs can be set; a no-op when * stage 3d did not run, init-done was never seen (an RPC before * init-done can be dropped) or the RPC was already sent in this boot; * 1. gsp_phys_boot_quiesce(): if a Booter Load kick was issued, reset the * GSP falcon (stops RISC-V/GSP-RM) BEFORE the FWSEC teardown runs; no * device access when bar0 is NULL (unresponsive-board abort); * 2. the existing gsp_phys_fwsec_release() teardown (FWSEC-SB + booter * unload); * 3. gsp_phys_boot_staging_free(): release every host buffer. Called after * both falcons have been reset so no ucode can still DMA from it. */ void gsp_phys_boot_quiesce(void __iomem *bar0); void gsp_phys_boot_staging_free(struct pci_dev *pdev); /* * Backend session transport API (P0 skeleton; inert until the guest plane * forwards). The session state/refcount/owner live in gsp_phys_gpu.c; these * entry points share gsp_phys_rm_lock with the boot/probe/ladder paths so * there is one real-GSP RPC at a time. Callers check gsp_phys_backend_state() * first and call here; this file never takes the session lock itself (lock * order: session lock outermost, rm lock inner). * * gsp_phys_backend_rpc() re-checks READY and the generation under the rm * lock and refuses (-EAGAIN) without touching the device when the session is * not live. On success *reply owns a kmalloc'd copy of the reply payload * (*reply_len bytes, NULL/0 for an empty payload, caller frees). * * gsp_phys_backend_static_get() returns the cached static-config fields * decoded from the function-65 reply; the cache is filled by the READY-time * probe (gsp_phys_backend_static_probe() on the backend work item) and is * invalidated whenever the backend session leaves READY. It returns -ENODATA * while no valid cache exists and -EAGAIN when the session is not READY or * the cache's generation key no longer matches the backend's. * * gsp_phys_backend_static_wait() polls the same getter in bounded sleeps * (never holding gsp_phys_rm_lock) and fails open with -ENODATA/-EAGAIN so a * caller can fall back to the emulated model. */ #define GSP_PHYS_BACKEND_GPU_NAME_LEN 64 struct gsp_phys_backend_static_config { bool valid; /* cache holds a decoded function-65 reply */ u32 generation; /* backend session generation the fill saw */ u32 fw_crc32; /* cache key: loaded firmware image */ u32 payload_len; /* decoded reply payload bytes */ u32 h_internal_client; u32 h_internal_device; u32 h_internal_subdevice; u64 bar1_pde_base; u64 bar2_pde_base; u64 fb_length; u64 fbio_mask; u64 fbp_mask; u32 fb_bus_width; u32 fb_ram_type; u32 l2_cache_size; u32 engine_caps[3]; char gpu_name[GSP_PHYS_BACKEND_GPU_NAME_LEN + 1]; char gpu_short_name[GSP_PHYS_BACKEND_GPU_NAME_LEN + 1]; u32 sriov_max_gfid; u32 vbios_sub_vendor; u32 vbios_sub_device; u8 b_vbios_valid; u8 b_page_retirement; u8 b_split_vas; u8 b_self_hosted; u8 b_ats_supported; u8 b_poison_fuse; u8 b_gpu_uefi; u8 b_efi_init; }; int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, void **reply, u32 *reply_len, u32 timeout_ms); int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out); int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, u32 timeout_ms); void gsp_phys_backend_static_invalidate(void); /* * Run one fn1+fn65 probe on a READY backend to fill the static snapshot * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must * run on a work item, never inline under a caller-held rm lock or on the mdev * create path. Non-fatal: a failed probe leaves the cache as it was and the * caller keeps the emulation path. */ int gsp_phys_backend_static_probe(void); /* * P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1; default off). * * gsp_phys_backend_work_ready() is a lock-free readiness gate: the * gsp_phys_work_forward parameter is set, the backend session is READY, the * claim is live with GSP-RM init-done and no unload submitted, and the debugfs * ladder has a usable client-memory channel (channel handle, work-submit * token, non-server memory, PDB window and all ladder VAs present). It never * waits on gsp_phys_rm_lock; every submit re-checks the same state under the * lock before touching the device. * * gsp_phys_backend_work_operands() reports the ladder's module-owned CE * operand VAs (src/dst/semaphore) and the fixed shadow copy size, or false * while the ladder window is not usable. * * gsp_phys_backend_ce_copy_submit() submits one bytes-long CE copy on the * ladder channel: it clears the module semaphore, encodes the method block, * publishes the GPFIFO entry and USERD GP_PUT and rings the PF doorbell under * gsp_phys_rm_lock, then releases the lock and polls the module semaphore for * up to poll_ms (clamped to 200 ms) without the lock. Returns 0 when the * release is observed, -ETIMEDOUT on the poll bound, -EAGAIN when the backend * is not ready (including a release that committed before the lock), -EIO for * a structural guard failure (BAR0/token/GP_PUT bounds/verify) and * -EOPNOTSUPP for the server-allocated-memory channel variant. No guest * memory or guest VA is ever touched; counters are maintained for all paths. * * gsp_phys_backend_work_counters() reads the shadow counters (attempts, * observed releases, timeouts, refusals) for logging or the state file. */ #define GSP_PHYS_BACKEND_WORK_POLL_MS 50u /* submit clamps to 200 ms */ bool gsp_phys_backend_work_ready(void); bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, u32 *bytes); int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, u32 poll_ms); void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, u64 *timed_out, u64 *refused); /* * P6b wave-1 guest-operand forwarding (gsp_phys_work_forward=1 AND * gsp_phys_guest_mem=1; both default off, either off keeps the P5 ladder * shadow exactly as it was). * * gsp_phys_backend_guest_ce_copy() runs one bounded CE copy on the ladder * channel with both operands mapped to real guest RAM by translated host PA: * it re-checks the READY backend, the ladder channel, work forwarding, guest * memory, the identity DMA domain and the active guest-RAM file; consumes one * whole fresh, never-reused 128 B PTE fetch line per copy pair from the guest * scratch PT pool built at stage 3 (no line reuse: the GMMU caches PTE * fetches at 128 B granularity and there is no invalidate, so two pairs * sharing a line would leave the second pair's PTEs stale); then submits * src_off..+bytes from src_hpa to dst_hpa with the module semaphore, polls * it lock-free for up to poll_ms (clamped to 200 ms) and verifies the copy * with a direct-map memcmp (safe: identity domain and pfn_valid HPAs). Returns 0 on a verified copy, * -ENOSPC when the scratch-PT pool is exhausted, -ETIMEDOUT on the poll * bound, -EAGAIN when a gate is closed and -EINVAL/-EIO on operand/structural * failures. No guest VA is ever submitted; the sema service falls back to its * ladder copy on every failure, so the software release never depends on it. * * gsp_phys_backend_guest_counters() reads the booter-side guest counters * (verified copies, direct-map verifies, verify failures, pool exhaustion, * refusals including poll timeouts) for logging and the state file. */ int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms); void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, u64 *pool_exhausted, u64 *refused); /* * Command-queue TX: build a msgq record with the 570.144 element/rpc header * layout, splitting it into 1..16 4 KB elements when it does not fit one * (reference elemCount = ceil((48 + rpc.length) / 4096), each element copied * to its own slot, checksum folded over the whole record up to the * 8-byte-aligned length), validate the peer command read pointer * (status+rxHdrOff when the status header carries MSGQ_FLAGS_SWAP_RX, else * cmd+rxHdrOff) and the peer-derived free space, copy the record into the * next command-queue slot(s), publish the new writePtr under wmb() and ring * the GSP queue-head doorbell. Requires the stage-3d command-queue staging; * bounded and refuses (with a log) when the TX header does not match what was * staged or the ring cannot hold elemCount free elements. Does not touch WPR2 * or the falcons. */ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, const void *payload, u32 payload_len); /* * Send NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER (function 47) with a normal * unload payload (bInPMTransition=0, bGc6Entering=0, newLevel=0), then wait * (bounded, 10 s) for the GSP-RM processor to suspend the way * kgspWaitForProcessorSuspend_TU102 does: GSP falcon MAILBOX0 == 0x80000000. * Sends at most once per boot (tracked across release and the debugfs * trigger). Returns 0 only when the suspend was observed and all four * scratch14 handoff bits (24..27) read DONE, -EALREADY if already sent, * -EINVAL when stage 3d did not run or GSP-RM init-done (function 0x1001, * rpc_result 0) was never observed (an RPC before init-done can be dropped), * -ETIMEDOUT on the suspend bound and -EIO when the handoff bits are not all * DONE after the suspend. Never writes WPR2. */ int gsp_phys_boot_rm_unload(void __iomem *bar0); /* * Module-owned debugfs files (dir "open-iov-vfio"), registered by * nv_vgpu_init() and removed by nv_vgpu_exit(). All file_operations carry * .owner = THIS_MODULE so an open fd pins the module: * - "gsp-phys-unload-rpc" (write-only, mode 0200): writing "1" runs * gsp_phys_boot_rm_unload() on a live claim; every other value or state * (parameter off, GPU unclaimed, stage 3d not run, init-done not * observed, already submitted) is a logged no-op. * - "gsp-phys-rm-probe" (write-only, mode 0200): writing "1" runs one host * RM probe cycle on a live, init-done claim: GET_GSP_STATIC_INFO * (function 65, 4-byte request), and when no reply arrives, * SET_GUEST_SYSTEM_INFO (function 1, 792-byte version payload) once * followed by one GET_GSP_STATIC_INFO retry. Serialised with the unload * trigger by the same mutex and bounded by the per-wait timeouts; never * sends when init-done was not observed and never touches WPR2. The * static-info reply is decoded into dmesg (internal handles, split-VA * flag, BAR1/2 PDE bases, FB geometry, engine caps, SRIOV/board fields). * - "gsp-phys-rm-call" (write-only, mode 0200): writing N (1..4) runs the * host RM client ladder cumulatively through stage N on a live, init-done * claim and remembers the reached stage for the rest of the session: * stage 1 allocates NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0 * (function 103); stage 2 allocates and registers the nine DMA buffers * (function 4, NV01_MEMORY_LIST_SYSTEM with real PFNs); stage 3 allocates * KEPLER_CHANNEL_GROUP_A (COPY0) and TURING_CHANNEL_GPFIFO_A with the * physical instance/userd/ramfc/mthdbuf descriptors then requests the * work-submit token (function 76); stage 4 schedules the channel, writes * the CE method block plus GPFIFO entry and USERD GP_PUT, rings the PF * channel doorbell and polls the sysmem semaphore for a real 4 KB * sysmem-to-sysmem copy (function 76 + the CE doorbell path). Writing 5 * is the alternative stage-1 path: it adopts the * hInternalClient/hInternalDevice/hInternalSubdevice handles from the * session's decoded GspStaticConfigInfo (running the fn1+fn65 probe cycle * inline first when needed), marks stage 1 satisfied, and a following * write of 4 runs stages 2-4 against the adopted handles. On a failed * reply (rpc_result != 0, e.g. the fn103 NV01_ROOT rejection) the RPC * helper logs the payload words at +0/+16/+32 and the first 64 payload * bytes in bounded hex. Serialised with the other triggers by * gsp_phys_rm_lock, per-RPC bounded (2000 ms), aborts at the first * failure with no object teardown (UNLOAD + rmmod is the fail-safe), * never touches WPR2, and is inert until the file is written. Ladder * buffers are freed by gsp_phys_boot_staging_free() on release. With * gsp_phys_guest_mem=1 and a GPA pinned through gsp-phys-guest-gpa, * writing 7 (G2H, guest read) and then 6 (W2G, guest write) additionally * run one 4 KB CE copy each against that guest RAM page: the GPA is * translated through the active mdev's VFIO type-1 domain, the physical * device's domain must be identity, the corresponding channel PDB slot * (0x106 read / 0x105 write) is programmed with the translated HPA, and * the result is verified with a direct-map memcmp. Both directions are * gated on a live READY claim, ladder stage >= 3 and the pinned GPA; * every refusal is a logged no-op and no guest byte is touched unless * G2H already proved the translation. * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM * GPA ("0x", 4 KB page-aligned) that the rm-call guest directions * may touch. One GPA per boot (a second different GPA is refused: * stale-GMMU-TLB safety); the pin is cleared with the ladder state on * release/re-prime. * - "gsp-phys-log-dump" (read-only, mode 0444): dumps the LOGINIT and * LOGRM libos task-log host buffers allocated at stage-3d prime. Per * buffer: a one-line header (name, allocated size, kernel VA, DMA * address), the raw content as "%08x: %08x ..." hex lines (bounded to * the allocated size and 64 KB per buffer; authoritative for offline * decoding with the evidence tree's decode_logs.py) and a decoded view * derived from the known libos record format (word[0]=put pointer, * records {args..., metadata_ptr, timestamp}, last <=60 timestamp-like * records). Read-only: no buffer byte is modified. * - "gsp-phys-state" (read-only, mode 0444): seq_file snapshot of the * claimed/booted state (falcon registers, scratch14, both queue TX * headers, a bounded read-only status-ring walk, the stage flags and the * last RM probe result). It returns a one-line status instead of * touching hardware when the device is not claimed or the stage-3d * staging is not primed, never writes and never advances the host * status-queue readPtr (no consumption). */ int gsp_phys_boot_debugfs_init(void); void gsp_phys_boot_debugfs_fini(void); #endif /* GSP_PHYS_BOOTER_H */