path: root/kernel/gsp_phys_booter.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef GSP_PHYS_BOOTER_H
#define GSP_PHYS_BOOTER_H

#include <linux/types.h>
#include <linux/pci.h>
#include <linux/io.h>

#include "gsp_phys_boot.h"

/*
 * Real-GSP boot stages 3d/3e: Booter Load on SEC2, RISC-V reset and libos
 * arguments, FALCON_OS, RISC-V-active check (3d), then the status-queue poll
 * for the GSP-RM init event (3e).
 *
 * Stage 3d runs only after a successful FWSEC run (WPR2 up) in the same
 * claim. Its host-side staging - FMC copy, radix3 page tables for the GSP-RM
 * ELF, signature copy, message queues and the libos/GSP arguments - runs
 * first; gsp_phys_boot_dryrun prepares exactly that staging with no BAR0
 * writes and no falcon kicks.
 */

bool gsp_phys_boot_enabled(void);
bool gsp_phys_boot_dryrun_enabled(void);

/* True when either boot parameter is set (forces the stage-3b layout). */
bool gsp_phys_boot_requested(void);

/*
 * Live-session accessor: true only when the real GSP-RM session is usable
 * (stage 3e saw init-done and the UNLOADING_GUEST_DRIVER RPC has not been
 * submitted in this claim). The backend marks itself READY only when this is
 * true; gsp_phys_boot_rm_flags() exposes the underlying bits for logging.
 */
bool gsp_phys_boot_rm_ready(void);
void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted);

/* Views into the staged GSP firmware ELF (set by gsp_phys_gpu.c). */
struct gsp_phys_boot_sources {
	const u8 *fw_image;		/* .fwimage section of the GSP-RM ELF */
	u64 fw_image_size;
	const u8 *fw_signature;		/* .fwsignature_tu10x section */
	u64 fw_signature_size;
};

/*
 * Stage 3d host-memory staging: allocate/copy every sysmem buffer the Booter
 * Load ucode DMAs from, fill the message queues / libos arguments / GSP
 * arguments, update the WPR metadata sysmem addresses and run the placement
 * self-checks. The only device access is the bounded read-only state log.
 * With dryrun=true nothing device-visible is written and no falcon is
 * touched. Staging stays allocated until gsp_phys_boot_quiesce() /
 * gsp_phys_boot_staging_free() on release.
 */
int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			void __iomem *bar0,
			const struct gsp_phys_boot_sources *src, bool dryrun);

/*
 * Stage 3d device sequence: requires primed staging and WPR2 up. Resets the
 * GSP falcon into RISC-V, programs the libos args mailbox, runs Booter Load
 * on SEC2 with the WPR metadata address, writes FALCON_OS and verifies the
 * RISC-V core is active.
 */
int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		      void __iomem *bar0);

/*
 * Stage 3e: bounded status-queue poll for the GSP-RM init event (function
 * 0x1001, rpc_result 0). When GSP-RM asks the host to execute an inline CPU
 * sequencer command buffer first (function 0x1002,
 * NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER), the buffer is executed against
 * the claimed BAR0 (register operations bounded and validated against
 * bar0_len) and the poll then continues. Non-fatal: a timeout or a sequencer
 * validation/execution failure leaves the successful 3d boot and the release
 * teardown valid.
 */
void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
			  resource_size_t bar0_len);

/*
 * Release sequence, in this order (gsp_phys_gpu_release):
 *   0. gsp_phys_boot_rm_unload(): if stage 3d ran, the GSP-RM init-done
 *      event (function 0x1001 with rpc_result 0) was observed, and the
 *      UNLOADING_GUEST_DRIVER RPC has not been submitted yet, submit it once
 *      (bounded) and wait for the GSP-RM processor suspend so the scratch14
 *      handoff bit the Booter Unload gate needs can be set; a no-op when
 *      stage 3d did not run, init-done was never seen (an RPC before
 *      init-done can be dropped) or the RPC was already sent in this boot;
 *   1. gsp_phys_boot_quiesce(): if a Booter Load kick was issued, reset the
 *      GSP falcon (stops RISC-V/GSP-RM) BEFORE the FWSEC teardown runs; no
 *      device access when bar0 is NULL (unresponsive-board abort);
 *   2. the existing gsp_phys_fwsec_release() teardown (FWSEC-SB + booter
 *      unload);
 *   3. gsp_phys_boot_staging_free(): release every host buffer. Called after
 *      both falcons have been reset so no ucode can still DMA from it.
 */
void gsp_phys_boot_quiesce(void __iomem *bar0);
void gsp_phys_boot_staging_free(struct pci_dev *pdev);

/*
 * Backend session transport API (P0 skeleton; inert until the guest plane
 * forwards). The session state/refcount/owner live in gsp_phys_gpu.c; these
 * entry points share gsp_phys_rm_lock with the boot/probe/ladder paths so
 * there is one real-GSP RPC at a time. Callers check gsp_phys_backend_state()
 * first and call here; this file never takes the session lock itself (lock
 * order: session lock outermost, rm lock inner).
 *
 * gsp_phys_backend_rpc() re-checks READY and the generation under the rm
 * lock and refuses (-EAGAIN) without touching the device when the session is
 * not live. On success *reply owns a kmalloc'd copy of the reply payload
 * (*reply_len bytes, NULL/0 for an empty payload, caller frees).
 *
 * gsp_phys_backend_static_get() returns the cached static-config fields
 * decoded from the function-65 reply; the cache is filled by the READY-time
 * probe (gsp_phys_backend_static_probe() on the backend work item) and is
 * invalidated whenever the backend session leaves READY. It returns -ENODATA
 * while no valid cache exists and -EAGAIN when the session is not READY or
 * the cache's generation key no longer matches the backend's.
 *
 * gsp_phys_backend_static_wait() polls the same getter in bounded sleeps
 * (never holding gsp_phys_rm_lock) and fails open with -ENODATA/-EAGAIN so a
 * caller can fall back to the emulated model.
 */
#define GSP_PHYS_BACKEND_GPU_NAME_LEN	64

struct gsp_phys_backend_static_config {
	bool valid;			/* cache holds a decoded function-65 reply */
	u32 generation;			/* backend session generation the fill saw */
	u32 fw_crc32;			/* cache key: loaded firmware image */
	u32 payload_len;		/* decoded reply payload bytes */
	u32 h_internal_client;
	u32 h_internal_device;
	u32 h_internal_subdevice;
	u64 bar1_pde_base;
	u64 bar2_pde_base;
	u64 fb_length;
	u64 fbio_mask;
	u64 fbp_mask;
	u32 fb_bus_width;
	u32 fb_ram_type;
	u32 l2_cache_size;
	u32 engine_caps[3];
	char gpu_name[GSP_PHYS_BACKEND_GPU_NAME_LEN + 1];
	char gpu_short_name[GSP_PHYS_BACKEND_GPU_NAME_LEN + 1];
	u32 sriov_max_gfid;
	u32 vbios_sub_vendor;
	u32 vbios_sub_device;
	u8 b_vbios_valid;
	u8 b_page_retirement;
	u8 b_split_vas;
	u8 b_self_hosted;
	u8 b_ats_supported;
	u8 b_poison_fuse;
	u8 b_gpu_uefi;
	u8 b_efi_init;
};

int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len,
			 void **reply, u32 *reply_len, u32 timeout_ms);
int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out);
int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out,
				 u32 timeout_ms);
void gsp_phys_backend_static_invalidate(void);

/*
 * Run one fn1+fn65 probe on a READY backend to fill the static snapshot
 * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must
 * run on a work item, never inline under a caller-held rm lock or on the mdev
 * create path. Non-fatal: a failed probe leaves the cache as it was and the
 * caller keeps the emulation path.
 */
int gsp_phys_backend_static_probe(void);

/*
 * P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1; default off).
 *
 * gsp_phys_backend_work_ready() is a lock-free readiness gate: the
 * gsp_phys_work_forward parameter is set, the backend session is READY, the
 * claim is live with GSP-RM init-done and no unload submitted, and the debugfs
 * ladder has a usable client-memory channel (channel handle, work-submit
 * token, non-server memory, PDB window and all ladder VAs present). It never
 * waits on gsp_phys_rm_lock; every submit re-checks the same state under the
 * lock before touching the device.
 *
 * gsp_phys_backend_work_operands() reports the ladder's module-owned CE
 * operand VAs (src/dst/semaphore) and the fixed shadow copy size, or false
 * while the ladder window is not usable.
 *
 * gsp_phys_backend_ce_copy_submit() submits one bytes-long CE copy on the
 * ladder channel: it clears the module semaphore, encodes the method block,
 * publishes the GPFIFO entry and USERD GP_PUT and rings the PF doorbell under
 * gsp_phys_rm_lock, then releases the lock and polls the module semaphore for
 * up to poll_ms (clamped to 200 ms) without the lock. Returns 0 when the
 * release is observed, -ETIMEDOUT on the poll bound, -EAGAIN when the backend
 * is not ready (including a release that committed before the lock), -EIO for
 * a structural guard failure (BAR0/token/GP_PUT bounds/verify) and
 * -EOPNOTSUPP for the server-allocated-memory channel variant. No guest
 * memory or guest VA is ever touched; counters are maintained for all paths.
 *
 * gsp_phys_backend_work_counters() reads the shadow counters (attempts,
 * observed releases, timeouts, refusals) for logging or the state file.
 */
#define GSP_PHYS_BACKEND_WORK_POLL_MS	50u	/* submit clamps to 200 ms */

bool gsp_phys_backend_work_ready(void);
bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va,
				    u32 *bytes);
int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va,
				    u32 bytes, u32 poll_ms);
void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
				    u64 *timed_out, u64 *refused);

/*
 * Command-queue TX: build a msgq record with the 570.144 element/rpc header
 * layout, splitting it into 1..16 4 KB elements when it does not fit one
 * (reference elemCount = ceil((48 + rpc.length) / 4096), each element copied
 * to its own slot, checksum folded over the whole record up to the
 * 8-byte-aligned length), validate the peer command read pointer
 * (status+rxHdrOff when the status header carries MSGQ_FLAGS_SWAP_RX, else
 * cmd+rxHdrOff) and the peer-derived free space, copy the record into the
 * next command-queue slot(s), publish the new writePtr under wmb() and ring
 * the GSP queue-head doorbell. Requires the stage-3d command-queue staging;
 * bounded and refuses (with a log) when the TX header does not match what was
 * staged or the ring cannot hold elemCount free elements. Does not touch WPR2
 * or the falcons.
 */
int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
			     const void *payload, u32 payload_len);

/*
 * Send NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER (function 47) with a normal
 * unload payload (bInPMTransition=0, bGc6Entering=0, newLevel=0), then wait
 * (bounded, 10 s) for the GSP-RM processor to suspend the way
 * kgspWaitForProcessorSuspend_TU102 does: GSP falcon MAILBOX0 == 0x80000000.
 * Sends at most once per boot (tracked across release and the debugfs
 * trigger). Returns 0 only when the suspend was observed and all four
 * scratch14 handoff bits (24..27) read DONE, -EALREADY if already sent,
 * -EINVAL when stage 3d did not run or GSP-RM init-done (function 0x1001,
 * rpc_result 0) was never observed (an RPC before init-done can be dropped),
 * -ETIMEDOUT on the suspend bound and -EIO when the handoff bits are not all
 * DONE after the suspend. Never writes WPR2.
 */
int gsp_phys_boot_rm_unload(void __iomem *bar0);

/*
 * Module-owned debugfs files (dir "open-iov-vfio"), registered by
 * nv_vgpu_init() and removed by nv_vgpu_exit(). All file_operations carry
 * .owner = THIS_MODULE so an open fd pins the module:
 *   - "gsp-phys-unload-rpc" (write-only, mode 0200): writing "1" runs
 *     gsp_phys_boot_rm_unload() on a live claim; every other value or state
 *     (parameter off, GPU unclaimed, stage 3d not run, init-done not
 *     observed, already submitted) is a logged no-op.
 *   - "gsp-phys-rm-probe" (write-only, mode 0200): writing "1" runs one host
 *     RM probe cycle on a live, init-done claim: GET_GSP_STATIC_INFO
 *     (function 65, 4-byte request), and when no reply arrives,
 *     SET_GUEST_SYSTEM_INFO (function 1, 792-byte version payload) once
 *     followed by one GET_GSP_STATIC_INFO retry. Serialised with the unload
 *     trigger by the same mutex and bounded by the per-wait timeouts; never
 *     sends when init-done was not observed and never touches WPR2. The
 *     static-info reply is decoded into dmesg (internal handles, split-VA
 *     flag, BAR1/2 PDE bases, FB geometry, engine caps, SRIOV/board fields).
 *   - "gsp-phys-rm-call" (write-only, mode 0200): writing N (1..4) runs the
 *     host RM client ladder cumulatively through stage N on a live, init-done
 *     claim and remembers the reached stage for the rest of the session:
 *     stage 1 allocates NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0
 *     (function 103); stage 2 allocates and registers the nine DMA buffers
 *     (function 4, NV01_MEMORY_LIST_SYSTEM with real PFNs); stage 3 allocates
 *     KEPLER_CHANNEL_GROUP_A (COPY0) and TURING_CHANNEL_GPFIFO_A with the
 *     physical instance/userd/ramfc/mthdbuf descriptors then requests the
 *     work-submit token (function 76); stage 4 schedules the channel, writes
 *     the CE method block plus GPFIFO entry and USERD GP_PUT, rings the PF
 *     channel doorbell and polls the sysmem semaphore for a real 4 KB
 *     sysmem-to-sysmem copy (function 76 + the CE doorbell path). Writing 5
 *     is the alternative stage-1 path: it adopts the
 *     hInternalClient/hInternalDevice/hInternalSubdevice handles from the
 *     session's decoded GspStaticConfigInfo (running the fn1+fn65 probe cycle
 *     inline first when needed), marks stage 1 satisfied, and a following
 *     write of 4 runs stages 2-4 against the adopted handles. On a failed
 *     reply (rpc_result != 0, e.g. the fn103 NV01_ROOT rejection) the RPC
 *     helper logs the payload words at +0/+16/+32 and the first 64 payload
 *     bytes in bounded hex. Serialised with the other triggers by
 *     gsp_phys_rm_lock, per-RPC bounded (2000 ms), aborts at the first
 *     failure with no object teardown (UNLOAD + rmmod is the fail-safe),
 *     never touches WPR2, and is inert until the file is written. Ladder
 *     buffers are freed by gsp_phys_boot_staging_free() on release.
 *   - "gsp-phys-log-dump" (read-only, mode 0444): dumps the LOGINIT and
 *     LOGRM libos task-log host buffers allocated at stage-3d prime. Per
 *     buffer: a one-line header (name, allocated size, kernel VA, DMA
 *     address), the raw content as "%08x: %08x ..." hex lines (bounded to
 *     the allocated size and 64 KB per buffer; authoritative for offline
 *     decoding with the evidence tree's decode_logs.py) and a decoded view
 *     derived from the known libos record format (word[0]=put pointer,
 *     records {args..., metadata_ptr, timestamp}, last <=60 timestamp-like
 *     records). Read-only: no buffer byte is modified.
 *   - "gsp-phys-state" (read-only, mode 0444): seq_file snapshot of the
 *     claimed/booted state (falcon registers, scratch14, both queue TX
 *     headers, a bounded read-only status-ring walk, the stage flags and the
 *     last RM probe result). It returns a one-line status instead of
 *     touching hardware when the device is not claimed or the stage-3d
 *     staging is not primed, never writes and never advances the host
 *     status-queue readPtr (no consumption).
 */
int gsp_phys_boot_debugfs_init(void);
void gsp_phys_boot_debugfs_fini(void);

#endif /* GSP_PHYS_BOOTER_H */