path: root/kernel/gsp_phys_gpu.h

#ifndef GSP_PHYS_GPU_H
#define GSP_PHYS_GPU_H

#include <linux/types.h>

/*
 * Physical GPU ownership (real-GSP execution backend, bring-up rung 1).
 * Claim is disabled unless the gsp_phys_gpu module parameter is set.
 */
bool gsp_phys_gpu_claimed(void);
int gsp_phys_gpu_claim(void);
void gsp_phys_gpu_release(void);
void __iomem *gsp_phys_gpu_bar0(void);
u32 gsp_phys_gpu_fw_crc32(void);

/*
 * Read by the backend session to enforce the mdev-owned start gate; the
 * gsp_phys_gpu module parameter itself is owned by main.c.
 */
bool gsp_phys_gpu_enabled(void);

/*
 * Backend session state (P0 skeleton + P1 lifecycle; one physical GPU, one
 * live claim). OFF until a start claims the device; READY is the only state
 * in which live RPC forwarding is legal. The refcount and owner flag decide
 * who releases: the load-time module-owned claim is joined, never released,
 * by a stop(); an mdev-owned claim is released by the last stop().
 */
enum gsp_phys_backend_state {
	GSP_PHYS_BACKEND_OFF = 0,
	GSP_PHYS_BACKEND_CLAIMING,
	GSP_PHYS_BACKEND_BOOTING,
	GSP_PHYS_BACKEND_READY,
	GSP_PHYS_BACKEND_STOPPING,
};

/*
 * Refcounted session start/stop around gsp_phys_gpu_claim()/_release(); safe
 * (and idempotent) in any state. gsp_phys_backend_start() never blocks on the
 * claim/boot: an mdev-owned start schedules a work item and returns, and a
 * failed claim leaves the session OFF (fail-open: the caller keeps using the
 * emulation path). The generation counter is bumped when the session leaves
 * READY; a forward that captured the previous value re-checks it under
 * gsp_phys_rm_lock and refuses, so a stop cannot race an in-flight forward.
 * The lock itself stays private to gsp_phys_gpu.c; the state and generation
 * accessors are lock-free reads for use from the RPC path.
 *
 * Module exit uses the two-phase form so the §1.4 ordering can be kept:
 * begin_stop() gates the session (STOPPING + generation bump) before the
 * caller drains the emulation threads, then end_stop() runs the release.
 */
int gsp_phys_backend_start(void);
void gsp_phys_backend_stop(void);
void gsp_phys_backend_begin_stop(void);
void gsp_phys_backend_end_stop(void);
int gsp_phys_backend_state(void);
u32 gsp_phys_backend_generation(void);

/*
 * True when the gsp_phys_mdev_owned module parameter is set (default 0:
 * load-time module-owned claim/boot, today's behavior).
 */
bool gsp_phys_mdev_owned_enabled(void);

/*
 * True when the gsp_phys_static_live module parameter is set (default 0:
 * the guest GET_GSP_STATIC_INFO reply is the emulated model unchanged). When
 * set, the handler overlays the live GPU name/short-name/VBIOS identity from
 * the backend snapshot, failing open to the emulated reply on any miss.
 */
bool gsp_phys_static_live_enabled(void);

/*
 * True when the gsp_phys_ctrl_forward module parameter is set (default 0:
 * guest GSP_RM_CONTROL replies are the emulated answers unchanged). When set,
 * a whitelist of side-effect-free control queries is forwarded to the live
 * backend with the snapshot's adopted internal handles, falling back to the
 * emulated per-command handler on any miss.
 */
bool gsp_phys_ctrl_forward_enabled(void);
bool gsp_phys_srv_ctxbuf_value(void);

/*
 * gsp_phys_rmcall_srv_ctxcommit mode (default 1). On the server-managed
 * ctx-buffer path (gsp_phys_srv_ctxbuf=1) the ladder maps the MAIN at the
 * ctx VA and commits it: 1 = GPU_INITIALIZE_CTX + UVM-shape VA-only
 * GPU_PROMOTE_CTX; 2 = V6 client variant, one combined PA+VA MAIN entry with
 * bInitialize=1 and no INITIALIZE_CTX; 0 = pure server-managed, no client
 * controls. gsp_phys_srv_ctxcommit_value() is true for any nonzero mode.
 */
bool gsp_phys_srv_ctxcommit_value(void);

/*
 * gsp_phys_rmcall_srv_main_ignore (default 0): when set, the srv-ctx commit
 * ignores a server-reported MAIN ctx buffer and forces the client
 * INITIALIZE_CTX + VA-only promote fallback.
 */
bool gsp_phys_srv_main_ignore_value(void);
int gsp_phys_srv_ctxcommit_mode(void);

/*
 * gsp_phys_rmcall_channel: force one channel-alloc variant id on the GR
 * ladder (0 = the promote-mode-selected default). 5 = V5 (client memdescs,
 * no skip-ctxbuffer), 6 = V6 (SKIP_CTXBUFFER_ALLOC, KMD-managed ctx).
 */
int gsp_phys_rmcall_channel_value(void);

/*
 * gsp_phys_rmcall_fwclient_mask (default 0x3f): bitmask selecting the mode-24
 * FWCLIENT promote entries to emit (bit0 MAIN, bit1 PM, bit2 PATCH,
 * bit3 FECS_EVENT, bit4 PRIV_ACCESS_MAP, bit5 UNRESTRICTED_PRIV_ACCESS_MAP).
 */
u32 gsp_phys_rmcall_fwclient_mask_value(void);

/*
 * gsp_phys_rmcall_golden_exec_shape (default 7): mode-28 golden execution
 * channel shape bitmask. bit0 GPFIFO ring at VA_GPFIFO, bit1
 * gpFifoEntries=8 (else the captured 32), bit2 sysmem USERD chunk (else the
 * captured FBMEM slice).
 */
u32 gsp_phys_rmcall_golden_exec_shape_value(void);

/*
 * gsp_phys_rmcall_golden (default 0): replay the stock client's golden-image
 * channel allocations in stage 3 before the ladder's own channel when set and
 * gsp_phys_gpu=1. With 0, the ladder is byte-identical to before.
 */
bool gsp_phys_rmcall_golden_enabled(void);

/*
 * True when gsp_phys_rmcall_ctxshare_force is set (default 0): allocate
 * FERMI_CONTEXT_SHARE_A on the ladder TSG before the golden replay for any GR
 * promote mode, so the mode-31 execution channel V6 attaches to the share
 * while the mode-31 construct/commit flow stays intact.
 */
bool gsp_phys_rmcall_ctxshare_force_value(void);

/*
 * True when the gsp_phys_work_forward module parameter is set (default 0: the
 * guest's CE pushes stay software-emulated exactly as before). When set, the
 * sema service shadows each decoded CE push as one copy on the debugfs ladder's
 * real channel using only module-owned ladder buffers; the software release
 * remains authoritative and is applied unchanged either way, unless
 * gsp_phys_work_defer_release moves a forwarded push's release after its
 * hardware copy (see below).
 */
bool gsp_phys_work_forward_enabled(void);

/*
 * True when the gsp_phys_gr_forward module parameter is set (default 0: guest
 * GR (compute) launches stay on the existing host CUDA bridge / software
 * path). When set, the sema service captures a completed guest inline QMD and
 * the booter submits it as real work on the ladder's physical GR channel,
 * but only after boot_stage.rmcall_gr_exec_ok proved a real GR submission at
 * boot (gsp_phys_backend_gr_ready() enforces the gate). Every forward
 * refusal/failure falls back to the CUDA bridge (when enabled) and then the
 * unchanged software release path. One-line rollback: gsp_phys_gr_forward=0.
 */
bool gsp_phys_gr_forward_enabled(void);

/*
 * True when the gsp_phys_guest_mem module parameter is set (default 0: the
 * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as
 * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real
 * CE engine at one pinned guest RAM page identified by GPA, proving the
 * translation path and both copy directions.
 */
bool gsp_phys_guest_mem_enabled(void);

/*
 * True when the gsp_phys_work_defer_release module parameter is set (default
 * 0: a forwarded guest CE push's release is applied inline with the software
 * copy, byte-identical to wave 1). When set with gsp_phys_work_forward and
 * gsp_phys_guest_mem, the release(s) a forwarded push decoded are held on the
 * service thread until the hardware copy completes, its bounded timeout
 * expires, or the forwarding is aborted; they are then applied from the
 * service thread. One-line rollback: gsp_phys_work_defer_release=0.
 */
bool gsp_phys_work_defer_release_enabled(void);

/*
 * True when the gsp_phys_work_hw_primary module parameter is set (default
 * 0: byte-identical to wave 1.5, the software copy always runs inline). When
 * set with gsp_phys_work_forward, gsp_phys_guest_mem and
 * gsp_phys_work_defer_release, a forwarded guest CE copy skips the software
 * copy (the hardware authors the data) and the release still lands only
 * after hardware completion through the wave-1.5 hold. Any hardware failure,
 * timeout, abort or failed guest queue falls back to exactly one software
 * copy in the service thread before the releases are applied, so the guest
 * always reads correct data. One-line rollback: gsp_phys_work_hw_primary=0.
 */
bool gsp_phys_work_hw_primary_enabled(void);

/*
 * True when the gsp_phys_relay module parameter is set (default 0: a
 * forwarded guest CE copy polls the module-owned ladder semaphore and the
 * software release path stays the sole author of the guest-visible release).
 * When set with gsp_phys_work_forward, gsp_phys_guest_mem and
 * gsp_phys_work_defer_release, one selected one-word release of a forwarded
 * guest push is authored by the real CE engine directly into the
 * guest-visible semaphore page (a third scratch PTE maps that page) and the
 * service thread skips the software apply of that record once the guest word
 * carries the payload; every failure, timeout, abort or failed verification
 * falls back to the unchanged software apply, so a release is never lost or
 * duplicated. One-line rollback: gsp_phys_relay=0.
 */
bool gsp_phys_relay_enabled(void);

/*
 * True when the gsp_phys_fault_relay module parameter is set (default 0:
 * a relayed forwarded CE copy that times out is abandoned to the unchanged
 * software release fallback, exactly as in P6c C0+C1). When set together
 * with gsp_phys_relay, gsp_phys_work_forward, gsp_phys_guest_mem and
 * gsp_phys_work_defer_release, the relay-timeout path drains the GSP-RM
 * status queue, decodes an engine fault (function 0x1006, corroborated by a
 * 0x1020 NOCAT record) and delivers it to the guest only when its
 * runlistId/chid match the ladder channel: the guest channel notifier is
 * written (info32=exceptType, info16=Nv2080 engine type, status=0xffff) and
 * the channel event posted (MSI-X). The software releases still run; a fault
 * is never delivered for a mismatch, a late release or a stale record.
 * One-line rollback: gsp_phys_fault_relay=0.
 */
bool gsp_phys_fault_relay_enabled(void);

/*
 * True when the gsp_phys_fault_inject module parameter is set (default 0:
 * forwarded guest CE copies program the real source PTE, byte-identical
 * behavior). When set, the relayed pair's source PTE is written as an
 * invalid (zero) entry on the next relay submission, so the engine faults
 * and the C3 fault path can be exercised end to end. It only ever changes
 * the source PTE of a relayed pair (a nonzero sem_hpa); every other copy and
 * every non-relay submission is untouched.
 */
bool gsp_phys_fault_inject_enabled(void);

/*
 * RM-client ladder engine selector (gsp_phys_rmcall_engine; default 0):
 * returns the raw module-parameter value, where 0 = COPY0 (the existing CE
 * ladder) and 1 = GR0 (the TURING_COMPUTE_A engine object plus a GR
 * semaphore release). The booter snapshots this when the rm-call ladder's
 * stage 3 starts and ignores later changes, so this accessor is for the
 * snapshot and diagnostics only.
 */
int gsp_phys_rmcall_engine_value(void);

/*
 * GR context-promote bisection selector (gsp_phys_rmcall_gr_promote; default
 * 0). Returns the module-parameter value, one of:
 *   0 = never send NV2080_CTRL_CMD_GPU_PROMOTE_CTX. The GR ladder still runs
 *       the engine-context geometry queries, allocates and PDB-maps the MAIN
 *       buffer, allocates the channel/compute object, then stops before BIND
 *       with a clear log (the whole safe half; reviewable without a promote);
 *   1 = single MAIN entry, bInitialize=1/bNonmapped=0, size =
 *       ALIGN_UP(raw save size, 4 KB) + maxSubctx * 4 KB (the p6h1b shape
 *       that asserted GSP-RM with errorCode 0x52a8b04);
 *   2 = reference external-VAS two-phase shape: a PA-only nonmapped entry
 *       (bInitialize=1/bNonmapped=1, no VA) first, then a VA-only entry
 *       (bInitialize=0/bNonmapped=0) with the same PA/VA/size;
 *   3 = single MAIN entry like 1 but with size = the exact raw save size;
 *   4 = like 2 but the VA-only step uses the reference client's zero-field
 *       entry (only bufferId/gpuVirtAddr/bNonmapped set);
 *   5 = one control, entryCount=2: PA-only nonmapped init entry followed by
 *       the zero-field VA-only entry;
 *   6 = no-entry hVirtMemory: allocate an NV01_MEMORY_VIRTUAL (class 0x70)
 *       VA-range object on the ladder VAS at the context window, then
 *       PROMOTE_CTX with hVirtMemory, virtAddress/size and entryCount=0, so
 *       the server creates the memdesc/vaList entry itself;
 *   7 = no-entry bare (virtAddress, size): PROMOTE_CTX with entryCount=0 and
 *       no hVirtMemory (the other documented no-entry branch);
 *   8..23 = later KMD-channel/old-interface/hVirtMemory/FBMEM bisection
 *       shapes; see gsp_phys_gpu.c MODULE_PARM_DESC for the full list;
 *   24 = the reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/
 *       PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the
 *       V6 KMD-managed channel: combined PA+VA entries where the ladder has
 *       a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted
 *       and logged;
 *   25 = the real golden-client construct on the golden replay handles
 *       (captured channel shape + single atomic 9-entry PROMOTE_CTX on the
 *       golden channel) followed by the V6 FWCLIENT promote; requires
 *       gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1.
 * The parameter's set callback clamps any value outside 0..25 to 0 with a
 * warning and warns on every nonzero enable (a failed promote can assert
 * GSP-RM and wedge the device; recovery needs a reboot with human approval).
 * The booter snapshots the value when the rm-call ladder's stage 3 starts
 * and ignores later changes, so this accessor is for the snapshot and
 * diagnostics only.
 */
int gsp_phys_rmcall_gr_promote_value(void);

/*
 * gsp_phys_rmcall_gr_prelude (default 0): 0 skips the captured verbatim
 * guest prelude and emits the rebuilt inline-QMD window (proven
 * init+window+PCAS+sem shape); 1 replays the verbatim prelude.
 */
int gsp_phys_rmcall_gr_prelude_value(void);

/*
 * gsp_phys_rmcall_gr_qmdvirt (default 1): emit
 * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE |
 * SEND_PCAS_ENABLE) in the GR launch block; 0 omits the method pair.
 */
int gsp_phys_rmcall_gr_qmdvirt_value(void);

/*
 * gsp_phys_rmcall_gr_pcas (default 1): emit the explicit
 * SEND_PCAS_A/SEND_SIGNALING_PCAS_B pair (vendor turingPushPcas copy form);
 * 0 emits the stock window-only shape (implicit IQ2M copy PCAS only).
 */
int gsp_phys_rmcall_gr_pcas_value(void);

/*
 * gsp_phys_rmcall_gr_pcas_two_step (default 0): set the QMD
 * REQUIRE_SCHEDULING_PCAS bit and emit copy PCAS followed by a schedule-only
 * PCAS; requires gsp_phys_rmcall_gr_pcas=1.
 */
int gsp_phys_rmcall_gr_pcas_two_step_value(void);

/*
 * gsp_phys_rmcall_gr_qmdwin (default 0): 1 relocates the forwarded GR
 * inline-QMD destination from the guest QMD VA to a dedicated module page
 * mapped at the proven engine-reachable window-0 VA 0x107000 (the PTE is
 * planted in the ladder small-PT line at stage 3, before the engine's first
 * fetch); 0 keeps the guest destination exactly as before. Rank-2
 * diagnostic: prewrite/post-mortem use the module page only.
 */
int gsp_phys_rmcall_gr_qmdwin_value(void);

/* m64 discriminant: force the forwarded GR QMD release0 to the module
 * semaphore so engine completion is always observable (default 0). */
int gsp_phys_rmcall_gr_rel_mod_value(void);

/* m66: register the mode-35 TSG fault buffers as host sysmem so engine fault
 * entries are host-readable (default 0 = server FBMEM first). */
int gsp_phys_rmcall_faultbuf_sys_value(void);

/* m69: copy forwarded GR program/param pages into the window-0 shadow pool
 * (proven small-PT line) instead of the window-1 scratch range (default 0). */
int gsp_phys_rmcall_gr_shadow_win0_value(void);

/* m72: append the vendor local-memory context block to the GR prelude so
 * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE passes (default 0); base VA and
 * per-TPC size are overridable for A/B runs. */
int gsp_phys_rmcall_gr_lmem_en_value(void);
unsigned int gsp_phys_rmcall_gr_lmem_base_value(void);
unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void);

/* m72: log SKED CWD/HWW register transitions during the launch poll. */
int gsp_phys_rmcall_gr_skedrec_value(void);

/* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */
int gsp_phys_rmcall_gr_inst_sys_value(void);

/* m74: send the R1 VA booking before F2's mapped-MAIN promote (image-init
 * ordering; default 0 = legacy R1-after-F2). */
int gsp_phys_rmcall_gr_r1_first_value(void);

/* m76: send the R2 #1073/#1075/#1077 lane tail after the mode-35 lane
 * enable (TSG preempt bWait, device-sched off, perf boost; default 0). */
int gsp_phys_rmcall_gr_lane_tail_value(void);

/* m78: skip lane-enable controls by bit (1=#1059 CILP, 2=#1061 timeslice,
 * 4=#1063 device-sched); default 0 = send all. */
int gsp_phys_rmcall_gr_lane_skip_value(void);

/* m83: mode-35 error notifier as host sysmem so the RM's channel-error
 * records are readable (default 0 = server FBMEM). */
int gsp_phys_rmcall_gr_notifier_sys_value(void);

/* m84: stage-4 CE read-back of the GR context image header + subctx header
 * (default 0). */
int gsp_phys_rmcall_gr_img_probe_value(void);

/*
 * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
 * 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
 * behaviour: the client owns page tables, VA bookings and engine-context
 * commitment, which is why every host GR promote failed or wedged).
 * 1 = SHARED_MANAGEMENT: the ladder still supplies and commits its root PDB
 * via SET_PAGE_DIRECTORY (legal for shared management, gpu_vaspace.c:3108)
 * while the RM owns the lower page tables and maps/commits engine contexts
 * itself, like the normal KMD client (nv_gpu_ops.c:2495). Read at the
 * stage-3 VAS alloc; a mid-session change is ignored.
 */
bool gsp_phys_rmcall_vas_shared_value(void);

/*
 * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder
 * support). The mdev is registered with vfio_register_emulated_iommu_dev():
 * its VFIO type-1 container keeps the guest DMA mappings privately, so
 * iommu_get_domain_for_dev() returns NULL there and the original
 * IOMMU-domain path cannot resolve a GPA at runtime. The authoritative
 * translation is the guest RAM backing file configured through debugfs
 * gsp-phys-guest-ram (QEMU memory-backend-file, share=yes): the range's
 * file page is pinned with read_cache_page() and page_to_phys() gives the
 * host PA. Returns 0 and stores the PA on success; -ENODEV with no active
 * mdev, -ENOENT when the file page cannot be read or the GPA is not mapped
 * by the fallback IOMMU path, -ERANGE when the result is not a valid RAM
 * PFN (so a caller never treats MMIO/dead space as RAM), and -EINVAL for a
 * NULL output. The caller must hold no guest or rm lock; this helper takes
 * nv_vgpu_driver.lock only for the mdev lookup and guest_ram_lock across
 * the range walk/read.
 */
int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out);

/*
 * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One
 * shared file per module cycle, up to four GPA->file-offset ranges keyed by
 * gpa_base. The first configure opens the file; the same path adds/replaces
 * ranges, a different path is refused -EBUSY, and the ranges are dropped and
 * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The
 * state accessor is lock-safe and writes "active=0" or the active path,
 * range count and per-range gpa/off/size into the caller's buffer.
 */
int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base,
				 u64 file_offset, u64 size);
void gsp_phys_guest_ram_clear(void);
bool gsp_phys_guest_ram_active(void);
void gsp_phys_guest_ram_state(char *buf, size_t len);

/*
 * Domain shape of the physical (claimed) device, as the P6a engine path
 * needs it: the module programs raw host PAs into the CE page tables, so the
 * engine may only run when the physical device's DMA addresses ARE host PAs.
 * Returns 1 when the claimed device's IOMMU domain is
 * IOMMU_DOMAIN_IDENTITY, 0 when another domain type, and -1 when the device
 * is not claimed or has no domain (either way the caller must refuse).
 */
int gsp_phys_dma_domain_identity(void);

#endif /* GSP_PHYS_GPU_H */