/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _GSP_SEMA_SERVICE_H_
#define _GSP_SEMA_SERVICE_H_

/*
 * Emulated-GPU progress service: discovers the guest's GPU-VA -> guest-PA
 * mapping and applies the copy-engine semaphore releases found in mapped
 * pages, so that the guest's UVM progress waits complete.
 *
 * See gsp_sema_service.c for the full description.
 */

/*
 * A guest channel, registered when the driver allocates it. The service uses
 * the channel's GPFIFO + USERD to notice submissions as they happen (instead
 * of discovering the pushed work by scanning guest RAM) and to run the pushed
 * work promptly.
 *
 * All addresses are guest-physical except gpfifo_va, which is the GPU VA of
 * the channel's GPFIFO ring (translated through the discovered VA->PA map).
 *
 * vgpu names the owning mdev (guest session). Two guests' stock allocators
 * choose the same object handles, so the channel table is keyed by
 * (vgpu, h_object): a registration can never overwrite another guest's slot,
 * and ranks/session bookkeeping are computed per guest.
 */
struct nv_vgpu_device;

struct sema_channel_desc {
	struct nv_vgpu_device *vgpu;	/* owning mdev (guest session) */
	u32 h_object;
	u32 engine_type;
	u32 cid;		/* RM channel id (the doorbell token vector) */
	u64 gpfifo_va;
	u32 gpfifo_entries;
	u64 userd_pa;		/* NvA06FControl: GPGet @0x88, GPPut @0x8c */
	u32 userd_as;		/* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */
	u64 instance_pa;
	u64 notifier_pa;
	u32 notifier_as;	/* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */
};

void gsp_sema_service_channel_add(const struct sema_channel_desc *desc);
void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object);

/*
 * Release every piece of service state that belongs to one mdev before its
 * guest session disappears: drop the mdev's channels (per-vgpu session end),
 * clear a release hold that names it (never dereference a removed device),
 * and free its per-guest session record. Called from the mdev cleanup path
 * after the device's emulation thread stopped and before the mdev struct can
 * be freed. No-op for a vgpu that never registered a channel.
 */
void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu);

/*
 * Register the guest PMA scrubber's channel-buffer words (the CeUtils
 * finish payload the guest's scrub wait polls, and the channel's host
 * semaphore). The PAs come from the guest scanner (vmallocinfo + kcore):
 * the guest RM kernel channel lives in a GPU VA space that collides
 * numerically with the client's CE window in the global translation map,
 * so it cannot be resolved through that map.
 */
int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa);

/*
 * A GPU VA space whose top-level page directory the driver has published (the
 * driver builds its own page tables for the address spaces it owns, then hands
 * the directory's physical address to the GPU through SET_PAGE_DIRECTORY).
 *
 * pdb is the directory base in the aperture selected by b_vidmem (false =
 * guest RAM, true = emulated vidmem); num_entries is the root entry count and
 * fixes the number of VA bits the root covers.
 */
struct sema_vaspace_desc {
	u32 h_vaspace;
	u64 pdb;
	u32 num_entries;
	bool b_vidmem;
};

void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
void gsp_sema_service_vaspace_del(u32 h_vaspace);
/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state
 * (direct-map base, content-scanned tables) on the svc thread. Client
 * session flushes keep that state - see svc_session_flush. */
void gsp_sema_service_guest_reset(void);

int  gsp_sema_service_start(void);
void gsp_sema_service_stop(void);
int  gsp_sema_service_probe(u64 gpu_va, u32 count);
int  gsp_sema_service_walk_probe(u64 gpu_va);
int  gsp_sema_service_dump_channels(void);
int  gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap);
int  gsp_sema_service_dump_vidmem(u64 addr, u64 len);
int  gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits);
int  gsp_sema_service_counter_add(u32 h_object, u64 pa);
void gsp_sema_service_doorbell(u32 token);

/*
 * P5 wave-1 shadow work lifetime hooks. The backend release path must call
 * gsp_sema_service_work_quiesce() before it frees the ladder buffers: it
 * pauses new shadow requests and waits (bounded by the submit's 200 ms poll
 * cap) for the in-flight one to finish, so no poll ever touches freed memory.
 * gsp_sema_service_work_resume() re-enables requests when a new backend
 * session starts.
 */
void gsp_sema_service_work_quiesce(void);
void gsp_sema_service_work_resume(void);

/*
 * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest
 * line: guest-eligible requests seen by the pump, guest requests whose
 * hardware copy failed (wave 1: fell back to the ladder copy; with
 * gsp_phys_work_defer_release the service applies the held releases instead),
 * and verified hardware copies. Plain reads of the service counters; safe
 * from the debugfs show path.
 */
void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok);

/*
 * P6b wave-1 candidate accounting for the same work_guest line: how many
 * forwarding requests found no eligible captured copy, and of those that did,
 * the operand aperture mix (SYS/SYS, mixed, VID/VID), plus guest-work
 * resolution failures on the work item. All zero while the feature is off.
 */
void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed,
				     u64 *vid_vid, u64 *resolve_fail);

/*
 * P6b wave-1.5/wave-2 deferred-release and copy counters for the
 * gsp-phys-state work_guard line: copy_skipped (the software copy was
 * withheld because hardware-primary authorship is armed; always 0 when
 * gsp_phys_work_hw_primary=0), copy_soft (software fallback copies the
 * service thread performed before applying releases), copy_fail (fallback
 * copy attempts that failed), releases deferred until the hardware copy
 * completed, holds applied on completion (including a hold whose channel
 * vanished after a successful copy), holds applied because the copy failed /
 * timed out / forwarding was aborted, releases currently held (gauge),
 * deferred records dropped by the record bound, holds engaged, holds that hit
 * the timeout, and holds dropped unresolved after their channel vanished (no
 * successful copy had completed). All zero while the feature is off. Plain
 * reads, safe from the debugfs show path.
 */
void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
				     u64 *copy_fail, u64 *deferred,
				     u64 *applied, u64 *fallback,
				     u64 *pending, u64 *overflow,
				     u64 *engaged, u64 *timeouts,
				     u64 *canceled);

/*
 * P6c completion-relay counters for the gsp-phys-state work_relay line:
 * requested relay selections armed on a deferred guest request, relays whose
 * engine-authored guest semaphore was verified (ok) with their software
 * applies skipped (skipped_sw), relays that fell back to the software apply
 * (fallback: engine failure/timeout/abort or a word that did not carry the
 * payload), hold re-reads that did not see the payload (verify_fail, shown as
 * hold_verify_fail in the state line, distinct from the backend compare's
 * verify_fail) and work-item semaphore HPA resolution failures (resolve_fail).
 * All zero while gsp_phys_relay=0. Plain reads, safe from the debugfs show
 * path.
 */
void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
				      u64 *skipped_sw, u64 *verify_fail,
				      u64 *resolve_fail);

/*
 * P6c C3 fault-relay delivery counters for the gsp-phys-state fault_relay
 * line: faults that matched the ladder runlistId/chid and were delivered
 * (matched), faults refused because they did not match or had no notifier
 * (unmatched), notifiers written with the channel event posted (delivered),
 * notifier writes whose event post found no event object (no_event) and
 * faults skipped because the relayed guest semaphore already carried the
 * payload (late_release). All zero while gsp_phys_fault_relay=0. Plain reads,
 * safe from the debugfs show path.
 */
void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
				      u64 *delivered, u64 *no_event,
				      u64 *late_release);

/*
 * GR (compute) forwarding counters for the gsp-phys-state work_gr line:
 * captures offered to the request path (req), launches the physical GR
 * channel completed (ok), offerings that fell back to the CUDA bridge /
 * software release (fallback), gate refusals (refused), backend timeouts
 * (timeout) and timeouts carrying a decoded engine fault (fault). All zero
 * while gsp_phys_gr_forward=0. Plain reads, safe from the debugfs show path.
 */
void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback,
				   u64 *refused, u64 *timeout, u64 *fault);

/*
 * Discover and pin GR (compute) channel rings after the CE region of the
 * channel video-memory allocation. Set false to leave GR rings untranslated
 * (the pump then only runs the CE channels).
 */
extern bool gsp_sema_gr_rings;

/*
 * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an
 * injected-pin miss, resolve a counter slot from the structural translations
 * (the driver-published page directories, the self-discovered directory and
 * the targeted rm_mem record) and author the release there instead of waiting
 * for the external pin watcher. Default on; set 0 to restore the
 * injected-pin-only behaviour for an A/B without a rebuild.
 */
extern bool gsp_sema_counter_derive;

/*
 * Acknowledge tracking-semaphore releases from CE pushes the pump could not
 * read: write the expected payload into the channel's pool semaphore so the
 * driver's wait clears immediately instead of after the content sweep.
 */
extern bool gsp_sema_fake_rel;
extern bool gsp_sema_rel_recover;

/*
 * Master diagnostic switch for the per-push/per-doorbell/per-op tracing
 * (module parameter sema_dbg). Off by default so a healthy boot leaves the
 * kernel log quiet; the watch knobs below only take effect while it is on.
 */
extern bool gsp_sema_dbg;

/*
 * Skip executing a push's collected work (CE memsets/copies) while still
 * applying its release; see the definition for the trade-off.
 */
extern bool gsp_sema_skip_ops;

/*
 * How many executed CE ops to log with full operands (diagnostic bound; the
 * default keeps a healthy run quiet, raise it to audit a data path).
 */
extern unsigned gsp_sema_ops_log;

/*
 * Data-path watch: for copies of exactly this many bytes, log the first 8
 * source/destination bytes before the copy (0 = off).
 */
extern unsigned gsp_sema_copy_watch;

/* Data-path watch: 8-byte value matched in copy src/dst and push bytes. */
extern u64 gsp_sema_copy_pat;

/* Data-path watch: dump copies landing in this page (source bytes). */
extern u64 gsp_sema_copy_dst;

/* Second destination window for the same watch (0 = off). */
extern u64 gsp_sema_copy_dst2;

/* CE pending-entry grace window in ms before the release fallback fires. */
extern unsigned gsp_sema_ce_grace_ms;

/*
 * Bounded work-before-release hold (ms) for an unreadable CE entry whose
 * translation/pins typically land 0.5-2 s late (observed via the deferred
 * retry probe). Absolute deadline per entry, never reset; 0 disables.
 */
extern unsigned gsp_sema_push_hold_ms;

/*
 * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before
 * the pump consumes it as stale. The GR push pages are pinned from guest
 * truth a moment after the channels appear; the window lets those pins land
 * so the releases inside the pushes are applied.
 */
extern unsigned gsp_sema_gr_grace_ms;

/*
 * GRFIX FIX1b (r213): absolute per-launch bound (ms) for the GR
 * data-stalled hold. Keyed by the launch push VA (slot indices are not
 * stable across pin-batch rewinds); first hold fixes the deadline, nothing
 * re-arms it, and when it passes the launch proceeds (the pre-8b058884
 * behaviour) so the guest can never wait unboundedly.
 */
extern unsigned gsp_phys_gr_hold_max_ms;

/*
 * GRFIX FIX1e (r215): master gate for the GR fill hold (default 0 =
 * log-only, never hold). At 0 the fill logs dropped operand pages
 * (GRFILL-DROP) and takes the historical forward path immediately; the
 * GRFILL-HOLD/GRFILL-EXPIRED machinery is skipped. At 1 the page-count
 * hold and its absolute bound apply.
 */
extern unsigned gsp_phys_gr_fill_hold;

/*
 * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD
 * words but built no launch candidate; 0 disables the hold.
 */
extern unsigned gsp_sema_gr_nocap_hold_ms;

/*
 * m165 (D-F2a): skip the post-registration pool hold when a verified
 * injected pin exists for the VA (default on); 0 restores the blanket 5 s.
 */
extern bool gsp_sema_pool_hold_fresh;

/*
 * m169 (AGENT-F R1): hold device-destination copies translated only by a
 * walk/RAM scan until a pin confirms (default on).
 */
extern bool gsp_sema_copy_dst_pin_guard;

/*
 * COPYWIT fail4: per-boot line budget for the sema-bigcopy resolution
 * witnesses (svc_copy's large-copy VA -> pa proof). Raise it for a
 * discrimination cycle so the failing clients' H2D/D2H resolutions are
 * visible; 0 disables the lines.
 */
extern unsigned gsp_sema_bigcopy_log_max;

/*
 * F1-b: authorship gate for the GR completion release the forwarded launch's
 * poll waits on. When on (default), a RAM-sweep release whose VA equals the
 * in-flight forward request's gr_rel_va is deferred (skipped, not applied)
 * while the hardware copy is still in flight, so the poll can only break on
 * an engine-authored word; the deferred release is re-seen by later sweep
 * passes and the hold/fallback paths apply it if the hardware copy fails.
 * 0 restores the unconditional sweep apply.
 */
extern bool gsp_sema_gr_swrel_gate;

/*
 * relfix (agent-A FIX A): rewrite-time GR relocation resolver. Resolves one
 * device VA through the same trusted source chain the GR fill's per-page
 * resolution uses (ipin -> self pin -> learned cache -> discovered map ->
 * published vaspaces -> discovered directory), then the m180 ledger in the
 * fill's two roles (fallback when nothing else answered; m171 landing
 * preference above the ledger floor when it disagrees). The result is
 * converted to a host page the caller can copy/map. Deliberately read-only
 * w.r.t. the resolver tables - no pin or cache stores, no directory
 * discovery, no targeted RAM scan: the caller is the GR work item, not the
 * service thread, and must not write service state or block on a full-RAM
 * scan. The page-table walks may sleep in vfio_dma_rw exactly like the
 * fill's walks; callers run in process context (the GR work item, holding
 * gsp_phys_rm_lock) with no spinlocks held. A live forwarded GR request
 * (svc_fwd_req.vgpu) is required. A successful return (0) means the VA is a
 * translated page of that request's client - the caller may act on it; a
 * miss returns -ENOENT and the caller must change nothing.
 *
 * `stage` carries the per-source diagnostic chain of THIS call
 * ("ipin:-2,pin:-2,...,ledger:-2," with the resolving source ending in
 * ":0,") and `stage_rc` the overall result (0 on success, else the miss
 * errno). relfix PART 3: the booter's RELOC-GAP-STAGE line prints them for
 * words that stayed unrelocated, so the failing source is on the record.
 */
struct gsp_sema_gr_reloc {
	u64 pa;		/* resolved guest physical page base */
	u32 ap;		/* aperture: 1 = SYS (guest RAM), 2 = FB (vidmem) */
	u64 hpa;	/* host physical address of that guest page */
	char stage[96];	/* per-attempt "name:rc," source chain (diagnostic) */
	int stage_rc;	/* 0 on success; else the failure errno */
};

int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out);

#endif /* _GSP_SEMA_SERVICE_H_ */