diff options
Diffstat (limited to 'kernel/gsp_sema_service.h')
| -rw-r--r-- | kernel/gsp_sema_service.h | 367 |
1 files changed, 367 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index c1dd763..f2a40ee 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -31,6 +31,7 @@ struct sema_channel_desc { u32 h_object; u32 engine_type; u32 cid; /* RM channel id (the doorbell token vector) */ + u32 h_vaspace; /* r835: VAS handle the channel was allocated in */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */ @@ -44,6 +45,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); /* + * r616 diag: manually write a channel's error notifier (engine-exception + * shape) and post its channel event, to test whether a refused launch's + * hung client can be ended by a fault-conversion edge. h_object 0 = every + * valid channel of the device. Debug only. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object); + +/* * Release every piece of service state that belongs to one mdev before its * guest session disappears: drop the mdev's channels (per-vgpu session end), * clear a release hold that names it (never dereference a removed device), @@ -82,6 +91,54 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r820: a driver-declared kernel-buffer translation from an + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entry (gpuVirtAddr/gpuPhysAddr/size/ + * physAttr of one context buffer). These buffers - the nvidia-push + * pushbuffer, the progress-tracker segment, the FECS/patch/access-map + * blocks - are kernel-owned, so the userspace pin scanner cannot cover + * them and the page-table walk can answer with a stale/wrong page + * (r817 Xorg GL init: the walk resolved the 64 KB pushbuffer VA to a + * zero page while the promote entry named the real one). The sema + * service consults these records before the learned cache and the walks + * in svc_lookup_trusted; newest statement per VA wins. VA-only bookings + * (zero pa/size) and bNonmapped entries (va=0) are ignored. Records are + * session state, cleared by svc_session_flush. + */ +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id); +/* + * SESSION 153-CONTINUED-95: latest PROMOTE_CTX statement per + * (engine, bufferId), kept for the launch-time CTX-BIND receipt. MAIN is + * bufferId 0, PATCH bufferId 2 (NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_*). + * The note side never fails; the get side returns -ENOENT until the guest + * driver has promoted that buffer. + */ +void gsp_sema_service_ctx_promote_note(u32 engine, u32 buffer_id, u64 pa, + u64 va, u64 size, u32 phys_attr, + u32 b_init); +int gsp_sema_service_ctx_promote_get(u32 engine, u32 buffer_id, u64 *pa, + u64 *va, u64 *size, u32 *phys_attr, + u32 *b_init); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); +/* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state + * (direct-map base, content-scanned tables) on the svc thread. Client + * session flushes keep that state - see svc_session_flush. */ +void gsp_sema_service_guest_reset(void); + +/* + * ISSUE-0002 aid: force a client-session flush that executes even while + * channels are live (see addendum 8 and the sema_force_flush parameter). + */ +void gsp_sema_service_sess_flush_force(void); + int gsp_sema_service_start(void); void gsp_sema_service_stop(void); int gsp_sema_service_probe(u64 gpu_va, u32 count); @@ -89,7 +146,11 @@ int gsp_sema_service_walk_probe(u64 gpu_va); int gsp_sema_service_dump_channels(void); int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap); int gsp_sema_service_dump_vidmem(u64 addr, u64 len); +int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa); +int gsp_sema_service_xw_comp(u64 comp_pa); int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits); +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa); +int gsp_sema_service_chan_rewind(u32 h_object); int gsp_sema_service_counter_add(u32 h_object, u64 pa); void gsp_sema_service_doorbell(u32 token); @@ -187,6 +248,16 @@ void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, u64 *refused, u64 *timeout, u64 *fault); /* + * SESSION 153-CONTINUED-94: the first large beef-channel push of the boot, + * preserved verbatim for the gsp-phys-init-push debugfs dump (the client's + * pushbuffer is transient in guest RAM). Returns the buffer pointer and + * fills len/h/va; lmem_seen counts any SET_SHADER_LOCAL_MEMORY_A..D + * (0x790..0x79c) runs seen in beef pushes so far (0 = the GL client never + * emitted the download-state local-memory block). + */ +const u8 *gsp_sema_init_cap_get(u32 *len, u32 *h, u64 *va, u32 *lmem_seen); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). @@ -247,6 +318,7 @@ extern u64 gsp_sema_copy_dst2; /* CE pending-entry grace window in ms before the release fallback fires. */ extern unsigned gsp_sema_ce_grace_ms; +extern u32 gsp_phys_gr_peek; /* * Bounded work-before-release hold (ms) for an unreadable CE entry whose @@ -254,6 +326,8 @@ extern unsigned gsp_sema_ce_grace_ms; * retry probe). Absolute deadline per entry, never reset; 0 disables. */ extern unsigned gsp_sema_push_hold_ms; +/* SESSION 153-CONTINUED-54: GR setup-push (MME-setup / >=12 KB) hold. */ +extern unsigned gsp_sema_gr_setup_hold_ms; /* * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before @@ -263,4 +337,297 @@ extern unsigned gsp_sema_push_hold_ms; */ extern unsigned gsp_sema_gr_grace_ms; +/* + * GRFIX FIX1b (r213): absolute per-launch bound (ms) for the GR + * data-stalled hold. Keyed by the launch push VA (slot indices are not + * stable across pin-batch rewinds); first hold fixes the deadline, nothing + * re-arms it, and when it passes the launch proceeds (the pre-8b058884 + * behaviour) so the guest can never wait unboundedly. + */ +extern unsigned gsp_phys_gr_hold_max_ms; + +/* + * GRFIX FIX1e (r215): master gate for the GR fill hold (default 0 = + * log-only, never hold). At 0 the fill logs dropped operand pages + * (GRFILL-DROP) and takes the historical forward path immediately; the + * GRFILL-HOLD/GRFILL-EXPIRED machinery is skipped. At 1 the page-count + * hold and its absolute bound apply. + */ +extern unsigned gsp_phys_gr_fill_hold; + +/* + * r576 DIAG: per-boot witness budget for the GR launch-decision dump + * (default 0 = off). Nonzero logs, for the first N launches whose fill saw + * dropped pages / a partial argument set or whose decision point had a + * dstpend/copy-pending verdict: the decision verdicts, the launch's + * resolved data pages (VA -> guest PA), the live deferred-write records and + * the pending CE opdef entries - the evidence needed to explain a red draw + * of the readback lottery. + */ +extern unsigned gsp_phys_gr_dec_log_max; +extern unsigned gsp_sema_walk_trace; + +/* + * r365: execute fill-shaped CE transfers to VIDMEM-window VAs (the guest's + * page-table writes). Off by default; see the definition in the .c. + */ +extern bool gsp_sema_winfill; + +/* + * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD + * words but built no launch candidate; 0 disables the hold. + */ +extern unsigned gsp_sema_gr_nocap_hold_ms; + +/* + * m165 (D-F2a): skip the post-registration pool hold when a verified + * injected pin exists for the VA (default on); 0 restores the blanket 5 s. + */ +extern bool gsp_sema_pool_hold_fresh; + +/* + * m169 (AGENT-F R1): hold device-destination copies translated only by a + * walk/RAM scan until a pin confirms (default on). + */ +extern bool gsp_sema_copy_dst_pin_guard; +/* r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms). 0 restores the per-page windows (pre-r609). */ +extern unsigned gsp_sema_copy_dst_age_ms; +extern bool gsp_sema_wait_gate; +/* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the + * emulated launch completion latency for copy-after-launch ordering. */ +extern unsigned gsp_sema_wait_hold_ms; +/* r481: verbatim method-stream dump for the first N large GR pushes + * (default 0 off). Diagnostic only; no behaviour change. */ +extern unsigned gsp_sema_push_raw; + +/* + * r384 (guest boot-corruption lottery): fill-destination guards (default + * on). Large virtual fills whose only authority is the walk/scan resolver + * are held through a short bounded window while their VA is demanded + * through the pin channel (fail-open after gsp_sema_fill_guard_ms); zero + * fills targeting a page-table-shaped guest page are refused loudly. + */ +extern bool gsp_sema_fill_guard; +extern unsigned gsp_sema_fill_guard_ms; + +/* + * r231 READ-ORDER guard (default 0: witness only). Defer a device-src copy + * whose source VA has a pending device-dst write hold (READ-ORDER-WIT) so + * the read re-decodes after the write resolves. Opt-in pending an A/B. + */ +extern bool gsp_sema_read_order_guard; + +/* + * COPYWIT fail4: per-boot line budget for the sema-bigcopy resolution + * witnesses (svc_copy's large-copy VA -> pa proof). Raise it for a + * discrimination cycle so the failing clients' H2D/D2H resolutions are + * visible; 0 disables the lines. + */ +extern unsigned gsp_sema_bigcopy_log_max; + +/* + * F1-b: authorship gate for the GR completion release the forwarded launch's + * poll waits on. When on (default), a RAM-sweep release whose VA equals the + * in-flight forward request's gr_rel_va is deferred (skipped, not applied) + * while the hardware copy is still in flight, so the poll can only break on + * an engine-authored word; the deferred release is re-seen by later sweep + * passes and the hold/fallback paths apply it if the hardware copy fails. + * 0 restores the unconditional sweep apply. + */ +extern bool gsp_sema_gr_swrel_gate; + +/* + * relfix (agent-A FIX A): rewrite-time GR relocation resolver. Resolves one + * device VA through the same trusted source chain the GR fill's per-page + * resolution uses (ipin -> self pin -> learned cache -> discovered map -> + * published vaspaces -> discovered directory), then the m180 ledger in the + * fill's two roles (fallback when nothing else answered; m171 landing + * preference above the ledger floor when it disagrees). The result is + * converted to a host page the caller can copy/map. Deliberately read-only + * w.r.t. the resolver tables - no pin or cache stores, no directory + * discovery, no targeted RAM scan: the caller is the GR work item, not the + * service thread, and must not write service state or block on a full-RAM + * scan. The page-table walks may sleep in vfio_dma_rw exactly like the + * fill's walks; callers run in process context (the GR work item, holding + * gsp_phys_rm_lock) with no spinlocks held. A live forwarded GR request + * (svc_fwd_req.vgpu) is required. A successful return (0) means the VA is a + * translated page of that request's client - the caller may act on it; a + * miss returns -ENOENT and the caller must change nothing. + * + * `stage` carries the per-source diagnostic chain of THIS call + * ("ipin:-2,pin:-2,...,ledger:-2," with the resolving source ending in + * ":0,") and `stage_rc` the overall result (0 on success, else the miss + * errno). relfix PART 3: the booter's RELOC-GAP-STAGE line prints them for + * words that stayed unrelocated, so the failing source is on the record. + */ +struct gsp_sema_gr_reloc { + u64 pa; /* resolved guest physical page base */ + u32 ap; /* aperture: 1 = SYS (guest RAM), 2 = FB (vidmem) */ + u64 hpa; /* host physical address of that guest page */ + char stage[96]; /* per-attempt "name:rc," source chain (diagnostic) */ + int stage_rc; /* 0 on success; else the failure errno */ +}; + +int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); + +/* + * SESSION 153-CONTINUED-59: record one VA (page-aligned internally) in the + * pin-demand set exposed at /sys/kernel/debug/vgsp/sema_setup_demand, so + * pin_gr_stream demands its page in the guest. The GR launch builder calls + * this when a window/attr operand page cannot be resolved at build time + * (the r1104/r1105 MISS -> MMU-fault class); the demand gives the next + * forward a resolvable page instead of a lane-killing raw guest VA. + */ +void gsp_sema_demand_note(u64 va); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude accessor for the GR + * launch builder. Returns the captured setup runs (C597 MME family + * 0x114..0x124/0x234/0x550..0x568/0x3400..0x34fc, verbatim, first- + * encounter order), the word count in *n and the capture generation in + * *gen. The builder preplays the words once per generation, ahead of the + * first 3D window replay, so the ladder context's MME has its macro + * program before CALL_MME_MACRO runs. Reading the generation before and + * after the copy detects a concurrent append (the service thread only + * appends; a mismatch just defers emission to the next launch). *n is 0 + * until the client's setup push has been decoded. + */ +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen); + +/* + * SESSION 153-CONTINUED-57: is the captured MME prelude mature enough to + * launch against - i.e. does it hold the client's macro program (>= 64 + * captured 0x114 code words)? The r1094 premature-emission poison + * (MME ILLEGAL_OPCODE at +5 ms, capture still at 9 scratch words) is + * gated on this: the sema forward path withholds beef launches while it + * reads false, bounded by sema_gr_prelude_gate_ms. + */ +bool gsp_sema_mme_prelude_mature(void); + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude accessor (C597 + * families 0x1100..0x11fc and 0x2000..0x205c, verbatim, first-encounter + * order, deduped). Same contract as gsp_sema_mme_prelude_get: preplay once + * per generation ahead of the first 3D window replay. The runs carry the + * pipeline program binds whose SPH the draw shader validates, so the + * builder also registers the program VAs (gsp_sema_attr_prelude_ptrs) in + * the launch page set to make the address relocation land. + */ +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen); +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n); + +/* + * SESSION 153-CONTINUED-62: surface (render-target) setup accessor. Holds + * the last captured C597 colour-target bind run (INC 800+j*40, [upper, + * lower, w, h, fmt, blksz], fmt 0xe6, 2..4096 pages) from any trusted + * push. The GR launch builder replays it - aliased/relocated exactly like + * the window - ahead of a 3D window that carries no usable render-target + * bind of its own; without it the replayed draw rasterizes into an + * unbound target and stores nothing (the observed 0/N retire). + */ +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen); + +/* + * SESSION 153-CONTINUED-63: surface geometry for render-target tracking + * (base VA + page count of the captured colour target). + */ +bool gsp_sema_surf_va_get(u64 *va, u32 *npages); + +/* + * SESSION 153-CONTINUED-63: the fill's completed render-target tracking run + * (base VA + page count), true only when every page of the surface resolved + * into contiguous data recs this pass. The booter prefers these recs over + * the identity surface alias when rewriting the target's operand pair, so + * the engine's stores land in the bounce (witness + copy-out) instead of + * the guest pages directly. + */ +bool gsp_sema_gr_surf_recs(u64 *va, u32 *n); + +/* + * SESSION 153-CONTINUED-63: the captured MME START-ADDRESS RAM table + * (macro slot -> IR instruction address; present-bitmask returned, entries + * copied into `entries`). The prelude buffer is capped at 512 words, so + * the SA table is recorded independently; the GR launch builder completes + * the SA RAM it replays and stubs every called-but-unknown slot with a + * harmless end-noop (the r1116 0x238c wall: SA[0x22] = 0 on a fresh + * context made CALL_MME_MACRO(0x22) jump to IR 0 and the slot-0 macro's + * emission trapped the lane). + */ +u64 gsp_sema_mme_sa_get(u32 *entries, u32 n); + +/* + * SESSION 153-CONTINUED-87: the SPH VERSION of the live program images + * (sema_gl_vs/ps_hex blobs), so the GR launch builder emits a + * SET_SPH_VERSION window that MATCHES the images the engine will fetch + * (the client's own init push says 4/4, but the r950 gate proved the V3 + * images execute; a 4/4 window over a V3 SPH reproduces the + * SPH9|SPH11|SPH18 fault). 0 when no image is loaded. + */ +int gsp_sema_gl_sph_blob_version(void); + +/* + * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED + * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places + * its end-noop macro stub at this address (aligned past the replayed + * program) so the stub cannot clobber a captured macro body; 0 until the + * first 0x114 upload enters the prelude buffer. + */ +u32 gsp_sema_mme_ir_end_get(void); + +/* + * SESSION 153-CONTINUED-70: IR code words captured so far (the maturity + * gate's measure; printed by the MME-PRE-DUMP for the offline sim). + */ +u32 gsp_sema_mme_code_words_get(void); + +/* + * SESSION 153-CONTINUED-68: is the IR instruction at address `pc` present + * in the replay buffer (i.e. was it uploaded by a captured run that was + * NOT dropped as a duplicate/over-cap)? The launch builder stubs every + * called macro slot whose SA entry sits outside this set, so a truncated + * or generation-mismatched capture becomes a no-op instead of a macro + * fetch into never-replayed IR (Graphics Exception: MISSING_MACRO_DATA). + */ +bool gsp_sema_mme_ir_covered(u32 pc); + +/* + * r378 write guards. Both re-validate a guest-memory write target that was + * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's + * per-page resolution) against the CURRENT resolution before the bytes land, + * and return false when a positive disagreement proves the target stale: + * + * gsp_sema_gr_write_guard(va, hpa, path): GR backend pages. A fresh + * injected pin is checked first (the r372/r376 pin-wins rule - the guest's + * newest statement), then the trusted resolver chain for a pin-less VA. + * The submit-to-copy-out window is seconds wide (the GR poll), and a guest + * buffer freed/re-homed in it would otherwise receive a full page of + * another buffer's content (the boot-corruption class). The landing table + * is deliberately not the authority: our own launches feed it, so it would + * confirm the target by construction. + * + * gsp_sema_write_guard(va, pa, ap, path): service-side SYS/FB targets. A + * fresh injected pin is the guest's newest statement about the VA; a pin + * that disagrees with the recorded target means the page moved (the + * r372/r376 pin-wins rule), so the stale write is skipped. + * + * Both fail OPEN when no current answer exists (the recorded landing is then + * the write authority) and log every skip (bounded, 256 lines) with the path + * and the VA/target/current values. + */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path); +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path); + +/* + * r391 diagnostic (sum=0 class): resolve a GR data VA through the landing + * table (the exact source the D2H reader's ledger-first branch uses) and + * return the resolution plus the first 8 bytes read through it. The GR + * copy-out prints it next to the write target it used, so a launch/readback + * disagreement (the write went to another page/aperture than the reader + * resolves) is one line instead of a forensics session. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8); +u8 *gsp_sema_gr_shadow_base(void); + #endif /* _GSP_SEMA_SERVICE_H_ */ |