diff options
Diffstat (limited to 'kernel/gsp_sema_service.h')
| -rw-r--r-- | kernel/gsp_sema_service.h | 218 |
1 files changed, 218 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 3765cd2..7efc2f3 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -31,6 +31,7 @@ struct sema_channel_desc { u32 h_object; u32 engine_type; u32 cid; /* RM channel id (the doorbell token vector) */ + u32 h_vaspace; /* r835: VAS handle the channel was allocated in */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */ @@ -44,6 +45,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); /* + * r616 diag: manually write a channel's error notifier (engine-exception + * shape) and post its channel event, to test whether a refused launch's + * hung client can be ended by a fault-conversion edge. h_object 0 = every + * valid channel of the device. Debug only. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object); + +/* * Release every piece of service state that belongs to one mdev before its * guest session disappears: drop the mdev's channels (per-vgpu session end), * clear a release hold that names it (never dereference a removed device), @@ -81,11 +90,55 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); + +/* + * r820: a driver-declared kernel-buffer translation from an + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entry (gpuVirtAddr/gpuPhysAddr/size/ + * physAttr of one context buffer). These buffers - the nvidia-push + * pushbuffer, the progress-tracker segment, the FECS/patch/access-map + * blocks - are kernel-owned, so the userspace pin scanner cannot cover + * them and the page-table walk can answer with a stale/wrong page + * (r817 Xorg GL init: the walk resolved the 64 KB pushbuffer VA to a + * zero page while the promote entry named the real one). The sema + * service consults these records before the learned cache and the walks + * in svc_lookup_trusted; newest statement per VA wins. VA-only bookings + * (zero pa/size) and bNonmapped entries (va=0) are ignored. Records are + * session state, cleared by svc_session_flush. + */ +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id); +/* + * SESSION 153-CONTINUED-95: latest PROMOTE_CTX statement per + * (engine, bufferId), kept for the launch-time CTX-BIND receipt. MAIN is + * bufferId 0, PATCH bufferId 2 (NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_*). + * The note side never fails; the get side returns -ENOENT until the guest + * driver has promoted that buffer. + */ +void gsp_sema_service_ctx_promote_note(u32 engine, u32 buffer_id, u64 pa, + u64 va, u64 size, u32 phys_attr, + u32 b_init); +int gsp_sema_service_ctx_promote_get(u32 engine, u32 buffer_id, u64 *pa, + u64 *va, u64 *size, u32 *phys_attr, + u32 *b_init); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state * (direct-map base, content-scanned tables) on the svc thread. Client * session flushes keep that state - see svc_session_flush. */ void gsp_sema_service_guest_reset(void); +/* + * ISSUE-0002 aid: force a client-session flush that executes even while + * channels are live (see addendum 8 and the sema_force_flush parameter). + */ +void gsp_sema_service_sess_flush_force(void); + int gsp_sema_service_start(void); void gsp_sema_service_stop(void); int gsp_sema_service_probe(u64 gpu_va, u32 count); @@ -93,7 +146,11 @@ int gsp_sema_service_walk_probe(u64 gpu_va); int gsp_sema_service_dump_channels(void); int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap); int gsp_sema_service_dump_vidmem(u64 addr, u64 len); +int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa); +int gsp_sema_service_xw_comp(u64 comp_pa); int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits); +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa); +int gsp_sema_service_chan_rewind(u32 h_object); int gsp_sema_service_counter_add(u32 h_object, u64 pa); void gsp_sema_service_doorbell(u32 token); @@ -191,6 +248,16 @@ void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, u64 *refused, u64 *timeout, u64 *fault); /* + * SESSION 153-CONTINUED-94: the first large beef-channel push of the boot, + * preserved verbatim for the gsp-phys-init-push debugfs dump (the client's + * pushbuffer is transient in guest RAM). Returns the buffer pointer and + * fills len/h/va; lmem_seen counts any SET_SHADER_LOCAL_MEMORY_A..D + * (0x790..0x79c) runs seen in beef pushes so far (0 = the GL client never + * emitted the download-state local-memory block). + */ +const u8 *gsp_sema_init_cap_get(u32 *len, u32 *h, u64 *va, u32 *lmem_seen); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). @@ -251,6 +318,7 @@ extern u64 gsp_sema_copy_dst2; /* CE pending-entry grace window in ms before the release fallback fires. */ extern unsigned gsp_sema_ce_grace_ms; +extern u32 gsp_phys_gr_peek; /* * Bounded work-before-release hold (ms) for an unreadable CE entry whose @@ -258,6 +326,8 @@ extern unsigned gsp_sema_ce_grace_ms; * retry probe). Absolute deadline per entry, never reset; 0 disables. */ extern unsigned gsp_sema_push_hold_ms; +/* SESSION 153-CONTINUED-54: GR setup-push (MME-setup / >=12 KB) hold. */ +extern unsigned gsp_sema_gr_setup_hold_ms; /* * Grace window (ms) an untranslatable GR GPFIFO entry is left pending before @@ -284,6 +354,17 @@ extern unsigned gsp_phys_gr_hold_max_ms; * hold and its absolute bound apply. */ extern unsigned gsp_phys_gr_fill_hold; + +/* + * r576 DIAG: per-boot witness budget for the GR launch-decision dump + * (default 0 = off). Nonzero logs, for the first N launches whose fill saw + * dropped pages / a partial argument set or whose decision point had a + * dstpend/copy-pending verdict: the decision verdicts, the launch's + * resolved data pages (VA -> guest PA), the live deferred-write records and + * the pending CE opdef entries - the evidence needed to explain a red draw + * of the readback lottery. + */ +extern unsigned gsp_phys_gr_dec_log_max; extern unsigned gsp_sema_walk_trace; /* @@ -309,10 +390,17 @@ extern bool gsp_sema_pool_hold_fresh; * walk/RAM scan until a pin confirms (default on). */ extern bool gsp_sema_copy_dst_pin_guard; +/* r609: after a device-dst guard window expires WITHOUT a pin, skip the + * window for further first-write device-dst pages of the same channel for + * this long (ms). 0 restores the per-page windows (pre-r609). */ +extern unsigned gsp_sema_copy_dst_age_ms; extern bool gsp_sema_wait_gate; /* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the * emulated launch completion latency for copy-after-launch ordering. */ extern unsigned gsp_sema_wait_hold_ms; +/* r481: verbatim method-stream dump for the first N large GR pushes + * (default 0 off). Diagnostic only; no behaviour change. */ +extern unsigned gsp_sema_push_raw; /* * r384 (guest boot-corruption lottery): fill-destination guards (default @@ -385,6 +473,136 @@ struct gsp_sema_gr_reloc { int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); /* + * SESSION 153-CONTINUED-59: record one VA (page-aligned internally) in the + * pin-demand set exposed at /sys/kernel/debug/vgsp/sema_setup_demand, so + * pin_gr_stream demands its page in the guest. The GR launch builder calls + * this when a window/attr operand page cannot be resolved at build time + * (the r1104/r1105 MISS -> MMU-fault class); the demand gives the next + * forward a resolvable page instead of a lane-killing raw guest VA. + */ +void gsp_sema_demand_note(u64 va); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude accessor for the GR + * launch builder. Returns the captured setup runs (C597 MME family + * 0x114..0x124/0x234/0x550..0x568/0x3400..0x34fc, verbatim, first- + * encounter order), the word count in *n and the capture generation in + * *gen. The builder preplays the words once per generation, ahead of the + * first 3D window replay, so the ladder context's MME has its macro + * program before CALL_MME_MACRO runs. Reading the generation before and + * after the copy detects a concurrent append (the service thread only + * appends; a mismatch just defers emission to the next launch). *n is 0 + * until the client's setup push has been decoded. + */ +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen); + +/* + * SESSION 153-CONTINUED-57: is the captured MME prelude mature enough to + * launch against - i.e. does it hold the client's macro program (>= 64 + * captured 0x114 code words)? The r1094 premature-emission poison + * (MME ILLEGAL_OPCODE at +5 ms, capture still at 9 scratch words) is + * gated on this: the sema forward path withholds beef launches while it + * reads false, bounded by sema_gr_prelude_gate_ms. + */ +bool gsp_sema_mme_prelude_mature(void); + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude accessor (C597 + * families 0x1100..0x11fc and 0x2000..0x205c, verbatim, first-encounter + * order, deduped). Same contract as gsp_sema_mme_prelude_get: preplay once + * per generation ahead of the first 3D window replay. The runs carry the + * pipeline program binds whose SPH the draw shader validates, so the + * builder also registers the program VAs (gsp_sema_attr_prelude_ptrs) in + * the launch page set to make the address relocation land. + */ +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen); +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n); + +/* + * SESSION 153-CONTINUED-62: surface (render-target) setup accessor. Holds + * the last captured C597 colour-target bind run (INC 800+j*40, [upper, + * lower, w, h, fmt, blksz], fmt 0xe6, 2..4096 pages) from any trusted + * push. The GR launch builder replays it - aliased/relocated exactly like + * the window - ahead of a 3D window that carries no usable render-target + * bind of its own; without it the replayed draw rasterizes into an + * unbound target and stores nothing (the observed 0/N retire). + */ +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen); + +/* + * SESSION 153-CONTINUED-63: surface geometry for render-target tracking + * (base VA + page count of the captured colour target). + */ +bool gsp_sema_surf_va_get(u64 *va, u32 *npages); + +/* + * SESSION 153-CONTINUED-63: the fill's completed render-target tracking run + * (base VA + page count), true only when every page of the surface resolved + * into contiguous data recs this pass. The booter prefers these recs over + * the identity surface alias when rewriting the target's operand pair, so + * the engine's stores land in the bounce (witness + copy-out) instead of + * the guest pages directly. + */ +bool gsp_sema_gr_surf_recs(u64 *va, u32 *n); + +/* + * SESSION 153-CONTINUED-63: the captured MME START-ADDRESS RAM table + * (macro slot -> IR instruction address; present-bitmask returned, entries + * copied into `entries`). The prelude buffer is capped at 512 words, so + * the SA table is recorded independently; the GR launch builder completes + * the SA RAM it replays and stubs every called-but-unknown slot with a + * harmless end-noop (the r1116 0x238c wall: SA[0x22] = 0 on a fresh + * context made CALL_MME_MACRO(0x22) jump to IR 0 and the slot-0 macro's + * emission trapped the lane). + */ +u64 gsp_sema_mme_sa_get(u32 *entries, u32 n); + +/* + * SESSION 153-CONTINUED-101: the client's own SET_MME_SHADOW_SCRATCH(0x18) + * write (method 0x3460) - mmeCodeOGL_C597_SETUP_FIXED_CONST (slot 0) reads + * shadow scratch 0x18/0xd18 via STATE and emits it as the fixed-function + * constant-buffer selector (SET_CONSTANT_BUFFER_SELECTOR_B/C). Returns + * true (and stores the value exactly as pushed: fixed-CB address >> 8) + * once any decoded client push carried the write. false until then. + */ +bool gsp_sema_mme_scratch18_get(u32 *val); + +/* + * SESSION 153-CONTINUED-87: the SPH VERSION of the live program images + * (sema_gl_vs/ps_hex blobs), so the GR launch builder emits a + * SET_SPH_VERSION window that MATCHES the images the engine will fetch + * (the client's own init push says 4/4, but the r950 gate proved the V3 + * images execute; a 4/4 window over a V3 SPH reproduces the + * SPH9|SPH11|SPH18 fault). 0 when no image is loaded. + */ +int gsp_sema_gl_sph_blob_version(void); + +/* + * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED + * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places + * its end-noop macro stub at this address (aligned past the replayed + * program) so the stub cannot clobber a captured macro body; 0 until the + * first 0x114 upload enters the prelude buffer. + */ +u32 gsp_sema_mme_ir_end_get(void); + +/* + * SESSION 153-CONTINUED-70: IR code words captured so far (the maturity + * gate's measure; printed by the MME-PRE-DUMP for the offline sim). + */ +u32 gsp_sema_mme_code_words_get(void); + +/* + * SESSION 153-CONTINUED-68: is the IR instruction at address `pc` present + * in the replay buffer (i.e. was it uploaded by a captured run that was + * NOT dropped as a duplicate/over-cap)? The launch builder stubs every + * called macro slot whose SA entry sits outside this set, so a truncated + * or generation-mismatched capture becomes a no-op instead of a macro + * fetch into never-replayed IR (Graphics Exception: MISSING_MACRO_DATA). + */ +bool gsp_sema_mme_ir_covered(u32 pc); + +/* * r378 write guards. Both re-validate a guest-memory write target that was * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's * per-page resolution) against the CURRENT resolution before the bytes land, |