diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1640 |
1 files changed, 1558 insertions, 82 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2ab1060..3bb883d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -205,8 +205,28 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* + * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record pushbuffer VA. Constant across every measured boot (r908 + * documented the GL client ring pair {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every + * canonical push read named exactly this VA). Used to discriminate the + * kick ring page from shape lookalikes in the RAM ring scan - r978 + * latched a foreign table page (pa=0x104fa9000) whose entry pair passed + * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000) + * while the true ring (pa=0x1168ce000, found by scanning for the + * record's own entry pair) sat later in RAM scan order. + */ +#define SVC_GL_KICK_PUSH_VA 0x1203c0000ULL /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ -#define SVC_GR_USERD_WATCH 8 +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 /* * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM @@ -612,7 +632,23 @@ struct svc_channel { u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ /* * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The @@ -666,6 +702,14 @@ struct svc_channel { */ bool gr_ram_ring; /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring + * liveness re-validation pace for the kick channel (h 0xbfef00xx). + * While the kick USERD stays unresolved this channel's ring latch is + * re-checked each pass (at most once a second) and re-armed when it + * no longer decodes entries - see svc_kick_ring_revalidate. + */ + unsigned long gr_kick_ring_reval_at; + /* * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the * registered/ring-derived candidates all fail; the scan validates the * hit against the ring entries the put claims, so a foreign channel's @@ -1171,6 +1215,10 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 kick_badpg; /* r977: kick-family push reads + * that lacked the canonical + * submit-record prefix (wrong + * page; forced the rescue) */ u64 rescue_map_ok; /* SESSION 148: content-verified * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -1639,8 +1687,27 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int i; if (!desc || !desc->h_object || !desc->gpfifo_va || - desc->gpfifo_entries < 2) + desc->gpfifo_entries < 2) { + /* + * SESSION 153-CONTINUED-15 (draw attribution): a refused + * channel registration used to be silent, so a real client + * channel with an empty gpfifo VA looked identical to a + * channel that was never allocated. Name every refusal. + */ + static u32 refuse_log; + + if (refuse_log < 32) { + refuse_log++; + pr_info("sema-svc: chan add REFUSED h=0x%x gpfifo=0x%llx ent=%u eng=%u userd=0x%llx/%u\n", + desc ? desc->h_object : 0, + desc ? desc->gpfifo_va : 0, + desc ? desc->gpfifo_entries : 0, + desc ? desc->engine_type : 0, + desc ? desc->userd_pa : 0, + desc ? desc->userd_as : 0); + } return; + } if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; @@ -3452,6 +3519,7 @@ static void svc_pin_drop(u64 va) * every push was "implausible"). */ static inline bool svc_pool_sem_va(u64 va); +static bool svc_kick_family(const struct svc_channel *ch); static void svc_pool_pa_forget(u64 va); /* @@ -5525,9 +5593,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_owed_resolved, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, + svc.kick_badpg, svc.pool_alt_applied, svc.pool_residue, svc_wait_held, svc_wait_pass, svc_wait_giveup, svc_wait_ovf); @@ -6568,7 +6637,7 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_INLINE_BYTES 256 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -6821,8 +6890,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -7071,6 +7140,7 @@ struct svc_gl_up { u8 data[SVC_GR_INLINE_BYTES]; }; +#define SVC_GL_PROG_MAX 8 struct svc_gl_sec { bool seen; u32 subch; @@ -7080,6 +7150,18 @@ struct svc_gl_sec { * host-sem/2D ops whose VAs are not mapped in the * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ u64 rel_va; /* last VALID release decoded on the channel */ @@ -7109,9 +7191,146 @@ struct svc_gl_sec { */ u8 blob[SVC_GR_INLINE_BYTES]; u32 blob_len; + /* + * SESSION 153-CONTINUED-18: the kick notification this push carried + * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the + * C597 bind that opened this section). Executed at flush time by + * svc_kick_notify_exec when both the target and the NOTIFY are in + * this same push (the short variant carries the NOTIFY without the + * arm pair; it never executes and stays witness-only). + */ + u64 kick_notify_va; + u32 kick_notify_val; + bool kick_notify_seen; }; static struct svc_gl_sec svc_gl; /* service thread only */ /* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* + * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. + * + * A SET_OBJECT bind is sticky on real hardware: the class stays bound to + * the channel subchannel across pushes until another bind replaces it. The + * GLX server's C597 SET_OBJECT arrives once (the driver's init push at + * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s) + * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour + * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the + * SET_OBJECT-only recognizer below never opens a section for them and the + * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597 + * bind is seen - including catches of the init push - and treat any later + * method run on that pair, on engine_type==0, as C597 class space. + */ +static u64 svc_gl_att_h; /* channel whose subch carries the C597 bind */ +static u32 svc_gl_att_subch; +static u32 svc_gl_draw_log; /* bounded draw-section-open witness */ +/* + * SESSION 153-CONTINUED-13: the C597 attachment is per-channel. The single + * (h_object, subch) slot was overwritten by the RM channel's periodic + * C597-bind pushes (h=0xbfef0002, observed every ~7 s on this boot), after + * which the GLX client's draw pushes on 0x1000013 failed the BEGIN + * recogniser's `svc_gl_att_h == ch->h_object` test and no draw section was + * ever opened (a boot-level, order-dependent race: the r954 boot decoded + * no such pushes). A small table (first match wins, dedup on note) keeps + * every channel's bind sticky independently, so the draw channel's + * attachment cannot be stolen. + */ +#define SVC_GL_ATT_MAX 8 +static u64 svc_gl_att_tab_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_tab_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + /* refresh LRU position: move to slot 0 */ + if (i) { + memmove(&svc_gl_att_tab_h[1], + &svc_gl_att_tab_h[0], + (size_t)i * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], + &svc_gl_att_tab_sub[0], + (size_t)i); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + } + return; + } + if (!svc_gl_att_tab_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) { + /* table full: evict the oldest (last) slot */ + free = SVC_GL_ATT_MAX - 1; + } + memmove(&svc_gl_att_tab_h[1], &svc_gl_att_tab_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], &svc_gl_att_tab_sub[0], + (size_t)free); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + /* keep the legacy single slot pointing at the newest note (logging) */ + svc_gl_att_h = h; + svc_gl_att_subch = subch; +} + +static bool svc_gl_att_has(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) + return true; + return false; +} + +static u32 svc_gl_att_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return svc_gl_att_tab_sub[i]; + return 0xffu; +} + +static void svc_gl_att_clear(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + u32 n = SVC_GL_ATT_MAX - 1 - i; + + if (n) + memmove(&svc_gl_att_tab_h[i], + &svc_gl_att_tab_h[i + 1], + (size_t)n * sizeof(u64)); + if (n) + memmove(&svc_gl_att_tab_sub[i], + &svc_gl_att_tab_sub[i + 1], + (size_t)n); + svc_gl_att_tab_h[SVC_GL_ATT_MAX - 1] = 0; + svc_gl_att_tab_sub[SVC_GL_ATT_MAX - 1] = 0; + if (svc_gl_att_h == h) + svc_gl_att_h = svc_gl_att_tab_h[0]; + return; + } +} +/* * SESSION 153 (G-B gate: the client's own C597 draws must open sections). * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which * returns true when the push VA sits in the GR candidate ring - even when @@ -7247,39 +7466,74 @@ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ static char *sema_gl_prog_hex; module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); MODULE_PARM_DESC(sema_gl_prog_hex, - "explicit program bytes (hex) for the synthesized GL launch (<=64 B)"); + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; static u32 svc_gl_prog_len; -static bool svc_gl_prog_parsed; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; -static void svc_gl_prog_apply_param(void) +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) { - const char *h = sema_gl_prog_hex; u32 n, i; - if (svc_gl_prog_parsed) - return; - svc_gl_prog_parsed = true; + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; if (!h || !h[0]) return; n = (u32)strlen(h); if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { - pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n", - n); + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); return; } for (i = 0; i < n; i += 2) { unsigned int b; if (sscanf(h + i, "%2x", &b) != 1) { - pr_warn("sema-gl: prog-hex param bad at %u\n", i); + pr_warn("sema-gl: %s param bad at %u\n", tag, i); return; } - svc_gl_prog[i / 2] = (u8)b; + dst[i / 2] = (u8)b; } - svc_gl_prog_len = n / 2; - pr_info("sema-gl: prog-hex param: %u bytes stored\n", - svc_gl_prog_len); + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); } /* @@ -7320,6 +7574,61 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +/* + * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. + * + * The GL client's per-frame kick push on the kick-family channel + * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}] + * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA + * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16 + * status at +0xE). On real hardware the 2D engine writes a completed + * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) - + * that write is the client's per-frame completion edge; the lane used to + * log "flush skip" and drop it, leaving the member all-zero forever. This + * executor authors the NvNotification through the same trusted VA walk + * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies + * by readback. sema_kick_notify=0 disables (A/B); the witness line + * "sema-kick: exec ..." is bounded (first 48 + every 512th). + */ +static bool gsp_sema_kick_notify = true; +module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644); +MODULE_PARM_DESC(sema_kick_notify, + "Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)"); +static u32 svc_kick_log; /* bounded kick-notify witness */ +static u64 svc_kick_count; /* executor invocations */ +/* + * r982: the notify page a pending-marker hit proved, per knowledge the + * exec picked up. Kept in the service's own scope (not the channel + * snapshot): the kick channel is the only consumer and the entry is + * re-validated by a read before every use, so a stale PA can only cost + * one failed probe, never a write into freed memory. + */ +static struct { + u64 h; /* channel handle that proved the page */ + u64 pa; /* PA of the marker-proven notify page */ +} svc_kick_pa_cache[4]; + +static void svc_kick_pa_cache_store(u64 h, u64 pa) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (svc_kick_pa_cache[i].h == h) { + svc_kick_pa_cache[i].pa = pa; + return; + } + } + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (!svc_kick_pa_cache[i].h) { + svc_kick_pa_cache[i].h = h; + svc_kick_pa_cache[i].pa = pa; + return; + } + } + svc_kick_pa_cache[0].h = h; + svc_kick_pa_cache[0].pa = pa; +} + static bool gsp_sema_gl_draw_synth; module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); MODULE_PARM_DESC(sema_gl_draw_synth, @@ -7335,7 +7644,13 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, { u32 i, free = SVC_GL_HOLD_MAX; - if (!h || !prog || !param || !rel) + /* + * SESSION 153-CONTINUED-13: rel == 0 is admitted (a section that has + * real operands but flushed before the channel's release was decoded + * stashes them here); a later call with rel != 0 completes the entry + * and a rel == 0 call must NEVER wipe a stored release. + */ + if (!h || !prog || !param) return; for (i = 0; i < SVC_GL_HOLD_MAX; i++) { if (svc_gl_hold_tab[i].h == h) { @@ -7351,8 +7666,10 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, svc_gl_hold_tab[free].prog_va = prog; svc_gl_hold_tab[free].param_va = param; svc_gl_hold_tab[free].param_size = psize; - svc_gl_hold_tab[free].rel_va = rel; - svc_gl_hold_tab[free].rel_payload = rpay; + if (rel) { + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; + } } static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, @@ -7419,13 +7736,46 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) break; memcpy(&v[k], data + 4 * k, 4); } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. */ if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && - count >= 2 && !svc_gl.prog_va) { - svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) | - (u64)(v[1] & 0xfffffffc); + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } return; } switch (addr) { @@ -7510,12 +7860,383 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) } } return; + case 0x1c00: /* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run + * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex + * stream the draw reads positions/colours from. + * Observed 0x1c00 [00001000 00000001 2150b790] and + * 0x1c10 [00001008 00000001 215257c0] (SESSION + * 153-CONTINUED-10 draw-push decode). + */ + case 0x1c10: + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x2380: /* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper, + * lower]. Observed [00010000 00000001 2150a500]. + */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x0800: /* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER, + * OFFSET_LOWER, w, h, fmt, ...] - the surface the + * draw rasterizes into. Observed [00000001 222fa200 + * 00000040 00000004 000000f7 00000000]. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; default: return; } } /* + * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect + * the C597 operand fields of every method run on the section's subchannel. + * Used when a draw section opens at its BEGIN marker (0x1618): the state + * methods that precede the marker (vertex stream binds, colour target, + * constant buffer binds) were decoded before the section existed, so their + * VAs must be re-collected or the candidate's page set starves and the + * replay's shadow relocation leaves them pointing at unreachable guest + * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main + * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only. + */ +static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, + u32 att_subch) +{ + u32 o = start; + + while (o + 4 <= end && o + 4 <= len) { + u32 hdr, op, count, subch, addr; + + memcpy(&hdr, buf + o, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + if ((op == 1 || op == 3 || op == 5) && count && + o + 4 + count * 4 <= len) { + if (subch == att_subch) + svc_gl_field(addr, count, buf + o + 4, + len - o - 4); + o += 4 + count * 4; + continue; + } + o += 4; + } +} + +/* + * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame + * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write + * is a completed NvNotification: nonzero timestamp at +0x0 (ns since + * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0 + * at +0xE. Resolution is the release path's trusted walk + * (svc_lookup_trusted -> svc_write_ap), then readback verification. + */ +static bool svc_kick_family(const struct svc_channel *ch); +static u64 svc_push_pa_ceiling(void); +static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 val, + u64 push_va) +{ + u64 pa = 0, ts = ktime_get_real_ns(); + int ap = SVC_APERTURE_SYS; + u8 old[16], nw[16], rb[16]; + bool ok = false, changed = false; + u32 o0 = 0, n0 = 0; + int rc = -ENOENT; + const char *rsrc = "trusted"; + u32 kc = 0; + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + memcpy(nw + 8, &val, sizeof(val)); + if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { + int r1, r2; + + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + rsrc = "trusted-u"; + } + svc_cur_vas = save; + } + /* + * r978: r977's exec fires (27 calls) but authors the word in + * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads, + * while r973 - the boot where the kick consumption visibly + * advanced the client - wrote the RING-ADJACENT sysmem slot + * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring + + * 0x1000 + 0xb0; the r975 probe measured the identical rule, + * pa=0x104b640b0 over ring 0x104b63000). Both walk answers + * miss that page this boot, so before falling back to the + * shadow: (a) the general lookup (the discovered rm_mem map - + * r979's sema_probe proved it resolves this VA to SYS after + * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin + * for this VA (the guest pin stream is the client's own + * statement of the mapping), then (c) for a kick-family + * channel the ring-relative completion slot, computed from + * the live ring pin each call. + */ + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 mpa = 0; + int map_ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &mpa, &map_ap) && + svc_ap_of(map_ap) == SVC_APERTURE_SYS) { + pa = mpa; + ap = map_ap; + rsrc = "map"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + svc_ap_of(iap) == SVC_APERTURE_SYS) { + pa = ipa; + ap = iap; + rsrc = "ipin"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch && + svc_kick_family(ch)) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, + &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 cand = (ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE + + (va & (SEMA_PAGE_SIZE - 1)); + + if (cand < svc_push_pa_ceiling()) { + pa = cand; + ap = SVC_APERTURE_SYS; + rsrc = "ring+10b0"; + } + } + } + /* + * r981: the client's pending notification marker. r979/r980 + * measured the REAL slot's +0xC u32 reading 0x80000000 + * while the client waited (and 0 once completed; the write + * below clears it) - on the page the discovered map names, + * ring + 0x20b0 on both boots, one page ABOVE the r973-era + * ring + 0x10b0 heuristic the exec had been writing + * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs + * 0x12e0790b0 on r980 - the completion landed on the wrong + * page both boots). When the chosen candidate does not show + * the marker, probe the ring-relative alternatives and use + * the one that does; the readback below still verifies the + * write. A slot already completed (marker 0) keeps the + * resolved candidate unchanged. + * + * r982: remember the page a pending-marker hit named, per + * handle. r981's boot: exec n=1 found the marker page + * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the + * marker is 0 afterwards so every later exec fell back to + * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes) + * while the map/trusted stages were still catching up. The + * cached page is retried at the top of the next call (read + * first, used only while the read succeeds), so all later + * completions stay on the page the marker proved. + */ + if (ch && svc_kick_family(ch)) { + /* Retry the page a marker hit proved, if still readable. */ + for (kc = 0; kc < 4; kc++) { + u32 probe_buf = 0; + + if (svc_kick_pa_cache[kc].h != ch->h_object || + !svc_kick_pa_cache[kc].pa) + continue; + if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa, + SVC_APERTURE_SYS, &probe_buf, + sizeof(probe_buf)) && + probe_buf != 0xffffffffu) { + pa = svc_kick_pa_cache[kc].pa; + ap = SVC_APERTURE_SYS; + rsrc = "kickcache"; + } + break; + } + } + if (ch && svc_kick_family(ch)) { + u32 pend = 0; + + if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap), + &pend, sizeof(pend)) && + pend != 0x80000000u) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, + &ring_pa, &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 base = ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1)); + u64 cands[2]; + int i; + + cands[0] = base + + 2 * SEMA_PAGE_SIZE + off; + cands[1] = base + + SEMA_PAGE_SIZE + off; + for (i = 0; i < 2; i++) { + u32 m = 0; + + if (cands[i] >= + svc_push_pa_ceiling()) + continue; + if (!svc_read_ap(vgpu, + cands[i] + 0xc, + SVC_APERTURE_SYS, + &m, sizeof(m)) && + m == 0x80000000u) { + pa = cands[i]; + ap = SVC_APERTURE_SYS; + rsrc = i ? + "ring+10b0p" : + "ring+20b0p"; + svc_kick_pa_cache_store( + ch->h_object, + cands[i]); + break; + } + } + } + } else if (pend == 0x80000000u && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + /* + * The chosen candidate IS the marker page: + * prove it for the cache so later execs keep + * writing the same page (r981: after the + * single marker hit the fallback wrote two + * wrong ring-relative pages for 33 further + * execs). Only a SYS answer that showed the + * marker is cached; a zero read (already + * completed) caches nothing - unproven. + */ + svc_kick_pa_cache_store(ch->h_object, pa); + } + } + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); + if (!r1 && !r2 && + !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) { + rc = 0; + ok = true; + changed = memcmp(old, rb, sizeof(old)) != 0; + memcpy(&o0, old, 4); + memcpy(&n0, rb, 4); + } else { + /* CONT.18 probe: name the failing step. */ + rc = r1 ? (r1 < 0 ? r1 : -100) : + (r2 ? (r2 < 0 ? r2 : -101) : -102); + } + } + svc_kick_count++; + if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { + svc_kick_log++; + pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", + ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val, + (unsigned long long)ts, o0, n0, changed ? 1 : 0, + (unsigned long long)svc_kick_count, rc); + } + /* + * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting + * client's completion edge - post the channel event so its OS event + * fires (r934's lesson on the notify2d path: without the event the + * GL client's kick submit wait never completes and the app retries + * on a ~16-22 s timer; r972 measured exactly that cadence with the + * event missing here). Only for a real channel (the dev probe + * passes ch=NULL). + */ + if (ok && ch) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return ok; +} + +/* + * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify + * executor on an explicit VA against the service's own device. The + * execution MUST run in a kernel-thread context: vfio_iommu_type1's + * dma_rw only copies when current->mm == dma->mm or the caller is a + * kthread (kthread_use_mm); a sysfs write from a shell process has + * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The + * workqueue item below is a kthread - the same context class as the + * sema service thread that runs the real kick lane. + * echo 0x12000e0b0 > .../parameters/sema_kick_probe + */ +static u64 svc_kick_probe_va; +static void svc_kick_probe_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn); + +static void svc_kick_probe_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (vgpu) + svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0); +} + +static int nv_vgpu_sema_kick_probe_set(const char *val, + const struct kernel_param *kp) +{ + if (!val || kstrtoull(val, 0, &svc_kick_probe_va)) + return -EINVAL; + if (svc_kick_probe_va) + schedule_work(&svc_kick_probe_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = { + .set = nv_vgpu_sema_kick_probe_set, +}; + +module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200); +MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)"); + +/* * Close the open section: synthesize the QMD from the collected fields and * capture it as the forwarding candidate. The QMD word layout mirrors * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 @@ -7531,7 +8252,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 at_off) { struct svc_push_state s2; - struct gsp_bridge_inline pr; + struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; u32 n_recs = 0, k; /* * SESSION 149 R16 (M3): the inline attachment's source. Priority: @@ -7597,7 +8318,53 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { - if (gsp_sema_dbg && svc_gl_skip_log < 16) { + /* + * SESSION 153-CONTINUED-13: the release may be + * decoded AFTER this section in ring order + * (the r955 init: the 24 B release pushes sit + * behind the 1524 B init push in the ring, so + * every replay pass of the section flushed + * before svc_gl.rel_va was known). When the + * operands are real but only the release is + * missing, stash them for the channel so the + * next section/pass (by then the constant + * release stream has been decoded) captures + * with src=hold instead of losing the init + * capture - without which the guest's GLX + * init completion degrades and the client + * never emits its draw pushes at all. + */ + if (svc_gl.prog_va && svc_gl.param_va && + !svc_gl.rel_va) + svc_gl_hold_store(ch->h_object, + svc_gl.prog_va, + svc_gl.param_va, + svc_gl.param_size, + 0, 0); + /* + * SESSION 153-CONTINUED-18 (task 1): replace + * the old "flush skip" drop with NOTIFY + * EXECUTION when this push carried the kick + * member (SET_NOTIFY_A/B + NOTIFY in the same + * push). The section has no draw operands - + * it is the client's completion kick - so the + * GPU's notification write is the only work + * it asks for. + */ + if (gsp_sema_kick_notify && + svc_gl.kick_notify_seen && + svc_gl.kick_notify_va && + svc_kick_notify_exec(vgpu, ch, + svc_gl.kick_notify_va, + svc_gl.kick_notify_val, + push_va)) { + /* authored once; a replay pass must not + * re-author an unchanged member. */ + svc_gl.kick_notify_seen = false; + svc_gl.kick_notify_va = 0; + goto out; + } + if (gsp_sema_dbg && svc_gl_skip_log < 2048) { svc_gl_skip_log++; pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", ch->h_object, push_va, @@ -7748,27 +8515,41 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } } memset(blob, 0, sizeof(blob)); - blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, - sizeof(blob)); - if (!blob_rc) { - u32 z = 0; - - for (k = 0; k < 16; k++) - z |= blob[k]; - if (z) - blob_trusted = true; - } - if (blob_trusted) { - blob_n = sizeof(blob); - blob_src = "trusted-prog"; - } else if (svc_gl_prog_len) { + /* + * SESSION 153-CONTINUED-10: the explicit operator program override + * (sema_gl_prog_hex) wins over the trusted page read. The GLX + * server's per-frame CONTENT pushes bind their own program slots + * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated + * nonzero bytes at that offset; the R16 trusted-read priority + * accepts any nonzero 16 bytes as "the real program" and attached + * garbage, and the engine rejected the draw with "Graphics + * Exception: Shader Program Header 11/18 Error" (r944 fault + * evidence, push 0x120014578). An unset override keeps the + * historical read priority unchanged. + */ + if (svc_gl_prog_len) { blob_n = svc_gl_prog_len; memcpy(blob, svc_gl_prog, blob_n); blob_src = "prog-hex"; - } else if (svc_gl.blob_len) { - blob_n = svc_gl.blob_len; - memcpy(blob, svc_gl.blob, blob_n); - blob_src = "lid-fallback"; + } else { + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } } /* * r929 (Path A): a stream-derived capture is a 3D method window - the @@ -7776,12 +8557,69 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * uploads/binds, so the synthesized compute attachments (the inline * program record, and with it the prog-hex fallback) do not apply. * Draw-synth/hold captures keep the old compute path unchanged. - */ - if (blob_n && op_src[0] != 's') { - memset(&pr, 0, sizeof(pr)); - pr.dst_va = svc_gl.prog_va; - pr.len = blob_n; - memcpy(pr.data, blob, blob_n); + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); n_recs = 1; } /* @@ -7790,11 +8628,11 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * replays of the same (push_va, content hash). */ svc_gl_cap_store(push_va, svc_push_hash(buf, len)); - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u, op_src); + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); /* * SESSION 153: remember real operands for this channel so a later * draw-only section on the same channel can still capture. A @@ -7820,7 +8658,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, get_unaligned_le32(blob + 8), get_unaligned_le32(blob + 12)); } - svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, push_va); /* * r929 (Path A): a stream-derived capture (op_src=stream: the section @@ -7838,8 +8676,18 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * 448-word cap leaves headroom for the backend's 512-word method * block (2 bind words + the window + the 8-word release tail). * Draw-synth/hold captures are never marked. - */ - if (op_src[0] == 's' && svc_gr_cand.valid && + * + * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX + * server's per-frame CONTENT pushes carry the draw window but no + * program/param operands (the class bind is sticky; no per-push + * SET_OBJECT), so their sections capture with the channel's real + * operands adopted from the hold table (filled by the init + * capture). The window itself is still the REAL push bytes sliced + * from its own first C597 state method to its END - exactly the + * raw 3D method window the r929 replay lane wants; the synthesized + * compute fields are what stream_3d bypasses. + */ + if ((op_src[0] == 's' || op_src[0] == 'h') && svc_gr_cand.valid && len == svc_gr_cand.push_n * 4 && svc_gl.start_off + 8u <= at_off && (at_off - svc_gl.start_off) / 4u <= 448u) { @@ -12019,16 +12867,34 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, if (!ch->notify2d_va) return false; if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { - if (gsp_sema_dbg && svc_notify2d_log < 16) { + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { svc_notify2d_log++; pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", ch->h_object, ch->notify2d_va); } return false; } - if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) - return false; - svc_notify2d_done++; + /* + * SESSION 153-CONTINUED-18: read the first word for the witness + * (the kick member's timestamp slot: 0 = never GPU-written; the + * completed write makes it nonzero - see svc_kick_notify_exec). + */ + { + u32 o0 = 0; + + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0))) + o0 = 0xffffffff; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return false; + svc_notify2d_done++; + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n", + ch->h_object, ch->notify2d_va, pa, o0); + } + } /* * r834: pin the page that accepted the ack; the same stale-map * defense as the push-side pin in the decode. @@ -12045,11 +12911,6 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, */ if (completion) gsp_rm_post_channel_event(vgpu, ch->h_object); - if (gsp_sema_dbg && svc_notify2d_log < 16) { - svc_notify2d_log++; - pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n", - ch->h_object, ch->notify2d_va, pa); - } return true; } @@ -12300,6 +13161,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* SESSION 153-CONTINUED-10: draw-window tracker. First method of + * the current C597 run on the attached subchannel after the last + * foreign-subchannel (2D/present) run; see the BEGIN recognizer + * below. Per pass, like the rest of the decode state. */ + u32 gl_win_start = 0; /* m164: fresh CLASS-2 QMD witness for this decode pass. */ svc_gr_qmd_seen = 0; @@ -12430,6 +13296,55 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-12: attach on the draw marker itself. + * + * The section recogniser below needs the sticky C597 bind + * (svc_gl_att_h) that the boot-class GLX init push carries. On a + * boot where the class is bound channel-side (inside the emulated + * RM channel setup) and no decoded push re-emits SET_OBJECT C597, + * every draw-shaped push is dropped and no 3D replay ever runs. + * Adopt the first trusted push that carries an NVC597_BEGIN IMMD + * (op 4, addr 0x1618, subchannel 0) as the attached channel; the + * per-pass gl_win_start tracker then runs over that same push from + * its start, exactly as it does after an explicit bind, so the + * window keeps the pre-BEGIN state methods (vertex stream bases, + * colour target, constant-buffer selectors). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && trusted && + !svc_gl_att_has(ch->h_object, 0)) { + u32 o2 = 0; + + while (o2 + 4 <= len) { + u32 h2, op2, cnt2, sc2, ad2; + + memcpy(&h2, buf + o2, 4); + op2 = (h2 >> 29) & 0x7; + cnt2 = (h2 >> 16) & 0xfff; + sc2 = (h2 >> 13) & 0x7; + ad2 = (h2 & 0x1fff) << 2; + if (op2 == 4 && ad2 == 0x1618 && sc2 == 0) { + svc_gl_att_note(ch->h_object, 0); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: attach-on-BEGIN h=0x%x push=0x%llx off=0x%x (no explicit C597 bind seen)\n", + ch->h_object, push_va, o2); + } + break; + } + if (op2 == 4 || op2 == 7) { + o2 += 4; + continue; + } + if ((op2 == 1 || op2 == 3 || op2 == 5) && cnt2 && + cnt2 <= 0xfff && o2 + 4 + cnt2 * 4 <= len) { + o2 += 4 + cnt2 * 4; + continue; + } + o2 += 4; + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -12441,6 +13356,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on + * the attached C597 channel. The per-frame CONTENT pushes + * interleave the 2D/present ops (subchannel 3) and the 3D + * block (subchannel 0); the replayable 3D window starts at + * the first subchannel-0 method after the last subchannel-3 + * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are + * excluded: they name client semaphore VAs (0x1_2000c000) + * the replay channel cannot translate (the r929 v1 fault + * class), so they must stay outside the sliced window. + */ + if (gsp_sema_gl_xlate && !ch->engine_type) { + u32 as = svc_gl_att_subch_of(ch->h_object); + + if (as != 0xffu) { + if (subch != as) + gl_win_start = 0; + else if (!gl_win_start && + !(op == 1 && addr >= 0x1b00 && + addr <= 0x1b0c)) + gl_win_start = off; + } + } + + /* * fail4: name every big method-data run and every run whose * address is the LOAD_INLINE_DATA method (any op/subch - the * encoding of the 64 KB H2D upload is what we are pinning). @@ -12676,6 +13615,68 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); } + /* + * SESSION 153-CONTINUED-10 (task 1): the client's per-frame + * CONTENT pushes carry real C597 draws but no per-push + * SET_OBJECT (the class bind is sticky), so the section + * recognizer above never opened for them and the 3D replay + * lane never saw a draw. Recognise the draw marker itself: + * NVC597_BEGIN (0x1618, an IMMD method - its immediate is + * the topology in the count field) on the attached + * subchannel of the attached channel opens the section. + * The window starts at gl_win_start (the first C597 state + * method after the last foreign-subchannel run); the state + * methods before the marker are re-scanned so their operand + * VAs (vertex stream bases 0x1c00/0x1c10, colour target + * 0x0800, constant buffer selectors 0x2380) enter the + * candidate's page set - without them the backend's shadow + * relocation cannot rewrite the window and the engine + * faults on an unreachable guest VA (the r929 v2 class). + * The window then runs from gl_win_start to the pass end + * (the section close at the END method / push tail), which + * svc_gl_flush slices into the verbatim 3D replay. + */ + if (gsp_sema_gl_xlate && !ch->engine_type && op == 4 && + addr == 0x1618 && !svc_gl.seen && + svc_gl_att_has(ch->h_object, subch)) { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + u32 wstart = gl_win_start ? gl_win_start : off; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = wstart; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + if (wstart < off) + svc_gl_scan_window(wstart, off, buf, len, + subch); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n", + ch->h_object, push_va, subch, wstart, + off, len, trusted); + } + } + + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + svc_gl_att_has(ch->h_object, subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -12773,6 +13774,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl.start_off = off; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* SESSION 153-CONTINUED-10: the bind is sticky - + * remember it so the class-space recognizer + * (BEGIN 0x1618, below) accepts this channel's + * later per-push-CONTENT pushes that carry no + * SET_OBJECT of their own. + * SESSION 153-CONTINUED-13: per-channel table - + * a foreign channel's periodic C597 bind must + * not steal this channel's attachment. */ + svc_gl_att_note(ch->h_object, subch); /* * SESSION 153: witness every C597 section open * (bounded) - it is the recognizer-side proof @@ -12786,9 +13796,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->h_object, push_va, subch, trusted); } - } else if (svc_gl.seen) { - svc_gl_flush(vgpu, ch, &st, buf, len, - push_va, trusted, replay, off); + } else { + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + /* A different class bound over the attached + * subchannel replaces the C597 bind: drop the + * attachment so the class-space recognizer + * cannot misread methods of the new class. + * SESSION 153-CONTINUED-13: clears only this + * channel's entry. */ + svc_gl_att_clear(ch->h_object, subch); } } else if (gsp_sema_gl_xlate && svc_gl.seen && subch == svc_gl.subch) { @@ -12951,6 +13970,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (count >= 2) ch->notify2d_va |= (u64)v1; /* + * SESSION 153-CONTINUED-18 (task 1): when + * this push also carries the C597 kick + * section, remember the arm so the section + * flush can execute the NOTIFY write (the + * 2D ack below is the watchdog's zero-ack; + * the kick member needs the completed + * NvNotification authored). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl.kick_notify_va = + ch->notify2d_va; + /* * r834: this method stream reached us through * a REAL read (svc_read_last_pa); pin its page * so the watchdog's push translation cannot go @@ -12971,6 +14002,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case SVC_2D_NOTIFY: if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + /* + * SESSION 153-CONTINUED-18: mark the + * kick section's NOTIFY (payload 0) so + * svc_gl_flush executes the completion + * write at the armed VA - the executor + * authors the NvNotification the 2D + * zero-ack alone does not. + */ + if (gsp_sema_gl_xlate && svc_gl.seen) { + svc_gl.kick_notify_val = v0; + svc_gl.kick_notify_seen = true; + } if (svc_notify2d_write(vgpu, ch, true)) { /* @@ -16758,10 +17801,18 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget * gate stays open while candidates are under watch, so the * liveness confirmation cannot be starved by slice accounting. + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. */ if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || - ch->gr_ram_userd_watch_n)) { + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -18590,6 +19641,22 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, continue; if (len1 < 4 || len1 > 0x100000) continue; + /* + * r979: kick-family ring discrimination. Shape alone + * admitted a foreign table page ahead of the real ring + * on r978 (pa=0x104fa9000, entries naming a bogus + * 0x12048b824/0x40000 push) while the true ring's + * entry pair names the client's per-frame record VA + * (SVC_GL_KICK_PUSH_VA, the r908-documented GL + * pushbuffer, constant across r908-r978). A + * kick-family candidate must name the record; the + * lookalike class then never latches. Non-kick + * channels keep the shape gate unchanged. + */ + if (svc_kick_family(ch) && + va0 != SVC_GL_KICK_PUSH_VA && + va1 != SVC_GL_KICK_PUSH_VA) + continue; *pa_out = gpa + o; return true; } @@ -18627,6 +19694,99 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance). + * + * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring + * found by the guest-RAM content scan) posts a per-frame completion kick + * the client's flush loop paces on. Its USERD is a sysmem control block + * the driver allocates far from the ring - r971 measured the ring at pa + * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the + * same nomination was +0.6..+2.3 GB across r966-r971) - while the general + * RAM USERD scan's allocation-locality gate only admits candidates within + * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939 + * nine-decoy class). That gate rejected the channel's real USERD on every + * boot since the gate landed, the scan retired ("RAM scan found no + * candidate in 20 slice(s)"), the pump never iterated the kick entries + * and the kick-notify lane never ran. For this family the ring-entry + * match (every PUT-claimed ring entry must decode to a pushbuffer VA + * inside the ring's own allocation window) is the ownership proof; the + * locality distance is not - so the gate is waived for these channels and + * the distance is logged instead. The pump's speculative-latch + * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final + * judge, and the rejected-latch memory prevents a frozen block from ever + * pinning the scan. + */ +static bool svc_kick_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu; +} + +/* + * Re-validate the kick channel's ring latch each pass (bounded to once a + * second per channel). The RAM content scan latches a ring page once; if + * the driver later zeroes or reuses that page (the r966-r971 class the + * design brief describes: the USERD scan then validates every candidate + * against a dead ring and adopts none), the latch is dropped here so the + * content scan re-runs and re-latches the channel's current ring page. + * The stale entry pair is logged on re-arm so the transition is visible + * in the record. A live ring (a first entry that still decodes to a + * window-bounded pushbuffer VA) is a cheap no-op. + */ +static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + unsigned long flags; + int i; + + if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok) + return; + if (!time_after(jiffies, ch->gr_kick_ring_reval_at)) + return; + ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000); + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + u64 eva; + u32 elen; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x100000) + return; /* ring page live; nothing to do */ + } + /* + * The latched page no longer decodes entries: drop the one-shot + * latch state so the RAM ring content scan re-runs. The pin is left + * in place - a stale pin only shadows VA reads, while the content + * scan validates against raw RAM and re-pins on a hit. + */ + clear_bit(ch->gr_index, &svc_gr_ring_done); + clear_bit(ch->gr_index, &svc_pb_gr_done); + ch->gr_ram_cursor = 0; + ch->gr_ram_passes = 0; + ch->gr_ram_logged = false; + ch->gr_ram_ring = false; + pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n", + ch->h_object, ch->gpfifo_va, e0, e1); + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + svc.chan[i].gr_ram_ring = ch->gr_ram_ring; + svc.chan[i].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). * Such a client's USERD is a system-memory object like its ring, so the @@ -18666,6 +19826,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, u64 live_pa = 0; u32 live_put = 0, live_get = 0, live_prev = 0; int ring_ap = SVC_APERTURE_SYS; + bool kick; u32 wi; int i; unsigned long flags; @@ -18684,6 +19845,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, if (!buf) return false; } + kick = svc_kick_family(ch); + /* + * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch + * before validating candidates against it (svc_kick_ring_revalidate + * re-latches a ring page the driver has zeroed since the latch; see + * its comment). + */ + svc_kick_ring_revalidate(vgpu, ch); (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); /* * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. @@ -18710,9 +19879,13 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * mapped by NO process, and never advanced on any pass * (two frozen snapshots that cost a 15 s reject/re-latch * cycle all boot). A control block this far from its own - * ring is not this channel's USERD. + * ring is not this channel's USERD. SESSION 153-CONT-19: + * waived for the kick family - its real USERD is measured + * 0.6-2.3 GB from the ring, and the ring-entry match (not + * proximity) is the ownership proof there (see + * svc_kick_family). */ - if (ring_pa && + if (ring_pa && !kick && (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) || wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) { ch->gr_ram_userd_watch_n--; @@ -18737,10 +19910,47 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } last = ch->gr_ram_userd_watch_put[wi]; if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } wi++; continue; } - if (hp != last + 1) { + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { /* * SESSION 154: a jump or a reset is the churn * signature (the r916-r919 lookalike pools move by @@ -18755,9 +19965,37 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; ch->gr_ram_userd_watch_hits[wi] = ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; continue; } ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } /* * SESSION 154: ONE observed +1 step adopts. The live block * advances ~2-8 s per step (measured) while the client's @@ -18814,9 +20052,15 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * latches read 12/8 and 16/16 and every previously * measured lookalike counted 16 or more. Also accept * a free-running put (the old put <= entries bound - * would reject the live block). + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. */ - if (!hp || hp > 8 || hg > hp) + if (!hp || hp > (kick ? 0x100000 : 8) || hg > hp) continue; { u32 z = 0, zi; @@ -18842,8 +20086,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * above. Excludes the r936-r939 nine-decoy pair * (~460 MB from the ring) before it can occupy a * watch slot and burn a reject/re-latch cycle. + * SESSION 153-CONT-19: waived for the kick family + * (h 0xbfef00xx) - its real USERD is measured far + * outside this window, so admission there is by the + * ring-entry match alone (the entry loop below) and + * the pump's +1/+2-step confirmation; the distance + * is logged at the latch. */ - if (ring_pa && + if (ring_pa && !kick && (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) || gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5))) continue; @@ -18915,6 +20165,64 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, hp == ch->gr_ram_userd_reject_put) continue; /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* * SESSION 154 (M-GL1.2): a candidate that passed the * shape gate, the entry validation, the page-density * gate and the put cap is latched SPECULATIVELY. The @@ -18942,6 +20250,23 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && !ch->gr_ram_userd_watch_n) { ch->gr_ram_userd_logged = true; @@ -18974,6 +20299,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_rejects; svc.chan[i].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[i].gr_ram_userd_watch[wj] = ch->gr_ram_userd_watch[wj]; @@ -18981,6 +20310,8 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[wj]; svc.chan[i].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } break; } @@ -19020,6 +20351,27 @@ static const u8 svc_gr_anchor_sig[16] = { }; #define SVC_GR_ANCHOR_SWEEPS 3 +/* + * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record is a fixed method stream opening with the SET_SUBDEVICE_MASK + * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the + * bytes r975's anchor pin answered and r973 decoded to n=512 execs). + * Every boot r966-r976 read that VA through some untrusted translation + * and, because a plausible-looking garbage page (r976: first8=9864be60 + * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the + * generic method-stream parse, the entry was consumed with the wrong + * bytes and the kick lane stayed silent for the whole session. For + * this family the canonical prefix IS the content identity: a read + * that lacks it came from a stale/foreign page no matter how + * plausible it parses, and must take the rescue chain (VAS walk -> + * content anchor -> pin). + */ +static bool svc_kick_canonical(const u8 *buf, size_t len) +{ + return len >= sizeof(svc_gr_anchor_pre) && + !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre)); +} + static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, u32 len) { @@ -29776,8 +31128,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * probably locked" while every other channel kept running. * Writing the ack on every pass that sees work pending is * idempotent. + * + * SESSION 153-CONTINUED-19: NOT for the kick family. Their + * notify2d_va is the kick record's own NOTIFY target (the + * shared completion word the GL client polls), and the kick + * lane authors that word (a completed NvNotification with a + * nonzero timestamp, svc_kick_notify_exec). The per-pass ack + * zeroed it 14029 times on r972 - the last kick write + * survived at most one pass (~60 ms) before being wiped, so + * the client's poll kept seeing "not fired". The GPU never + * clears this notification; the client does (re-arm). */ - if (!ch->engine_type && ch->notify2d_va && put != get) + if (!ch->engine_type && ch->notify2d_va && put != get && + !svc_kick_family(ch)) svc_notify2d_write(vgpu, ch, false); /* * SESSION 148 (r864): registration-based keep-alive that @@ -29870,6 +31233,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + /* * SESSION 154 (M-GL1.2): speculative-latch confirmation for * the RAM USERD. The scan latches a gated candidate right @@ -29880,13 +31247,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * candidate. The live block advances 1->2->3 in 2-10 s * (measured); the client's own release waits are ~11 s, so * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && - !ch->gr_ram_userd_confirmed) { + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + if (!ch->gr_ram_userd_latch_at) ch->gr_ram_userd_latch_at = jiffies; - if (put == ch->gr_ram_userd_latch_put + 1 || - put == ch->gr_ram_userd_latch_put + 2) { + stepped = kfam ? + put > ch->gr_ram_userd_latch_put : + (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2); + if (stepped) { ch->gr_ram_userd_confirmed = true; pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, @@ -30480,6 +31868,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } bool injected = false; bool allow_ops = (ch->engine_type != 0); + bool kick_badpg = false; /* * A resolved buffer that does not start @@ -30579,7 +31968,48 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) len >= 4 && len <= 16 && !memchr_inv(pb, 0, len); - if (!svc_method_stream_plausible(pb, len) && + /* + * r977/r979: a read of the + * kick family's per-frame + * record (SVC_GL_KICK_PUSH_VA; + * r978 showed the channel also + * carries bulk pushes that + * must NOT be held to this + * content identity - a 256 KB + * push at 0x12048b824) whose + * bytes lack the canonical + * submit-record prefix is a + * wrong page no matter how + * plausible it parses (r976: + * src=3 window garbage + * 9864be60 ffffffff consumed + * as a valid push and the kick + * lane ran zero execs). Force + * the implausible path so the + * VAS walk / content anchor / + * pin-drop chain below + * re-anchors the page. + */ + kick_badpg = !zero_nop && + svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128 && + !svc_kick_canonical(pb, len); + if (kick_badpg) { + svc.kick_badpg++; + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + } + + if ((!svc_method_stream_plausible(pb, len) || + kick_badpg) && !zero_nop) { svc.false_trans++; if (injected) { @@ -30657,6 +32087,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool pl = !rr ? svc_method_stream_plausible(vb, len) : false; + /* + * r977: for a kick-family + * entry a plausible walk + * answer still has to carry + * the canonical prefix - a + * wrong-VAS garbage page + * parses plausibly (r976) and + * would re-shadow the real + * page here. + */ + if (pl && kick_badpg && + !svc_kick_canonical(vb, len)) + pl = false; + if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30670,6 +32114,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, vb, len); svc.pushpg_fallback++; ok = true; + /* + * r977: a kick-family + * entry whose page the + * walk re-anchored is + * pinned so later passes + * read the canonical page + * directly (src=2) instead + * of re-paying the walk + * while the content anchor + * catches up (r975 needed + * ~78 s for the anchor; the + * walk answered in seconds). + */ + if (kick_badpg && + wap == SVC_APERTURE_SYS && + svc_kick_canonical(vb, len)) + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), + wpa, + SVC_APERTURE_SYS); if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30760,7 +32223,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) va, pb, len) && svc_method_stream_plausible(pb, - len)) { + len) && + (!(svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128) || + svc_kick_canonical(pb, + len))) { ch->rescue_last = 0; ch->push_hold_until = 0; svc.gpfifo_rescued++; @@ -32611,6 +34079,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[k].gr_ram_userd_watch[wj] = @@ -32619,6 +34091,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_watch_put[wj]; svc.chan[k].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } } svc.chan[k].gr_ram_put_seeded = @@ -32647,6 +34121,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_refresh_at = ch->gr_ram_refresh_at; svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].notify2d_va = ch->notify2d_va; svc.chan[k].zero_held = ch->zero_held; |