diff options
| -rw-r--r-- | kernel/gsp_cuda_bridge.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 585 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 77 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 1640 |
5 files changed, 2195 insertions, 111 deletions
diff --git a/kernel/gsp_cuda_bridge.h b/kernel/gsp_cuda_bridge.h index 2fb63d7..243392e 100644 --- a/kernel/gsp_cuda_bridge.h +++ b/kernel/gsp_cuda_bridge.h @@ -15,7 +15,7 @@ struct nv_vgpu_device; * targeted resolve). */ #define GSP_BRIDGE_INLINE_MAX 16 -#define GSP_BRIDGE_INLINE_BYTES 64 +#define GSP_BRIDGE_INLINE_BYTES 256 struct gsp_bridge_inline { u64 dst_va; diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f7b43fb..afe92b6 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -4959,6 +4959,96 @@ static u32 gsp_phys_engine_type_from_errstr(const char *err) } /* + * SESSION 153-CONTINUED-12: last published GR launch identity, set by the + * single-inflight submit path so a PBDMA fault dump can name the entry the + * PBDMA is stuck on (block length in dwords, guest push VA, object handle). + */ +static u32 gsp_phys_gr_last_words; +static u32 gsp_phys_gr_last_h; +static u64 gsp_phys_gr_last_push_va; + +/* + * SESSION 153-CONTINUED-12: on a robust-channel PBDMA error (exceptType + * 0x20) dump every PBDMA's latched instruction state and, for the PBDMA + * whose PB_FETCH sits inside the module pushbuffer page, the raw block words + * around the fetch point plus the whole block with indices. The latched + * raw instruction (HDR_SHADOW) can be matched against the block dump to + * name the exact word the PBDMA rejected ("invalid method encoding"); the + * fetch position shows how far the walk got. Read-only MMIO + a racy read + * of the pb page (best effort, diagnostic only); bounded per boot. + */ +#define GSP_PHYS_PBDMA_DUMP_MAX 6u + +static const char *gsp_phys_pbdma_type_name(u32 t) +{ + static const char *names[8] = { "SSDM", "INC", "STORE_SDM", "NON_INC", + "IMMD", "INC_ONCE", "USE_SDM", "END_SEG" }; + + return names[t & 7u]; +} + +static void gsp_phys_pbdma_fault_dump(u32 chid) +{ + static u32 dump_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 pb_va = boot_stage.rmcall_va_pb; + u32 i; + + if (!bar0 || !gsp_phys_gr_last_words || dump_n >= GSP_PHYS_PBDMA_DUMP_MAX) + return; + dump_n++; + pr_info("vgsp-gr: PB-FAULT-DUMP chid=%u last_block words=%u h=0x%x push=0x%llx pb_va=0x%llx\n", + chid, gsp_phys_gr_last_words, gsp_phys_gr_last_h, + gsp_phys_gr_last_push_va, pb_va); + for (i = 0; i < 12; i++) { + u32 b = 0x40000u + i * 0x2000u; + u32 intr0 = readl(bar0 + b + 0x108u); + u32 gpf = readl(bar0 + b + 0x050u); + u32 pbf = readl(bar0 + b + 0x054u); + u32 pbfhi = readl(bar0 + b + 0x058u); + u32 pbhd = readl(bar0 + b + 0x084u); + u32 pbcnt = readl(bar0 + b + 0x088u); + u32 pbd0 = readl(bar0 + b + 0x08cu); + u32 hdrsh = readl(bar0 + b + 0x118u); + u32 fetch64 = ((pbfhi & 0xffu) << 32) | (pbf & ~3u); + + if (!intr0 && !pbf && !hdrsh && !pbhd) + continue; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x\n", + i, intr0, gpf, fetch64, pbfhi >> 11, + hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, + pbcnt, pbd0, hdrsh); + } + { + struct gsp_phys_boot_chunk *pbch = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 *pw = pbch->pages ? gsp_phys_chunk_page_va(pbch, 0) : NULL; + + if (pw && pb_va) { + u32 n = gsp_phys_gr_last_words < 384u ? + gsp_phys_gr_last_words : 384u; + u32 w; + + for (w = 0; w < n; w += 16) { + u32 m; + + pr_info("vgsp-gr: PB-FAULT-DUMP pb[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w, pw[w], w + 1 < n ? pw[w + 1] : 0, + w + 2 < n ? pw[w + 2] : 0, w + 3 < n ? pw[w + 3] : 0, + w + 4 < n ? pw[w + 4] : 0, w + 5 < n ? pw[w + 5] : 0, + w + 6 < n ? pw[w + 6] : 0, w + 7 < n ? pw[w + 7] : 0, + w + 8 < n ? pw[w + 8] : 0, w + 9 < n ? pw[w + 9] : 0, + w + 10 < n ? pw[w + 10] : 0, w + 11 < n ? pw[w + 11] : 0, + w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, + w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); + } + } + } +} + +/* * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault * epoch (the relay submit snapshot compares against it) and log the designed * observation line when the fault relay is armed. Decoding and counting are @@ -4978,6 +5068,8 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); + if (out->except_type == 0x20u) + gsp_phys_pbdma_fault_dump(out->chid); } /* Note every consumed 0x1020 for the C3 corroboration ring. */ @@ -15259,10 +15351,24 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * site passes false, so the default COPY0 ladder emits the same bytes as * before. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +/* + * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer + * page the method block is written and fetched from. The warm-semaphore + * probe (gsp_phys_rmcall_gr_sem_push_locked) uses the upper half so it can + * never overwrite a forwarded launch's pending method block: measured with + * the PBDMA dump, the probe's 10-dword block landing on words 0..9 of a + * pending 139-word 3D replay made the PBDMA walk the probe's + * SET_OBJECT/REPORT_SEM tail and then decode the replay window's operand + * 0x00230200 as an instruction -> PBENTRY on chid 4 and no retire. Every + * forwarded GR/CE block stays at offset 0 (<= 512 dwords = 0x800 bytes by + * the GSP_PHYS_RMCALL_GR_METHOD_WORDS bound). + */ +static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, + u64 src_va, u64 dst_va, + u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + u32 pb_off, + struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; @@ -15291,11 +15397,16 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; + if (pb_off & 3u) + return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; - pb_va = gsp_phys_chunk_page_va(pb, 0); + if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) + return -EIO; + pb_va = (u32 *)((u8 *)gsp_phys_chunk_page_va(pb, 0) + pb_off); + pb_gpu_va += pb_off; ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || @@ -15316,9 +15427,22 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, dst_va, bytes, sem_payload); + if (pb_off + words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); } +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, + sem_va, bytes, sem_payload, + gr, 0u, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -15376,12 +15500,22 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) unsigned long deadline; int ret; - ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + /* + * SESSION 153-CONTINUED-12: this probe runs while a forwarded launch + * may still be pending in the same pushbuffer page. It must not land + * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's + * 10-dword block overwriting a pending 139-word 3D replay's head, so + * the PBDMA walks the probe then decodes a replay operand as an + * instruction (PBENTRY, no retire). Use the upper half (0x800); the + * forwarded block is bounded to 512 dwords = 0x800 bytes. + */ + ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, + GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, - &sub); + 0x800u, &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", ret); @@ -19096,7 +19230,7 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + GSP_PHYS_FAULT_NOTE_MAX - 1u) % GSP_PHYS_FAULT_NOTE_MAX]; - pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.200s\"\n", ok ? 1u : 0u, faults, fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, @@ -19343,22 +19477,193 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, * number of relocated pair(s). Run widths follow the service's decoder: * op 1/3/5 are 1+count dwords, everything else resyncs one dword. */ +/* + * SESSION 153-CONTINUED-13 (draw-window strip experiment): comma-separated + * hex NVC597 method byte-addresses to REMOVE from a captured 3D window + * before the verbatim replay (whole methods - a dropped run takes its + * count data words with it). Answers whether a DRAW window's non-retire + * lives in the draw content (BEGIN/END/immediate vertices/DRAW_ARRAY) or + * in the state binds/method-stream shape. Empty = off (historical + * behaviour); live-tunable via sysfs so one boot can sweep the variants. + */ +static char *gsp_phys_rmcall_gr_win_drop; +module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); +MODULE_PARM_DESC(sema_gl_win_drop, + "Hex method byte-addresses (comma-separated) to strip from a captured 3D window before replay (draw isolation; default empty)"); + +static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) +{ + const char *p = gsp_phys_rmcall_gr_win_drop; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0; + u32 digits = 0; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 d; + + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | d; + p++; + digits++; + } + if (!digits) + break; + n++; + if (v == addr) + return true; + } + return false; +} + +/* + * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch + * the DATA of one method inside a replayed 3D window before it reaches the + * engine. Format: comma-separated `addr:data` hex pairs, addr = method + * byte-address (e.g. 1618 = BEGIN, 1614 = END), data = the replacement + * value. Header-only methods (IMMD) get their 13-bit data field rewritten + * in place; count-carrying methods (INC/NON_INC/INC_ONCE) get their FIRST + * payload word replaced. Live-tunable via sysfs so one boot can sweep + * variants (BEGIN OP=7 QUADS -> TRIANGLES etc.) against the draw-stall. + */ +static char *gsp_phys_rmcall_gr_win_patch; +module_param_named(sema_gl_win_patch, gsp_phys_rmcall_gr_win_patch, charp, + 0644); +MODULE_PARM_DESC(sema_gl_win_patch, + "Hex addr:data pairs (comma-separated) patched into a captured 3D window method before replay (default empty)"); + +static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) +{ + const char *p = gsp_phys_rmcall_gr_win_patch; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0, d = 0; + u32 digits = 0; + bool have_data = false; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | h; + p++; + digits++; + } + if (!digits) + break; + if (*p == ':' || *p == '=') { + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (d <= 0xfffffffu) + d = (d << 4) | h; + p++; + have_data = true; + } + } + n++; + if (have_data && v == addr) { + *out_data = d; + return true; + } + } + return false; +} + static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + u32 *out_n, u32 *n_dropped, const struct gsp_phys_gr_submit *req, u32 *n_shadow) { - u32 off = 0, reloc = 0; + u32 off = 0, wout = 0, reloc = 0, dropped = 0; - memcpy(out, in, (size_t)n * 4); + *out_n = n; + *n_dropped = 0; while (off < n) { - u32 hdr = out[off]; + u32 hdr = in[off]; u32 op = (hdr >> 29) & 0x7u; u32 cnt = (hdr >> 16) & 0xfffu; u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + if (gsp_phys_rmcall_gr_win_drop_match(addr)) { + dropped++; + off += span; + continue; + } + memcpy(&out[wout], &in[off], (size_t)span * 4); + { + /* + * SESSION 153-CONTINUED-14: live data patch (BEGIN + * OP sweep etc.). Header-only forms rewrite the 13-bit + * data field; count-carrying forms rewrite the first + * payload word. + */ + u32 pv; + + if (gsp_phys_rmcall_gr_win_patch_find(addr, &pv)) { + static u32 patch_log; + u32 old; + if (span > 1u) { + old = out[wout + 1]; + out[wout + 1] = pv; + } else { + old = (out[wout] >> 16) & 0x1fffu; + out[wout] = (out[wout] & ~0x1fff0000u) | + ((pv & 0x1fffu) << 16); + } + if (patch_log < 32) { + patch_log++; + pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", + addr, off, old, pv); + } + } + } if ((op == 1 || op == 3 || op == 5) && cnt && off + 1u + cnt <= n) { - u32 *d = &out[off + 1]; + u32 *d = &out[wout + 1]; u32 lo_i = 0; bool pair = false; @@ -19375,16 +19680,58 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, addr < 0x4000 && (addr & 0x3fu) == 0x14 && cnt >= 2) { pair = true; + } else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) && + cnt >= 3) { + /* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_ + * A_FORMAT(j)+BASE(j) run [fmt, upper, lower] + * (j*0x10 stride) - the vertex data the draw + * reads. Observed [00001000 1 2150b790]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr == 0x2380 && cnt >= 3) { + /* SET_CONSTANT_BUFFER_SELECTOR_A/B: + * [size, upper, lower]; observed + * [00010000 1 2150a500]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 2) { + /* SET_COLOR_TARGET_A/B(j) offset pair: + * [upper, lower, w, h, fmt]; the surface the + * draw rasterizes into (must live in guest + * RAM for the present lane to see pixels). + * Observed [1 222fa200 40 4 f7 0]. */ + lo_i = 0; + pair = true; } - if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i], - &d[lo_i + 1], - req, n_shadow)) - reloc++; + if (pair) { + u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu); + bool ok = gsp_phys_rmcall_gr_3d_reloc( + &d[lo_i], &d[lo_i + 1], req, + n_shadow); + static u32 pair_log; + + if (ok) + reloc++; + if (pair_log < 48) { + pair_log++; + pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n", + addr, pva, ok ? "" : + "MISS ", + ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu)); + } + } + wout += span; off += 1u + cnt; continue; } off++; /* header-only / resync, like the service decoder */ + wout++; } + *out_n = wout; + *n_dropped = dropped; return reloc; } @@ -19435,8 +19782,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; bool sem_gated = false; - static u32 sked_prev[11]; - static u32 sked_sample[11]; + static u32 sked_prev[14]; + static u32 sked_sample[14]; static u32 sked_tick; static bool sked_first; /* @@ -20177,6 +20524,48 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, memcpy(sh_page + (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-10: a stream3d (3D + * method-window) submission maps the engine's + * VAs straight at the GUEST page HPA + * (gsp_phys_rmcall_gr_map_shadow_locked + * installs the guest PTE - there is no + * shadow-chunk copy in that path), so the + * engine's I-fetch reads the guest page, not + * sh_page. The GLX program page is never + * written by the guest itself (the driver's + * uploads never ran), and the r945 evidence is + * exactly that: "Graphics Exception: Shader + * Program Header 11/18 Error" on the draw + * whose inline prog-hex blob had only reached + * this shadow copy. Author the inline bytes + * into the guest page too, emulating the + * upload that never executed. + */ + if (stream3d && pg->hpa) + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + stream3d && pg->hpa ? + 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + } copied++; } /* @@ -20600,14 +20989,56 @@ gr_page_unmapped: * held, before the method block is built. */ if (stream3d) { + u32 win_n = req->stream_n; + u32 ndrop = 0; u32 nrel = gsp_phys_rmcall_gr_3d_patch_window( - req->stream, req->stream_n, stream_patch, req, - &n_shadow); + req->stream, req->stream_n, stream_patch, &win_n, + &ndrop, req, &n_shadow); win3d = stream_patch; - win3d_n = req->stream_n; - pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n", - nrel, win3d_n, n_shadow); + win3d_n = win_n; + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u)\n", + nrel, win3d_n, n_shadow, ndrop); + if (ndrop) { + static u32 gr3d_drop_log; + + if (gr3d_drop_log < 32) { + gr3d_drop_log++; + pr_info("vgsp-gr: 3D win-drop h=0x%x push=0x%llx raw=%u kept=%u dropped=%u list=[%s]\n", + req->h_object, req->push_va, + req->stream_n, win3d_n, ndrop, + gsp_phys_rmcall_gr_win_drop ? + gsp_phys_rmcall_gr_win_drop : ""); + } + } + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 4) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, @@ -20615,6 +21046,10 @@ gr_page_unmapped: ch_sem_va, want_ch, win3d, win3d_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ + WRITE_ONCE(gsp_phys_gr_last_words, words); + WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); + WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); @@ -20997,10 +21432,20 @@ gr_page_unmapped: sked_sample[8] = ioread32(bar0 + 0x00409b00u); sked_sample[9] = ioread32(bar0 + 0x00409b04u); sked_sample[10] = ioread32(bar0 + 0x00409400u); + /* + * SESSION 153-CONTINUED-14: FECS/FE ctxsw handshake + * lanes. fe0 = FECS_CTXSW_STATUS_FE_0 (0x409c00), + * fehost = FECS_HOST_INT_STATUS (0x409c18), fecsirq + * = FECS_IRQSTAT (0x409008). If the FECS dies in a + * switch, it should die visibly here first. + */ + sked_sample[11] = ioread32(bar0 + 0x00409c00u); + sked_sample[12] = ioread32(bar0 + 0x00409c18u); + sked_sample[13] = ioread32(bar0 + 0x00409008u); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -21008,7 +21453,8 @@ gr_page_unmapped: sked_sample[4], sked_sample[5], sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], - sked_sample[10]); + sked_sample[10], sked_sample[11], + sked_sample[12], sked_sample[13]); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; @@ -21021,6 +21467,50 @@ gr_page_unmapped: else w0 = READ_ONCE(*(u32 *)sem_va); } + /* + * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D + * stream launch - where is the front end while a DRAW window + * does not retire? One shot per launch at ~150 ms and ~1.2 s + * (5 ms poll), bounded per boot. pbfetch inside the module + * pushbuffer page = the walk/exec is stuck in our block; + * pbfetch past it = the whole stream was decoded and the + * stall is in execution/completion. + */ + if (stream3d && bar0 && + (sked_tick == 30 || sked_tick == 240)) { + static u32 winprobe_log; + u32 pi; + + if (winprobe_log < 10) { + winprobe_log++; + pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, + req->push_va, words, sub.pb_dma, + ioread32(bar0 + 0x00400504u), + ioread32(bar0 + 0x00400704u), + ioread32(bar0 + 0x00400708u)); + for (pi = 0; pi < 12; pi++) { + u32 b = 0x40000u + pi * 0x2000u; + u32 intr0 = ioread32(bar0 + b + 0x108u); + u32 gpf = ioread32(bar0 + b + 0x050u); + u32 pbf = ioread32(bar0 + b + 0x054u); + u32 pbfhi = ioread32(bar0 + b + 0x058u); + u32 hdrsh = ioread32(bar0 + b + 0x118u); + u64 fetch64 = + ((u64)(pbfhi & 0xffu) << 32) | + (u64)(pbf & ~3u); + + if (!intr0 && !gpf && !pbf && !hdrsh) + continue; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx hdr=%08x(%s m=0x%x cnt=%u sub=%u)\n", + pi, intr0, gpf, fetch64, hdrsh, + gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, + (hdrsh >> 13) & 7u); + } + } + } if (ch_rel_hpa) w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off)); @@ -21456,6 +21946,16 @@ gr_page_unmapped: ioread32(bar0 + 0x00405b00u), ioread32(bar0 + 0x00400100u), ioread32(bar0 + 0x0040013cu)); + if (bar0) + pr_info("vgsp-gr: FECS readback: CUR=0x%08x NEW=0x%08x ST1=0x%08x FE0=0x%08x HOSTINT=0x%08x IRQSTAT=0x%08x CPUCTL=0x%08x OS=0x%08x\n", + ioread32(bar0 + 0x00409b00u), + ioread32(bar0 + 0x00409b04u), + ioread32(bar0 + 0x00409400u), + ioread32(bar0 + 0x00409c00u), + ioread32(bar0 + 0x00409c18u), + ioread32(bar0 + 0x00409008u), + ioread32(bar0 + 0x00409100u), + ioread32(bar0 + 0x00409080u)); if (bar0) { u32 g; @@ -24670,6 +25170,36 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -25382,8 +25912,9 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index dfdf1d9..b6c3930 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -354,7 +354,7 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, */ #define GSP_PHYS_GR_MAX_PAGES 16u #define GSP_PHYS_GR_MAX_INLINE 8u -#define GSP_PHYS_GR_INLINE_BYTES 64u +#define GSP_PHYS_GR_INLINE_BYTES 256u #define GSP_PHYS_GR_QMD_WORDS 64u /* * IQ2M sentinel: written to the inline-QMD destination's word 63 after the diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 7b7e230..653ff1d 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3143,6 +3143,24 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM + GSPRM_CHAN_MEM_APERTURE, 4); + /* + * SESSION 153-CONTINUED-15 (draw attribution): every channel-class + * allocation is named with its wire shape before registration, so a + * channel the service silently refuses (gpfifo VA 0 / entries < 2) + * cannot hide behind an absent "registered" line. Bounded. + */ + { + static u32 chan_alloc_log; + + if (chan_alloc_log < 32) { + chan_alloc_log++; + pr_info("vgsp-rm: chan alloc hObject=0x%x gpfifo=0x%llx ent=%u eng=%u vas=0x%x cid=%u userd=0x%llx/%u notif=0x%llx/%u\n", + h_object, desc.gpfifo_va, desc.gpfifo_entries, + desc.engine_type, desc.h_vaspace, desc.cid, + desc.userd_pa, desc.userd_as, + desc.notifier_pa, desc.notifier_as); + } + } gsp_sema_service_channel_add(&desc); } @@ -3505,6 +3523,19 @@ MODULE_PARM_DESC(gsp_rm_binapi07_fwd, "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); /* + * SESSION 153-CONTINUED-17: reply word (little-endian u16) for BINAPI + * 0x20810107. The guest GL client reads reply byte[1] as a feature flag + * (glcore+0xFC74E1: setne <ctx>+0x203 after cmpb $0x0,reply[1]) and the + * flag gates method emission at glcore+0xB5B48B (see the control handler). + * 0x0101 sets both bytes; 0x0100 sets byte[1] only. Default 0 = the + * historical {0,0} reply. + */ +static uint gsp_rm_binapi07_rep; +module_param_named(binapi07_rep, gsp_rm_binapi07_rep, uint, 0644); +MODULE_PARM_DESC(binapi07_rep, + "reply word (LE u16) for BINAPI cmd 0x20810107; the GL client reads byte[1] as a feature flag (default 0)"); + +/* * session 75: reply words for the client-pending control 0x2080a097 (the * libnvidia-eglcore engine-readiness poll). Runtime-tunable so one boot can * sweep reply variants: the client reads +0x204, +0x488 and +0x48c back @@ -4510,6 +4541,52 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, out_params_size = req.paramsSize; break; + case 0x20810107: { /* NV2081_BINAPI reserved-interface method 7 */ + /* + * SESSION 153-CONTINUED-17: the guest GL client + * (libnvidia-glcore 570.144, file offset 0xFC74A6) sends this + * 2-byte query right after allocating its BINAPI object and + * CONSUMES REPLY BYTE [1] AS A FEATURE FLAG: + * + * glcore+0xFC74D9: cmpb $0x0, 0x131(%rsp) + * glcore+0xFC74E1: setne 0x203(%r14) ; flag = replies[1] + * + * and the flag gates 3D method emission into the command + * buffer: + * + * glcore+0xB5B48B: mov 0x1248(%r14),%rax + * cmpb $0x0,0x203(%rax) + * je <skip> + * glcore+0xB5B49B: movabs $0xf020010e66,%rdi + * mov %rdi,(0x80(%r14)) ; emit + advance + * + * With the historical {0,0} generic reply the flag stays 0 and + * the client's submission pipeline never activates (no GL + * channel, no pushes, black window). binapi07_rep selects the + * reply word (LE u16) at runtime so one boot can test the + * feature without a rebuild: 0x0101 = both bytes set, + * 0x0100 = byte[1] only. Default 0 reproduces {0,0}. + */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 2 && req.paramsSize <= 16) { + memcpy(rp, pp, req.paramsSize); + rp[0] = gsp_rm_binapi07_rep & 0xff; + rp[1] = (gsp_rm_binapi07_rep >> 8) & 0xff; + out_params_size = req.paramsSize; + } else { + out_params_size = 0; + } + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI 0x20810107 reply in=%u rep=0x%04x bytes=%02x %02x", + req.paramsSize, gsp_rm_binapi07_rep, + rp[0], rp[1]); + out_status = 0; + break; + } + case TU104_CTRL_I2C_TRANSACTION: /* I2C transactions have variable-length FINN-serialized * params: an all-zero reply fails the driver's DeserializeUp diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2ab1060..3bb883d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -205,8 +205,28 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* + * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record pushbuffer VA. Constant across every measured boot (r908 + * documented the GL client ring pair {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every + * canonical push read named exactly this VA). Used to discriminate the + * kick ring page from shape lookalikes in the RAM ring scan - r978 + * latched a foreign table page (pa=0x104fa9000) whose entry pair passed + * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000) + * while the true ring (pa=0x1168ce000, found by scanning for the + * record's own entry pair) sat later in RAM scan order. + */ +#define SVC_GL_KICK_PUSH_VA 0x1203c0000ULL /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ -#define SVC_GR_USERD_WATCH 8 +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 /* * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM @@ -612,7 +632,23 @@ struct svc_channel { u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ /* * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The @@ -666,6 +702,14 @@ struct svc_channel { */ bool gr_ram_ring; /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring + * liveness re-validation pace for the kick channel (h 0xbfef00xx). + * While the kick USERD stays unresolved this channel's ring latch is + * re-checked each pass (at most once a second) and re-armed when it + * no longer decodes entries - see svc_kick_ring_revalidate. + */ + unsigned long gr_kick_ring_reval_at; + /* * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the * registered/ring-derived candidates all fail; the scan validates the * hit against the ring entries the put claims, so a foreign channel's @@ -1171,6 +1215,10 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 kick_badpg; /* r977: kick-family push reads + * that lacked the canonical + * submit-record prefix (wrong + * page; forced the rescue) */ u64 rescue_map_ok; /* SESSION 148: content-verified * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -1639,8 +1687,27 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int i; if (!desc || !desc->h_object || !desc->gpfifo_va || - desc->gpfifo_entries < 2) + desc->gpfifo_entries < 2) { + /* + * SESSION 153-CONTINUED-15 (draw attribution): a refused + * channel registration used to be silent, so a real client + * channel with an empty gpfifo VA looked identical to a + * channel that was never allocated. Name every refusal. + */ + static u32 refuse_log; + + if (refuse_log < 32) { + refuse_log++; + pr_info("sema-svc: chan add REFUSED h=0x%x gpfifo=0x%llx ent=%u eng=%u userd=0x%llx/%u\n", + desc ? desc->h_object : 0, + desc ? desc->gpfifo_va : 0, + desc ? desc->gpfifo_entries : 0, + desc ? desc->engine_type : 0, + desc ? desc->userd_pa : 0, + desc ? desc->userd_as : 0); + } return; + } if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; @@ -3452,6 +3519,7 @@ static void svc_pin_drop(u64 va) * every push was "implausible"). */ static inline bool svc_pool_sem_va(u64 va); +static bool svc_kick_family(const struct svc_channel *ch); static void svc_pool_pa_forget(u64 va); /* @@ -5525,9 +5593,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_owed_resolved, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, + svc.kick_badpg, svc.pool_alt_applied, svc.pool_residue, svc_wait_held, svc_wait_pass, svc_wait_giveup, svc_wait_ovf); @@ -6568,7 +6637,7 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_INLINE_BYTES 256 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -6821,8 +6890,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -7071,6 +7140,7 @@ struct svc_gl_up { u8 data[SVC_GR_INLINE_BYTES]; }; +#define SVC_GL_PROG_MAX 8 struct svc_gl_sec { bool seen; u32 subch; @@ -7080,6 +7150,18 @@ struct svc_gl_sec { * host-sem/2D ops whose VAs are not mapped in the * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ u64 rel_va; /* last VALID release decoded on the channel */ @@ -7109,9 +7191,146 @@ struct svc_gl_sec { */ u8 blob[SVC_GR_INLINE_BYTES]; u32 blob_len; + /* + * SESSION 153-CONTINUED-18: the kick notification this push carried + * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the + * C597 bind that opened this section). Executed at flush time by + * svc_kick_notify_exec when both the target and the NOTIFY are in + * this same push (the short variant carries the NOTIFY without the + * arm pair; it never executes and stays witness-only). + */ + u64 kick_notify_va; + u32 kick_notify_val; + bool kick_notify_seen; }; static struct svc_gl_sec svc_gl; /* service thread only */ /* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* + * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. + * + * A SET_OBJECT bind is sticky on real hardware: the class stays bound to + * the channel subchannel across pushes until another bind replaces it. The + * GLX server's C597 SET_OBJECT arrives once (the driver's init push at + * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s) + * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour + * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the + * SET_OBJECT-only recognizer below never opens a section for them and the + * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597 + * bind is seen - including catches of the init push - and treat any later + * method run on that pair, on engine_type==0, as C597 class space. + */ +static u64 svc_gl_att_h; /* channel whose subch carries the C597 bind */ +static u32 svc_gl_att_subch; +static u32 svc_gl_draw_log; /* bounded draw-section-open witness */ +/* + * SESSION 153-CONTINUED-13: the C597 attachment is per-channel. The single + * (h_object, subch) slot was overwritten by the RM channel's periodic + * C597-bind pushes (h=0xbfef0002, observed every ~7 s on this boot), after + * which the GLX client's draw pushes on 0x1000013 failed the BEGIN + * recogniser's `svc_gl_att_h == ch->h_object` test and no draw section was + * ever opened (a boot-level, order-dependent race: the r954 boot decoded + * no such pushes). A small table (first match wins, dedup on note) keeps + * every channel's bind sticky independently, so the draw channel's + * attachment cannot be stolen. + */ +#define SVC_GL_ATT_MAX 8 +static u64 svc_gl_att_tab_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_tab_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + /* refresh LRU position: move to slot 0 */ + if (i) { + memmove(&svc_gl_att_tab_h[1], + &svc_gl_att_tab_h[0], + (size_t)i * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], + &svc_gl_att_tab_sub[0], + (size_t)i); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + } + return; + } + if (!svc_gl_att_tab_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) { + /* table full: evict the oldest (last) slot */ + free = SVC_GL_ATT_MAX - 1; + } + memmove(&svc_gl_att_tab_h[1], &svc_gl_att_tab_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], &svc_gl_att_tab_sub[0], + (size_t)free); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + /* keep the legacy single slot pointing at the newest note (logging) */ + svc_gl_att_h = h; + svc_gl_att_subch = subch; +} + +static bool svc_gl_att_has(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) + return true; + return false; +} + +static u32 svc_gl_att_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return svc_gl_att_tab_sub[i]; + return 0xffu; +} + +static void svc_gl_att_clear(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + u32 n = SVC_GL_ATT_MAX - 1 - i; + + if (n) + memmove(&svc_gl_att_tab_h[i], + &svc_gl_att_tab_h[i + 1], + (size_t)n * sizeof(u64)); + if (n) + memmove(&svc_gl_att_tab_sub[i], + &svc_gl_att_tab_sub[i + 1], + (size_t)n); + svc_gl_att_tab_h[SVC_GL_ATT_MAX - 1] = 0; + svc_gl_att_tab_sub[SVC_GL_ATT_MAX - 1] = 0; + if (svc_gl_att_h == h) + svc_gl_att_h = svc_gl_att_tab_h[0]; + return; + } +} +/* * SESSION 153 (G-B gate: the client's own C597 draws must open sections). * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which * returns true when the push VA sits in the GR candidate ring - even when @@ -7247,39 +7466,74 @@ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ static char *sema_gl_prog_hex; module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); MODULE_PARM_DESC(sema_gl_prog_hex, - "explicit program bytes (hex) for the synthesized GL launch (<=64 B)"); + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; static u32 svc_gl_prog_len; -static bool svc_gl_prog_parsed; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; -static void svc_gl_prog_apply_param(void) +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) { - const char *h = sema_gl_prog_hex; u32 n, i; - if (svc_gl_prog_parsed) - return; - svc_gl_prog_parsed = true; + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; if (!h || !h[0]) return; n = (u32)strlen(h); if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { - pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n", - n); + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); return; } for (i = 0; i < n; i += 2) { unsigned int b; if (sscanf(h + i, "%2x", &b) != 1) { - pr_warn("sema-gl: prog-hex param bad at %u\n", i); + pr_warn("sema-gl: %s param bad at %u\n", tag, i); return; } - svc_gl_prog[i / 2] = (u8)b; + dst[i / 2] = (u8)b; } - svc_gl_prog_len = n / 2; - pr_info("sema-gl: prog-hex param: %u bytes stored\n", - svc_gl_prog_len); + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); } /* @@ -7320,6 +7574,61 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +/* + * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. + * + * The GL client's per-frame kick push on the kick-family channel + * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}] + * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA + * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16 + * status at +0xE). On real hardware the 2D engine writes a completed + * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) - + * that write is the client's per-frame completion edge; the lane used to + * log "flush skip" and drop it, leaving the member all-zero forever. This + * executor authors the NvNotification through the same trusted VA walk + * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies + * by readback. sema_kick_notify=0 disables (A/B); the witness line + * "sema-kick: exec ..." is bounded (first 48 + every 512th). + */ +static bool gsp_sema_kick_notify = true; +module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644); +MODULE_PARM_DESC(sema_kick_notify, + "Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)"); +static u32 svc_kick_log; /* bounded kick-notify witness */ +static u64 svc_kick_count; /* executor invocations */ +/* + * r982: the notify page a pending-marker hit proved, per knowledge the + * exec picked up. Kept in the service's own scope (not the channel + * snapshot): the kick channel is the only consumer and the entry is + * re-validated by a read before every use, so a stale PA can only cost + * one failed probe, never a write into freed memory. + */ +static struct { + u64 h; /* channel handle that proved the page */ + u64 pa; /* PA of the marker-proven notify page */ +} svc_kick_pa_cache[4]; + +static void svc_kick_pa_cache_store(u64 h, u64 pa) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (svc_kick_pa_cache[i].h == h) { + svc_kick_pa_cache[i].pa = pa; + return; + } + } + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (!svc_kick_pa_cache[i].h) { + svc_kick_pa_cache[i].h = h; + svc_kick_pa_cache[i].pa = pa; + return; + } + } + svc_kick_pa_cache[0].h = h; + svc_kick_pa_cache[0].pa = pa; +} + static bool gsp_sema_gl_draw_synth; module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); MODULE_PARM_DESC(sema_gl_draw_synth, @@ -7335,7 +7644,13 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, { u32 i, free = SVC_GL_HOLD_MAX; - if (!h || !prog || !param || !rel) + /* + * SESSION 153-CONTINUED-13: rel == 0 is admitted (a section that has + * real operands but flushed before the channel's release was decoded + * stashes them here); a later call with rel != 0 completes the entry + * and a rel == 0 call must NEVER wipe a stored release. + */ + if (!h || !prog || !param) return; for (i = 0; i < SVC_GL_HOLD_MAX; i++) { if (svc_gl_hold_tab[i].h == h) { @@ -7351,8 +7666,10 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, svc_gl_hold_tab[free].prog_va = prog; svc_gl_hold_tab[free].param_va = param; svc_gl_hold_tab[free].param_size = psize; - svc_gl_hold_tab[free].rel_va = rel; - svc_gl_hold_tab[free].rel_payload = rpay; + if (rel) { + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; + } } static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, @@ -7419,13 +7736,46 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) break; memcpy(&v[k], data + 4 * k, 4); } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. */ if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && - count >= 2 && !svc_gl.prog_va) { - svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) | - (u64)(v[1] & 0xfffffffc); + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } return; } switch (addr) { @@ -7510,12 +7860,383 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) } } return; + case 0x1c00: /* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run + * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex + * stream the draw reads positions/colours from. + * Observed 0x1c00 [00001000 00000001 2150b790] and + * 0x1c10 [00001008 00000001 215257c0] (SESSION + * 153-CONTINUED-10 draw-push decode). + */ + case 0x1c10: + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x2380: /* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper, + * lower]. Observed [00010000 00000001 2150a500]. + */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x0800: /* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER, + * OFFSET_LOWER, w, h, fmt, ...] - the surface the + * draw rasterizes into. Observed [00000001 222fa200 + * 00000040 00000004 000000f7 00000000]. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; default: return; } } /* + * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect + * the C597 operand fields of every method run on the section's subchannel. + * Used when a draw section opens at its BEGIN marker (0x1618): the state + * methods that precede the marker (vertex stream binds, colour target, + * constant buffer binds) were decoded before the section existed, so their + * VAs must be re-collected or the candidate's page set starves and the + * replay's shadow relocation leaves them pointing at unreachable guest + * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main + * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only. + */ +static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, + u32 att_subch) +{ + u32 o = start; + + while (o + 4 <= end && o + 4 <= len) { + u32 hdr, op, count, subch, addr; + + memcpy(&hdr, buf + o, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + if ((op == 1 || op == 3 || op == 5) && count && + o + 4 + count * 4 <= len) { + if (subch == att_subch) + svc_gl_field(addr, count, buf + o + 4, + len - o - 4); + o += 4 + count * 4; + continue; + } + o += 4; + } +} + +/* + * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame + * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write + * is a completed NvNotification: nonzero timestamp at +0x0 (ns since + * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0 + * at +0xE. Resolution is the release path's trusted walk + * (svc_lookup_trusted -> svc_write_ap), then readback verification. + */ +static bool svc_kick_family(const struct svc_channel *ch); +static u64 svc_push_pa_ceiling(void); +static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 val, + u64 push_va) +{ + u64 pa = 0, ts = ktime_get_real_ns(); + int ap = SVC_APERTURE_SYS; + u8 old[16], nw[16], rb[16]; + bool ok = false, changed = false; + u32 o0 = 0, n0 = 0; + int rc = -ENOENT; + const char *rsrc = "trusted"; + u32 kc = 0; + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + memcpy(nw + 8, &val, sizeof(val)); + if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { + int r1, r2; + + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + rsrc = "trusted-u"; + } + svc_cur_vas = save; + } + /* + * r978: r977's exec fires (27 calls) but authors the word in + * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads, + * while r973 - the boot where the kick consumption visibly + * advanced the client - wrote the RING-ADJACENT sysmem slot + * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring + + * 0x1000 + 0xb0; the r975 probe measured the identical rule, + * pa=0x104b640b0 over ring 0x104b63000). Both walk answers + * miss that page this boot, so before falling back to the + * shadow: (a) the general lookup (the discovered rm_mem map - + * r979's sema_probe proved it resolves this VA to SYS after + * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin + * for this VA (the guest pin stream is the client's own + * statement of the mapping), then (c) for a kick-family + * channel the ring-relative completion slot, computed from + * the live ring pin each call. + */ + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 mpa = 0; + int map_ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &mpa, &map_ap) && + svc_ap_of(map_ap) == SVC_APERTURE_SYS) { + pa = mpa; + ap = map_ap; + rsrc = "map"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + svc_ap_of(iap) == SVC_APERTURE_SYS) { + pa = ipa; + ap = iap; + rsrc = "ipin"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch && + svc_kick_family(ch)) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, + &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 cand = (ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE + + (va & (SEMA_PAGE_SIZE - 1)); + + if (cand < svc_push_pa_ceiling()) { + pa = cand; + ap = SVC_APERTURE_SYS; + rsrc = "ring+10b0"; + } + } + } + /* + * r981: the client's pending notification marker. r979/r980 + * measured the REAL slot's +0xC u32 reading 0x80000000 + * while the client waited (and 0 once completed; the write + * below clears it) - on the page the discovered map names, + * ring + 0x20b0 on both boots, one page ABOVE the r973-era + * ring + 0x10b0 heuristic the exec had been writing + * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs + * 0x12e0790b0 on r980 - the completion landed on the wrong + * page both boots). When the chosen candidate does not show + * the marker, probe the ring-relative alternatives and use + * the one that does; the readback below still verifies the + * write. A slot already completed (marker 0) keeps the + * resolved candidate unchanged. + * + * r982: remember the page a pending-marker hit named, per + * handle. r981's boot: exec n=1 found the marker page + * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the + * marker is 0 afterwards so every later exec fell back to + * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes) + * while the map/trusted stages were still catching up. The + * cached page is retried at the top of the next call (read + * first, used only while the read succeeds), so all later + * completions stay on the page the marker proved. + */ + if (ch && svc_kick_family(ch)) { + /* Retry the page a marker hit proved, if still readable. */ + for (kc = 0; kc < 4; kc++) { + u32 probe_buf = 0; + + if (svc_kick_pa_cache[kc].h != ch->h_object || + !svc_kick_pa_cache[kc].pa) + continue; + if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa, + SVC_APERTURE_SYS, &probe_buf, + sizeof(probe_buf)) && + probe_buf != 0xffffffffu) { + pa = svc_kick_pa_cache[kc].pa; + ap = SVC_APERTURE_SYS; + rsrc = "kickcache"; + } + break; + } + } + if (ch && svc_kick_family(ch)) { + u32 pend = 0; + + if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap), + &pend, sizeof(pend)) && + pend != 0x80000000u) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, + &ring_pa, &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 base = ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1)); + u64 cands[2]; + int i; + + cands[0] = base + + 2 * SEMA_PAGE_SIZE + off; + cands[1] = base + + SEMA_PAGE_SIZE + off; + for (i = 0; i < 2; i++) { + u32 m = 0; + + if (cands[i] >= + svc_push_pa_ceiling()) + continue; + if (!svc_read_ap(vgpu, + cands[i] + 0xc, + SVC_APERTURE_SYS, + &m, sizeof(m)) && + m == 0x80000000u) { + pa = cands[i]; + ap = SVC_APERTURE_SYS; + rsrc = i ? + "ring+10b0p" : + "ring+20b0p"; + svc_kick_pa_cache_store( + ch->h_object, + cands[i]); + break; + } + } + } + } else if (pend == 0x80000000u && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + /* + * The chosen candidate IS the marker page: + * prove it for the cache so later execs keep + * writing the same page (r981: after the + * single marker hit the fallback wrote two + * wrong ring-relative pages for 33 further + * execs). Only a SYS answer that showed the + * marker is cached; a zero read (already + * completed) caches nothing - unproven. + */ + svc_kick_pa_cache_store(ch->h_object, pa); + } + } + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); + if (!r1 && !r2 && + !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) { + rc = 0; + ok = true; + changed = memcmp(old, rb, sizeof(old)) != 0; + memcpy(&o0, old, 4); + memcpy(&n0, rb, 4); + } else { + /* CONT.18 probe: name the failing step. */ + rc = r1 ? (r1 < 0 ? r1 : -100) : + (r2 ? (r2 < 0 ? r2 : -101) : -102); + } + } + svc_kick_count++; + if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { + svc_kick_log++; + pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", + ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val, + (unsigned long long)ts, o0, n0, changed ? 1 : 0, + (unsigned long long)svc_kick_count, rc); + } + /* + * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting + * client's completion edge - post the channel event so its OS event + * fires (r934's lesson on the notify2d path: without the event the + * GL client's kick submit wait never completes and the app retries + * on a ~16-22 s timer; r972 measured exactly that cadence with the + * event missing here). Only for a real channel (the dev probe + * passes ch=NULL). + */ + if (ok && ch) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return ok; +} + +/* + * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify + * executor on an explicit VA against the service's own device. The + * execution MUST run in a kernel-thread context: vfio_iommu_type1's + * dma_rw only copies when current->mm == dma->mm or the caller is a + * kthread (kthread_use_mm); a sysfs write from a shell process has + * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The + * workqueue item below is a kthread - the same context class as the + * sema service thread that runs the real kick lane. + * echo 0x12000e0b0 > .../parameters/sema_kick_probe + */ +static u64 svc_kick_probe_va; +static void svc_kick_probe_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn); + +static void svc_kick_probe_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (vgpu) + svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0); +} + +static int nv_vgpu_sema_kick_probe_set(const char *val, + const struct kernel_param *kp) +{ + if (!val || kstrtoull(val, 0, &svc_kick_probe_va)) + return -EINVAL; + if (svc_kick_probe_va) + schedule_work(&svc_kick_probe_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = { + .set = nv_vgpu_sema_kick_probe_set, +}; + +module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200); +MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)"); + +/* * Close the open section: synthesize the QMD from the collected fields and * capture it as the forwarding candidate. The QMD word layout mirrors * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 @@ -7531,7 +8252,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 at_off) { struct svc_push_state s2; - struct gsp_bridge_inline pr; + struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; u32 n_recs = 0, k; /* * SESSION 149 R16 (M3): the inline attachment's source. Priority: @@ -7597,7 +8318,53 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { - if (gsp_sema_dbg && svc_gl_skip_log < 16) { + /* + * SESSION 153-CONTINUED-13: the release may be + * decoded AFTER this section in ring order + * (the r955 init: the 24 B release pushes sit + * behind the 1524 B init push in the ring, so + * every replay pass of the section flushed + * before svc_gl.rel_va was known). When the + * operands are real but only the release is + * missing, stash them for the channel so the + * next section/pass (by then the constant + * release stream has been decoded) captures + * with src=hold instead of losing the init + * capture - without which the guest's GLX + * init completion degrades and the client + * never emits its draw pushes at all. + */ + if (svc_gl.prog_va && svc_gl.param_va && + !svc_gl.rel_va) + svc_gl_hold_store(ch->h_object, + svc_gl.prog_va, + svc_gl.param_va, + svc_gl.param_size, + 0, 0); + /* + * SESSION 153-CONTINUED-18 (task 1): replace + * the old "flush skip" drop with NOTIFY + * EXECUTION when this push carried the kick + * member (SET_NOTIFY_A/B + NOTIFY in the same + * push). The section has no draw operands - + * it is the client's completion kick - so the + * GPU's notification write is the only work + * it asks for. + */ + if (gsp_sema_kick_notify && + svc_gl.kick_notify_seen && + svc_gl.kick_notify_va && + svc_kick_notify_exec(vgpu, ch, + svc_gl.kick_notify_va, + svc_gl.kick_notify_val, + push_va)) { + /* authored once; a replay pass must not + * re-author an unchanged member. */ + svc_gl.kick_notify_seen = false; + svc_gl.kick_notify_va = 0; + goto out; + } + if (gsp_sema_dbg && svc_gl_skip_log < 2048) { svc_gl_skip_log++; pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", ch->h_object, push_va, @@ -7748,27 +8515,41 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } } memset(blob, 0, sizeof(blob)); - blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, - sizeof(blob)); - if (!blob_rc) { - u32 z = 0; - - for (k = 0; k < 16; k++) - z |= blob[k]; - if (z) - blob_trusted = true; - } - if (blob_trusted) { - blob_n = sizeof(blob); - blob_src = "trusted-prog"; - } else if (svc_gl_prog_len) { + /* + * SESSION 153-CONTINUED-10: the explicit operator program override + * (sema_gl_prog_hex) wins over the trusted page read. The GLX + * server's per-frame CONTENT pushes bind their own program slots + * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated + * nonzero bytes at that offset; the R16 trusted-read priority + * accepts any nonzero 16 bytes as "the real program" and attached + * garbage, and the engine rejected the draw with "Graphics + * Exception: Shader Program Header 11/18 Error" (r944 fault + * evidence, push 0x120014578). An unset override keeps the + * historical read priority unchanged. + */ + if (svc_gl_prog_len) { blob_n = svc_gl_prog_len; memcpy(blob, svc_gl_prog, blob_n); blob_src = "prog-hex"; - } else if (svc_gl.blob_len) { - blob_n = svc_gl.blob_len; - memcpy(blob, svc_gl.blob, blob_n); - blob_src = "lid-fallback"; + } else { + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } } /* * r929 (Path A): a stream-derived capture is a 3D method window - the @@ -7776,12 +8557,69 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * uploads/binds, so the synthesized compute attachments (the inline * program record, and with it the prog-hex fallback) do not apply. * Draw-synth/hold captures keep the old compute path unchanged. - */ - if (blob_n && op_src[0] != 's') { - memset(&pr, 0, sizeof(pr)); - pr.dst_va = svc_gl.prog_va; - pr.len = blob_n; - memcpy(pr.data, blob, blob_n); + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); n_recs = 1; } /* @@ -7790,11 +8628,11 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * replays of the same (push_va, content hash). */ svc_gl_cap_store(push_va, svc_push_hash(buf, len)); - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u, op_src); + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); /* * SESSION 153: remember real operands for this channel so a later * draw-only section on the same channel can still capture. A @@ -7820,7 +8658,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, get_unaligned_le32(blob + 8), get_unaligned_le32(blob + 12)); } - svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, push_va); /* * r929 (Path A): a stream-derived capture (op_src=stream: the section @@ -7838,8 +8676,18 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * 448-word cap leaves headroom for the backend's 512-word method * block (2 bind words + the window + the 8-word release tail). * Draw-synth/hold captures are never marked. - */ - if (op_src[0] == 's' && svc_gr_cand.valid && + * + * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX + * server's per-frame CONTENT pushes carry the draw window but no + * program/param operands (the class bind is sticky; no per-push + * SET_OBJECT), so their sections capture with the channel's real + * operands adopted from the hold table (filled by the init + * capture). The window itself is still the REAL push bytes sliced + * from its own first C597 state method to its END - exactly the + * raw 3D method window the r929 replay lane wants; the synthesized + * compute fields are what stream_3d bypasses. + */ + if ((op_src[0] == 's' || op_src[0] == 'h') && svc_gr_cand.valid && len == svc_gr_cand.push_n * 4 && svc_gl.start_off + 8u <= at_off && (at_off - svc_gl.start_off) / 4u <= 448u) { @@ -12019,16 +12867,34 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, if (!ch->notify2d_va) return false; if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { - if (gsp_sema_dbg && svc_notify2d_log < 16) { + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { svc_notify2d_log++; pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", ch->h_object, ch->notify2d_va); } return false; } - if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) - return false; - svc_notify2d_done++; + /* + * SESSION 153-CONTINUED-18: read the first word for the witness + * (the kick member's timestamp slot: 0 = never GPU-written; the + * completed write makes it nonzero - see svc_kick_notify_exec). + */ + { + u32 o0 = 0; + + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0))) + o0 = 0xffffffff; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return false; + svc_notify2d_done++; + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n", + ch->h_object, ch->notify2d_va, pa, o0); + } + } /* * r834: pin the page that accepted the ack; the same stale-map * defense as the push-side pin in the decode. @@ -12045,11 +12911,6 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, */ if (completion) gsp_rm_post_channel_event(vgpu, ch->h_object); - if (gsp_sema_dbg && svc_notify2d_log < 16) { - svc_notify2d_log++; - pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n", - ch->h_object, ch->notify2d_va, pa); - } return true; } @@ -12300,6 +13161,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* SESSION 153-CONTINUED-10: draw-window tracker. First method of + * the current C597 run on the attached subchannel after the last + * foreign-subchannel (2D/present) run; see the BEGIN recognizer + * below. Per pass, like the rest of the decode state. */ + u32 gl_win_start = 0; /* m164: fresh CLASS-2 QMD witness for this decode pass. */ svc_gr_qmd_seen = 0; @@ -12430,6 +13296,55 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-12: attach on the draw marker itself. + * + * The section recogniser below needs the sticky C597 bind + * (svc_gl_att_h) that the boot-class GLX init push carries. On a + * boot where the class is bound channel-side (inside the emulated + * RM channel setup) and no decoded push re-emits SET_OBJECT C597, + * every draw-shaped push is dropped and no 3D replay ever runs. + * Adopt the first trusted push that carries an NVC597_BEGIN IMMD + * (op 4, addr 0x1618, subchannel 0) as the attached channel; the + * per-pass gl_win_start tracker then runs over that same push from + * its start, exactly as it does after an explicit bind, so the + * window keeps the pre-BEGIN state methods (vertex stream bases, + * colour target, constant-buffer selectors). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && trusted && + !svc_gl_att_has(ch->h_object, 0)) { + u32 o2 = 0; + + while (o2 + 4 <= len) { + u32 h2, op2, cnt2, sc2, ad2; + + memcpy(&h2, buf + o2, 4); + op2 = (h2 >> 29) & 0x7; + cnt2 = (h2 >> 16) & 0xfff; + sc2 = (h2 >> 13) & 0x7; + ad2 = (h2 & 0x1fff) << 2; + if (op2 == 4 && ad2 == 0x1618 && sc2 == 0) { + svc_gl_att_note(ch->h_object, 0); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: attach-on-BEGIN h=0x%x push=0x%llx off=0x%x (no explicit C597 bind seen)\n", + ch->h_object, push_va, o2); + } + break; + } + if (op2 == 4 || op2 == 7) { + o2 += 4; + continue; + } + if ((op2 == 1 || op2 == 3 || op2 == 5) && cnt2 && + cnt2 <= 0xfff && o2 + 4 + cnt2 * 4 <= len) { + o2 += 4 + cnt2 * 4; + continue; + } + o2 += 4; + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -12441,6 +13356,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on + * the attached C597 channel. The per-frame CONTENT pushes + * interleave the 2D/present ops (subchannel 3) and the 3D + * block (subchannel 0); the replayable 3D window starts at + * the first subchannel-0 method after the last subchannel-3 + * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are + * excluded: they name client semaphore VAs (0x1_2000c000) + * the replay channel cannot translate (the r929 v1 fault + * class), so they must stay outside the sliced window. + */ + if (gsp_sema_gl_xlate && !ch->engine_type) { + u32 as = svc_gl_att_subch_of(ch->h_object); + + if (as != 0xffu) { + if (subch != as) + gl_win_start = 0; + else if (!gl_win_start && + !(op == 1 && addr >= 0x1b00 && + addr <= 0x1b0c)) + gl_win_start = off; + } + } + + /* * fail4: name every big method-data run and every run whose * address is the LOAD_INLINE_DATA method (any op/subch - the * encoding of the 64 KB H2D upload is what we are pinning). @@ -12676,6 +13615,68 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); } + /* + * SESSION 153-CONTINUED-10 (task 1): the client's per-frame + * CONTENT pushes carry real C597 draws but no per-push + * SET_OBJECT (the class bind is sticky), so the section + * recognizer above never opened for them and the 3D replay + * lane never saw a draw. Recognise the draw marker itself: + * NVC597_BEGIN (0x1618, an IMMD method - its immediate is + * the topology in the count field) on the attached + * subchannel of the attached channel opens the section. + * The window starts at gl_win_start (the first C597 state + * method after the last foreign-subchannel run); the state + * methods before the marker are re-scanned so their operand + * VAs (vertex stream bases 0x1c00/0x1c10, colour target + * 0x0800, constant buffer selectors 0x2380) enter the + * candidate's page set - without them the backend's shadow + * relocation cannot rewrite the window and the engine + * faults on an unreachable guest VA (the r929 v2 class). + * The window then runs from gl_win_start to the pass end + * (the section close at the END method / push tail), which + * svc_gl_flush slices into the verbatim 3D replay. + */ + if (gsp_sema_gl_xlate && !ch->engine_type && op == 4 && + addr == 0x1618 && !svc_gl.seen && + svc_gl_att_has(ch->h_object, subch)) { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + u32 wstart = gl_win_start ? gl_win_start : off; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = wstart; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + if (wstart < off) + svc_gl_scan_window(wstart, off, buf, len, + subch); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n", + ch->h_object, push_va, subch, wstart, + off, len, trusted); + } + } + + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + svc_gl_att_has(ch->h_object, subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -12773,6 +13774,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl.start_off = off; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* SESSION 153-CONTINUED-10: the bind is sticky - + * remember it so the class-space recognizer + * (BEGIN 0x1618, below) accepts this channel's + * later per-push-CONTENT pushes that carry no + * SET_OBJECT of their own. + * SESSION 153-CONTINUED-13: per-channel table - + * a foreign channel's periodic C597 bind must + * not steal this channel's attachment. */ + svc_gl_att_note(ch->h_object, subch); /* * SESSION 153: witness every C597 section open * (bounded) - it is the recognizer-side proof @@ -12786,9 +13796,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->h_object, push_va, subch, trusted); } - } else if (svc_gl.seen) { - svc_gl_flush(vgpu, ch, &st, buf, len, - push_va, trusted, replay, off); + } else { + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + /* A different class bound over the attached + * subchannel replaces the C597 bind: drop the + * attachment so the class-space recognizer + * cannot misread methods of the new class. + * SESSION 153-CONTINUED-13: clears only this + * channel's entry. */ + svc_gl_att_clear(ch->h_object, subch); } } else if (gsp_sema_gl_xlate && svc_gl.seen && subch == svc_gl.subch) { @@ -12951,6 +13970,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (count >= 2) ch->notify2d_va |= (u64)v1; /* + * SESSION 153-CONTINUED-18 (task 1): when + * this push also carries the C597 kick + * section, remember the arm so the section + * flush can execute the NOTIFY write (the + * 2D ack below is the watchdog's zero-ack; + * the kick member needs the completed + * NvNotification authored). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl.kick_notify_va = + ch->notify2d_va; + /* * r834: this method stream reached us through * a REAL read (svc_read_last_pa); pin its page * so the watchdog's push translation cannot go @@ -12971,6 +14002,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case SVC_2D_NOTIFY: if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + /* + * SESSION 153-CONTINUED-18: mark the + * kick section's NOTIFY (payload 0) so + * svc_gl_flush executes the completion + * write at the armed VA - the executor + * authors the NvNotification the 2D + * zero-ack alone does not. + */ + if (gsp_sema_gl_xlate && svc_gl.seen) { + svc_gl.kick_notify_val = v0; + svc_gl.kick_notify_seen = true; + } if (svc_notify2d_write(vgpu, ch, true)) { /* @@ -16758,10 +17801,18 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget * gate stays open while candidates are under watch, so the * liveness confirmation cannot be starved by slice accounting. + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. */ if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || - ch->gr_ram_userd_watch_n)) { + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -18590,6 +19641,22 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, continue; if (len1 < 4 || len1 > 0x100000) continue; + /* + * r979: kick-family ring discrimination. Shape alone + * admitted a foreign table page ahead of the real ring + * on r978 (pa=0x104fa9000, entries naming a bogus + * 0x12048b824/0x40000 push) while the true ring's + * entry pair names the client's per-frame record VA + * (SVC_GL_KICK_PUSH_VA, the r908-documented GL + * pushbuffer, constant across r908-r978). A + * kick-family candidate must name the record; the + * lookalike class then never latches. Non-kick + * channels keep the shape gate unchanged. + */ + if (svc_kick_family(ch) && + va0 != SVC_GL_KICK_PUSH_VA && + va1 != SVC_GL_KICK_PUSH_VA) + continue; *pa_out = gpa + o; return true; } @@ -18627,6 +19694,99 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance). + * + * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring + * found by the guest-RAM content scan) posts a per-frame completion kick + * the client's flush loop paces on. Its USERD is a sysmem control block + * the driver allocates far from the ring - r971 measured the ring at pa + * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the + * same nomination was +0.6..+2.3 GB across r966-r971) - while the general + * RAM USERD scan's allocation-locality gate only admits candidates within + * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939 + * nine-decoy class). That gate rejected the channel's real USERD on every + * boot since the gate landed, the scan retired ("RAM scan found no + * candidate in 20 slice(s)"), the pump never iterated the kick entries + * and the kick-notify lane never ran. For this family the ring-entry + * match (every PUT-claimed ring entry must decode to a pushbuffer VA + * inside the ring's own allocation window) is the ownership proof; the + * locality distance is not - so the gate is waived for these channels and + * the distance is logged instead. The pump's speculative-latch + * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final + * judge, and the rejected-latch memory prevents a frozen block from ever + * pinning the scan. + */ +static bool svc_kick_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu; +} + +/* + * Re-validate the kick channel's ring latch each pass (bounded to once a + * second per channel). The RAM content scan latches a ring page once; if + * the driver later zeroes or reuses that page (the r966-r971 class the + * design brief describes: the USERD scan then validates every candidate + * against a dead ring and adopts none), the latch is dropped here so the + * content scan re-runs and re-latches the channel's current ring page. + * The stale entry pair is logged on re-arm so the transition is visible + * in the record. A live ring (a first entry that still decodes to a + * window-bounded pushbuffer VA) is a cheap no-op. + */ +static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + unsigned long flags; + int i; + + if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok) + return; + if (!time_after(jiffies, ch->gr_kick_ring_reval_at)) + return; + ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000); + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + u64 eva; + u32 elen; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x100000) + return; /* ring page live; nothing to do */ + } + /* + * The latched page no longer decodes entries: drop the one-shot + * latch state so the RAM ring content scan re-runs. The pin is left + * in place - a stale pin only shadows VA reads, while the content + * scan validates against raw RAM and re-pins on a hit. + */ + clear_bit(ch->gr_index, &svc_gr_ring_done); + clear_bit(ch->gr_index, &svc_pb_gr_done); + ch->gr_ram_cursor = 0; + ch->gr_ram_passes = 0; + ch->gr_ram_logged = false; + ch->gr_ram_ring = false; + pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n", + ch->h_object, ch->gpfifo_va, e0, e1); + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + svc.chan[i].gr_ram_ring = ch->gr_ram_ring; + svc.chan[i].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). * Such a client's USERD is a system-memory object like its ring, so the @@ -18666,6 +19826,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, u64 live_pa = 0; u32 live_put = 0, live_get = 0, live_prev = 0; int ring_ap = SVC_APERTURE_SYS; + bool kick; u32 wi; int i; unsigned long flags; @@ -18684,6 +19845,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, if (!buf) return false; } + kick = svc_kick_family(ch); + /* + * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch + * before validating candidates against it (svc_kick_ring_revalidate + * re-latches a ring page the driver has zeroed since the latch; see + * its comment). + */ + svc_kick_ring_revalidate(vgpu, ch); (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); /* * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. @@ -18710,9 +19879,13 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * mapped by NO process, and never advanced on any pass * (two frozen snapshots that cost a 15 s reject/re-latch * cycle all boot). A control block this far from its own - * ring is not this channel's USERD. + * ring is not this channel's USERD. SESSION 153-CONT-19: + * waived for the kick family - its real USERD is measured + * 0.6-2.3 GB from the ring, and the ring-entry match (not + * proximity) is the ownership proof there (see + * svc_kick_family). */ - if (ring_pa && + if (ring_pa && !kick && (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) || wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) { ch->gr_ram_userd_watch_n--; @@ -18737,10 +19910,47 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } last = ch->gr_ram_userd_watch_put[wi]; if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } wi++; continue; } - if (hp != last + 1) { + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { /* * SESSION 154: a jump or a reset is the churn * signature (the r916-r919 lookalike pools move by @@ -18755,9 +19965,37 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; ch->gr_ram_userd_watch_hits[wi] = ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; continue; } ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } /* * SESSION 154: ONE observed +1 step adopts. The live block * advances ~2-8 s per step (measured) while the client's @@ -18814,9 +20052,15 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * latches read 12/8 and 16/16 and every previously * measured lookalike counted 16 or more. Also accept * a free-running put (the old put <= entries bound - * would reject the live block). + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. */ - if (!hp || hp > 8 || hg > hp) + if (!hp || hp > (kick ? 0x100000 : 8) || hg > hp) continue; { u32 z = 0, zi; @@ -18842,8 +20086,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * above. Excludes the r936-r939 nine-decoy pair * (~460 MB from the ring) before it can occupy a * watch slot and burn a reject/re-latch cycle. + * SESSION 153-CONT-19: waived for the kick family + * (h 0xbfef00xx) - its real USERD is measured far + * outside this window, so admission there is by the + * ring-entry match alone (the entry loop below) and + * the pump's +1/+2-step confirmation; the distance + * is logged at the latch. */ - if (ring_pa && + if (ring_pa && !kick && (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) || gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5))) continue; @@ -18915,6 +20165,64 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, hp == ch->gr_ram_userd_reject_put) continue; /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* * SESSION 154 (M-GL1.2): a candidate that passed the * shape gate, the entry validation, the page-density * gate and the put cap is latched SPECULATIVELY. The @@ -18942,6 +20250,23 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && !ch->gr_ram_userd_watch_n) { ch->gr_ram_userd_logged = true; @@ -18974,6 +20299,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_rejects; svc.chan[i].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[i].gr_ram_userd_watch[wj] = ch->gr_ram_userd_watch[wj]; @@ -18981,6 +20310,8 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[wj]; svc.chan[i].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } break; } @@ -19020,6 +20351,27 @@ static const u8 svc_gr_anchor_sig[16] = { }; #define SVC_GR_ANCHOR_SWEEPS 3 +/* + * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record is a fixed method stream opening with the SET_SUBDEVICE_MASK + * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the + * bytes r975's anchor pin answered and r973 decoded to n=512 execs). + * Every boot r966-r976 read that VA through some untrusted translation + * and, because a plausible-looking garbage page (r976: first8=9864be60 + * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the + * generic method-stream parse, the entry was consumed with the wrong + * bytes and the kick lane stayed silent for the whole session. For + * this family the canonical prefix IS the content identity: a read + * that lacks it came from a stale/foreign page no matter how + * plausible it parses, and must take the rescue chain (VAS walk -> + * content anchor -> pin). + */ +static bool svc_kick_canonical(const u8 *buf, size_t len) +{ + return len >= sizeof(svc_gr_anchor_pre) && + !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre)); +} + static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, u32 len) { @@ -29776,8 +31128,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * probably locked" while every other channel kept running. * Writing the ack on every pass that sees work pending is * idempotent. + * + * SESSION 153-CONTINUED-19: NOT for the kick family. Their + * notify2d_va is the kick record's own NOTIFY target (the + * shared completion word the GL client polls), and the kick + * lane authors that word (a completed NvNotification with a + * nonzero timestamp, svc_kick_notify_exec). The per-pass ack + * zeroed it 14029 times on r972 - the last kick write + * survived at most one pass (~60 ms) before being wiped, so + * the client's poll kept seeing "not fired". The GPU never + * clears this notification; the client does (re-arm). */ - if (!ch->engine_type && ch->notify2d_va && put != get) + if (!ch->engine_type && ch->notify2d_va && put != get && + !svc_kick_family(ch)) svc_notify2d_write(vgpu, ch, false); /* * SESSION 148 (r864): registration-based keep-alive that @@ -29870,6 +31233,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + /* * SESSION 154 (M-GL1.2): speculative-latch confirmation for * the RAM USERD. The scan latches a gated candidate right @@ -29880,13 +31247,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * candidate. The live block advances 1->2->3 in 2-10 s * (measured); the client's own release waits are ~11 s, so * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && - !ch->gr_ram_userd_confirmed) { + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + if (!ch->gr_ram_userd_latch_at) ch->gr_ram_userd_latch_at = jiffies; - if (put == ch->gr_ram_userd_latch_put + 1 || - put == ch->gr_ram_userd_latch_put + 2) { + stepped = kfam ? + put > ch->gr_ram_userd_latch_put : + (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2); + if (stepped) { ch->gr_ram_userd_confirmed = true; pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, @@ -30480,6 +31868,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } bool injected = false; bool allow_ops = (ch->engine_type != 0); + bool kick_badpg = false; /* * A resolved buffer that does not start @@ -30579,7 +31968,48 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) len >= 4 && len <= 16 && !memchr_inv(pb, 0, len); - if (!svc_method_stream_plausible(pb, len) && + /* + * r977/r979: a read of the + * kick family's per-frame + * record (SVC_GL_KICK_PUSH_VA; + * r978 showed the channel also + * carries bulk pushes that + * must NOT be held to this + * content identity - a 256 KB + * push at 0x12048b824) whose + * bytes lack the canonical + * submit-record prefix is a + * wrong page no matter how + * plausible it parses (r976: + * src=3 window garbage + * 9864be60 ffffffff consumed + * as a valid push and the kick + * lane ran zero execs). Force + * the implausible path so the + * VAS walk / content anchor / + * pin-drop chain below + * re-anchors the page. + */ + kick_badpg = !zero_nop && + svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128 && + !svc_kick_canonical(pb, len); + if (kick_badpg) { + svc.kick_badpg++; + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + } + + if ((!svc_method_stream_plausible(pb, len) || + kick_badpg) && !zero_nop) { svc.false_trans++; if (injected) { @@ -30657,6 +32087,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool pl = !rr ? svc_method_stream_plausible(vb, len) : false; + /* + * r977: for a kick-family + * entry a plausible walk + * answer still has to carry + * the canonical prefix - a + * wrong-VAS garbage page + * parses plausibly (r976) and + * would re-shadow the real + * page here. + */ + if (pl && kick_badpg && + !svc_kick_canonical(vb, len)) + pl = false; + if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30670,6 +32114,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, vb, len); svc.pushpg_fallback++; ok = true; + /* + * r977: a kick-family + * entry whose page the + * walk re-anchored is + * pinned so later passes + * read the canonical page + * directly (src=2) instead + * of re-paying the walk + * while the content anchor + * catches up (r975 needed + * ~78 s for the anchor; the + * walk answered in seconds). + */ + if (kick_badpg && + wap == SVC_APERTURE_SYS && + svc_kick_canonical(vb, len)) + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), + wpa, + SVC_APERTURE_SYS); if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30760,7 +32223,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) va, pb, len) && svc_method_stream_plausible(pb, - len)) { + len) && + (!(svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128) || + svc_kick_canonical(pb, + len))) { ch->rescue_last = 0; ch->push_hold_until = 0; svc.gpfifo_rescued++; @@ -32611,6 +34079,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[k].gr_ram_userd_watch[wj] = @@ -32619,6 +34091,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_watch_put[wj]; svc.chan[k].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } } svc.chan[k].gr_ram_put_seeded = @@ -32647,6 +34121,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_refresh_at = ch->gr_ram_refresh_at; svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].notify2d_va = ch->notify2d_va; svc.chan[k].zero_held = ch->zero_held; |