diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_cuda_bridge.h | 2 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1090 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 8 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 161 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 4 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 3998 | ||||
| -rw-r--r-- | kernel/main.c | 37 |
7 files changed, 5128 insertions, 172 deletions
diff --git a/kernel/gsp_cuda_bridge.h b/kernel/gsp_cuda_bridge.h index 2fb63d7..243392e 100644 --- a/kernel/gsp_cuda_bridge.h +++ b/kernel/gsp_cuda_bridge.h @@ -15,7 +15,7 @@ struct nv_vgpu_device; * targeted resolve). */ #define GSP_BRIDGE_INLINE_MAX 16 -#define GSP_BRIDGE_INLINE_BYTES 64 +#define GSP_BRIDGE_INLINE_BYTES 256 struct gsp_bridge_inline { u64 dst_va; diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f7b43fb..d78e0df 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -4959,6 +4959,96 @@ static u32 gsp_phys_engine_type_from_errstr(const char *err) } /* + * SESSION 153-CONTINUED-12: last published GR launch identity, set by the + * single-inflight submit path so a PBDMA fault dump can name the entry the + * PBDMA is stuck on (block length in dwords, guest push VA, object handle). + */ +static u32 gsp_phys_gr_last_words; +static u32 gsp_phys_gr_last_h; +static u64 gsp_phys_gr_last_push_va; + +/* + * SESSION 153-CONTINUED-12: on a robust-channel PBDMA error (exceptType + * 0x20) dump every PBDMA's latched instruction state and, for the PBDMA + * whose PB_FETCH sits inside the module pushbuffer page, the raw block words + * around the fetch point plus the whole block with indices. The latched + * raw instruction (HDR_SHADOW) can be matched against the block dump to + * name the exact word the PBDMA rejected ("invalid method encoding"); the + * fetch position shows how far the walk got. Read-only MMIO + a racy read + * of the pb page (best effort, diagnostic only); bounded per boot. + */ +#define GSP_PHYS_PBDMA_DUMP_MAX 6u + +static const char *gsp_phys_pbdma_type_name(u32 t) +{ + static const char *names[8] = { "SSDM", "INC", "STORE_SDM", "NON_INC", + "IMMD", "INC_ONCE", "USE_SDM", "END_SEG" }; + + return names[t & 7u]; +} + +static void gsp_phys_pbdma_fault_dump(u32 chid) +{ + static u32 dump_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 pb_va = boot_stage.rmcall_va_pb; + u32 i; + + if (!bar0 || !gsp_phys_gr_last_words || dump_n >= GSP_PHYS_PBDMA_DUMP_MAX) + return; + dump_n++; + pr_info("vgsp-gr: PB-FAULT-DUMP chid=%u last_block words=%u h=0x%x push=0x%llx pb_va=0x%llx\n", + chid, gsp_phys_gr_last_words, gsp_phys_gr_last_h, + gsp_phys_gr_last_push_va, pb_va); + for (i = 0; i < 12; i++) { + u32 b = 0x40000u + i * 0x2000u; + u32 intr0 = readl(bar0 + b + 0x108u); + u32 gpf = readl(bar0 + b + 0x050u); + u32 pbf = readl(bar0 + b + 0x054u); + u32 pbfhi = readl(bar0 + b + 0x058u); + u32 pbhd = readl(bar0 + b + 0x084u); + u32 pbcnt = readl(bar0 + b + 0x088u); + u32 pbd0 = readl(bar0 + b + 0x08cu); + u32 hdrsh = readl(bar0 + b + 0x118u); + u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + + if (!intr0 && !pbf && !hdrsh && !pbhd) + continue; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x\n", + i, intr0, gpf, fetch64, pbfhi >> 11, + hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, + pbcnt, pbd0, hdrsh); + } + { + struct gsp_phys_boot_chunk *pbch = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 *pw = pbch->pages ? gsp_phys_chunk_page_va(pbch, 0) : NULL; + + if (pw && pb_va) { + u32 n = gsp_phys_gr_last_words < 384u ? + gsp_phys_gr_last_words : 384u; + u32 w; + + for (w = 0; w < n; w += 16) { + u32 m; + + pr_info("vgsp-gr: PB-FAULT-DUMP pb[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w, pw[w], w + 1 < n ? pw[w + 1] : 0, + w + 2 < n ? pw[w + 2] : 0, w + 3 < n ? pw[w + 3] : 0, + w + 4 < n ? pw[w + 4] : 0, w + 5 < n ? pw[w + 5] : 0, + w + 6 < n ? pw[w + 6] : 0, w + 7 < n ? pw[w + 7] : 0, + w + 8 < n ? pw[w + 8] : 0, w + 9 < n ? pw[w + 9] : 0, + w + 10 < n ? pw[w + 10] : 0, w + 11 < n ? pw[w + 11] : 0, + w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, + w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); + } + } + } +} + +/* * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault * epoch (the relay submit snapshot compares against it) and log the designed * observation line when the fault relay is armed. Decoding and counting are @@ -4978,6 +5068,8 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); + if (out->except_type == 0x20u) + gsp_phys_pbdma_fault_dump(out->chid); } /* Note every consumed 0x1020 for the C3 corroboration ring. */ @@ -15168,7 +15260,35 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } - slot = gpput; + /* + * SESSION 153-CONTINUED-46: publish at the ENGINE's GP_GET, not at + * the raw GP_PUT. Observed on every stalled beef boot: the USERD + * drifts (GPGet=5 while GPPut cycles 1/2/4/7 - the engine's GET ran + * past the publisher's PUT, e.g. after a wrapped or otherwise + * deranged ring), and a ring is MODULO: PUT < GET numerically means + * (PUT-GET) mod ENTRIES pending entries - hundreds - so the PBDMA + * walks stale slots, executes garbage method blocks and the launch + * never retires (the trapped=0x8000022c / no-dispatch class). The + * forwarded channel is strictly single-inflight, so the engine's + * GET names exactly the next free slot: write our entry there and + * advance PUT to GET+1 so the PBDMA sees exactly one pending entry - + * ours. When GP_GET is out of range (uninitialised USERD), fall back + * to the old PUT-derived slot. + */ + slot = (gpget < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) ? gpget : gpput; + { + static u32 slotfix_log; + + if (gpget != gpput && slotfix_log < 64) { + slotfix_log++; + pr_info("vgsp-boot: rm-call: publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", + gpget, gpput, + gpput == gpget ? "equal" : + (gpput > gpget ? "ahead" : "behind"), + (gpput + GSP_PHYS_RMCALL_GPFIFO_ENTRIES - gpget) % + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + } + } /* * e0: bits 31:2 hold the GET field, which itself is (fetch address @@ -15195,8 +15315,10 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * GP_PUT advances on the client sysmem USERD chunk, which is the * channel's hardware USERD because the params carry that same chunk; * the host write is therefore what the engine polls (cycle 21). + * CONT-46: PUT = slot+1 (slot was chosen from GP_GET above), so the + * single new entry is the entire pending set. */ - put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + put_unaligned_le32((slot + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | @@ -15259,10 +15381,24 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * site passes false, so the default COPY0 ladder emits the same bytes as * before. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +/* + * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer + * page the method block is written and fetched from. The warm-semaphore + * probe (gsp_phys_rmcall_gr_sem_push_locked) uses the upper half so it can + * never overwrite a forwarded launch's pending method block: measured with + * the PBDMA dump, the probe's 10-dword block landing on words 0..9 of a + * pending 139-word 3D replay made the PBDMA walk the probe's + * SET_OBJECT/REPORT_SEM tail and then decode the replay window's operand + * 0x00230200 as an instruction -> PBENTRY on chid 4 and no retire. Every + * forwarded GR/CE block stays at offset 0 (<= 512 dwords = 0x800 bytes by + * the GSP_PHYS_RMCALL_GR_METHOD_WORDS bound). + */ +static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, + u64 src_va, u64 dst_va, + u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + u32 pb_off, + struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; @@ -15291,11 +15427,16 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; + if (pb_off & 3u) + return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; - pb_va = gsp_phys_chunk_page_va(pb, 0); + if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) + return -EIO; + pb_va = (u32 *)((u8 *)gsp_phys_chunk_page_va(pb, 0) + pb_off); + pb_gpu_va += pb_off; ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || @@ -15316,9 +15457,22 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, dst_va, bytes, sem_payload); + if (pb_off + words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); } +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, + sem_va, bytes, sem_payload, + gr, 0u, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -15376,12 +15530,22 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) unsigned long deadline; int ret; - ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + /* + * SESSION 153-CONTINUED-12: this probe runs while a forwarded launch + * may still be pending in the same pushbuffer page. It must not land + * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's + * 10-dword block overwriting a pending 139-word 3D replay's head, so + * the PBDMA walks the probe then decodes a replay operand as an + * instruction (PBENTRY, no retire). Use the upper half (0x800); the + * forwarded block is bounded to 512 dwords = 0x800 bytes. + */ + ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, + GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, - &sub); + 0x800u, &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", ret); @@ -16939,6 +17103,26 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, req->h_object, req->push_va, win3d_n, (u32)(p - pb)); } + /* + * SESSION 153-CONTINUED-45 diag: the exact block the + * PBDMA fetches (first/last window words). The beef + * family's replay schedule-stalls (grfifo frozen) while + * the X family's retires; this dump is the content A/B + * behind that divergence. Bounded per boot. + */ + if (gr3d_log < 16) { + u32 wi; + + pr_info("vgsp-gr: 3DSTREAM hdr=%08x %08x win0..9=", + pb[0], pb[1]); + for (wi = 0; wi < 10 && wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("| tail="); + for (wi = (win3d_n > 10 ? win3d_n - 10 : 0); + wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("\n"); + } } return (u32)(p - pb); } @@ -19096,7 +19280,7 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + GSP_PHYS_FAULT_NOTE_MAX - 1u) % GSP_PHYS_FAULT_NOTE_MAX]; - pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.200s\"\n", ok ? 1u : 0u, faults, fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, @@ -19291,6 +19475,64 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, } /* + * SESSION 153-CONTINUED-27: big-surface alias bookkeeping. A replayed + * 3D window's colour target (C597 INC 0x800+j*0x40 [upper, lower, w, h, + * fmt] and its TWOD twin INC 0x218 [w, h, upper, lower]) can be a surface + * far larger than the launch's 16-page set (the X screen 0x120400000, + * 1280x720x4 = 900 pages). Two earlier routes are dead ends for it: + * - one-page shadow relocation (r992: `pair addr=0x840 va=0x120400000 + * -> MISS` then FAULT_PDE @ +0xa000 - the raster walk crosses the + * single alias page); + * - Tier-1 windows at the surface's own VAs (CONT.27 r993: PDE_UPDATE + * for pdeIndex 2306/2307 status=0x0, yet GPC2/GPC5 still fault + * FAULT_PDE @ +0x4000/+0xa000 - same class as the r929 v2 FE faults; + * the GPC engines only see the scratch-PT pool). + * The surviving route is the proven one: a CONTIGUOUS run of never-used + * scratch-PT fetch lines (one line = 16 PTEs = 64 KB VA; lines are + * monotonic and consecutive lines are VA-contiguous inside each pool + * block), one PTE per surface page, so the engine reaches all 900 pages + * through one contiguous alias VA and the raster walk never crosses into + * an unmapped page. The pair operands are rewritten to <alias>|<offset> + * by gsp_phys_rmcall_gr_3d_reloc. The table is per-boot: the alias PTEs + * are never rewritten (fetch-line rule) and the screen's HPAs are fixed + * by its single allocation, so every later launch is a pure rewrite. + */ +struct gsp_phys_gr_surf_keep { + u64 va; /* guest surface base VA (page aligned) */ + u64 alias; /* engine-visible contiguous shadow alias base */ + u64 npages; /* pages aliased (declared + slack tail) */ +}; +static struct gsp_phys_gr_surf_keep gsp_phys_gr_surf[4]; +static u32 gsp_phys_gr_surf_n; + +/* + * SESSION 153-CONTINUED-27: slack tail for the alias. r994 measured the + * replay's colour writes landing past the declared w*h*4 footprint + * (1280x720x4 = 0x384000) by up to +0xE000 (11.2 rows) - FAULT_PTE on + * the last fetch line's unwritten PTE slots (0x5d6000..0x5e2000 with the + * alias at 0x250000). The tail pages are aliased too: a page that still + * resolves (inside the guest allocation's own padding - exactly where + * native overdraw would land) maps the real HPA; one that does not + * resolves to a dedicated zeroed module page so the stray writes vanish + * instead of faulting. The chunk is allocated lazily and lives until + * module unload (4 KB). + */ +#define GSP_PHYS_GR_SURF_SLACK_PAGES 32u +static struct gsp_phys_boot_chunk gsp_phys_gr_surf_dummy; + +static u64 gsp_phys_gr_surf_dummy_dma_locked(void) +{ + struct gsp_phys_boot_chunk *d = &gsp_phys_gr_surf_dummy; + + if (!d->pages) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, d, 1u)) + return 0; + gsp_phys_chunk_zero(d); + } + return (u64)gsp_phys_chunk_page_dma(d, 0); +} + +/* * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, * program, tex pools) are installed in the ladder page tree by the Tier-1 @@ -19313,6 +19555,25 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, if (!va || !page) return false; + /* + * SESSION 153-CONTINUED-27: a big surface (the X screen) is aliased + * as ONE contiguous run by the pre-pass; rewrite the operand to the + * alias base | offset. A one-page shadow relocation of it would + * break the raster walk at the first page crossing (r992), and the + * raw VA has no PTE chain the GPC engines can see (r993). + */ + for (i = 0; i < gsp_phys_gr_surf_n; i++) { + if (va >= gsp_phys_gr_surf[i].va && + va < gsp_phys_gr_surf[i].va + + (gsp_phys_gr_surf[i].npages << 12)) { + sh = gsp_phys_gr_surf[i].alias + + (va - gsp_phys_gr_surf[i].va); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } for (i = 0; i < req->n_pages; i++) { if ((req->pages[i].guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || @@ -19343,22 +19604,245 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, * number of relocated pair(s). Run widths follow the service's decoder: * op 1/3/5 are 1+count dwords, everything else resyncs one dword. */ +/* + * SESSION 153-CONTINUED-13 (draw-window strip experiment): comma-separated + * hex NVC597 method byte-addresses to REMOVE from a captured 3D window + * before the verbatim replay (whole methods - a dropped run takes its + * count data words with it). Answers whether a DRAW window's non-retire + * lives in the draw content (BEGIN/END/immediate vertices/DRAW_ARRAY) or + * in the state binds/method-stream shape. Empty = off (historical + * behaviour); live-tunable via sysfs so one boot can sweep the variants. + */ +static char *gsp_phys_rmcall_gr_win_drop; +module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); +MODULE_PARM_DESC(sema_gl_win_drop, + "Hex method byte-addresses (comma-separated) to strip from a captured 3D window before replay (draw isolation; default empty)"); + +static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) +{ + const char *p = gsp_phys_rmcall_gr_win_drop; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0; + u32 digits = 0; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + /* + * SESSION 153-CONTINUED-46: accept an optional 0x/0X prefix. + * The original parser scanned hex digits only, so the natural + * "0x3810,0x39c8" input parsed as 0 (the 'x' ended the digit + * run after the leading '0') and silently never matched - + * cost a full bisect round. + */ + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) + p += 2; + for (;;) { + char c = *p; + u32 d; + + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | d; + p++; + digits++; + } + if (!digits) + break; + n++; + if (v == addr) + return true; + } + return false; +} + +/* + * SESSION 153-CONTINUED-27: auto-drop of the software-emulated lanes from + * every verbatim 3D replay. Three method triggers in a captured window + * cause the physical engine to repeat work the sema service already + * executes in software - and, on the replay channel, with the WRONG + * context state (no previous-window sticky registers): + * - 0x1b0 LAUNCH_DMA (C597 CE lane): triggers both the CE copy and the + * LOAD_INLINE_DATA upload. The service authors every LID payload into + * the guest pages itself (svc_gl_lid/svc_grce_inline_write) and + * emulates the CE strips (svc_op2d); physically re-running them with + * an unset source/context walks unbounded memory (r992-r996: the + * "march" - faults at whatever page the alias cap ended, e.g. + * 0x5d6000/0x5f4000, after having written every mapped page). + * - 0x5e0 (TWOD RECTS fill trigger): the second point pair of the + * client's screen clear. The service's FERMI_TWOD executor runs every + * fill (r986: 46x `sema-twod-fill ... done=3686400 rc=0`); the + * physical re-run uses the replay channel's unset dst PITCH and + * marches (same fault class). + * - 0x10c NOTIFY: the completion write; without the preceding + * SET_NOTIFY_A/B (which live in other pushes) the engine writes + * address 0 (r996: `faulted @ 0x0_00000000 FAULT_PTE VIRT_WRITE`). + * The kick-notify lane + the backend's release tail author every + * completion the client actually waits on. + * All three are pure triggers - their setup methods stay in the window, + * so dropping the run can never misalign display state. The emulated + * lanes are the live ones (the r986 pixel proof: the screen paints via + * svc_twod fills, and every CE strip retires rc=0 in software). Default + * on; set 0 to replay the raw window unmodified. + */ +static bool gsp_phys_gr_win_drop_auto = 1; +module_param_named(sema_gl_win_drop_auto, gsp_phys_gr_win_drop_auto, bool, + 0644); +MODULE_PARM_DESC(sema_gl_win_drop_auto, + "Drop the software-emulated trigger methods (0x10c NOTIFY, 0x1b0 LAUNCH_DMA, 0x5e0 TWOD fill) from verbatim 3D replays (default 1)"); + +static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) +{ + if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) + return false; + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u; +} + +/* + * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch + * the DATA of one method inside a replayed 3D window before it reaches the + * engine. Format: comma-separated `addr:data` hex pairs, addr = method + * byte-address (e.g. 1618 = BEGIN, 1614 = END), data = the replacement + * value. Header-only methods (IMMD) get their 13-bit data field rewritten + * in place; count-carrying methods (INC/NON_INC/INC_ONCE) get their FIRST + * payload word replaced. Live-tunable via sysfs so one boot can sweep + * variants (BEGIN OP=7 QUADS -> TRIANGLES etc.) against the draw-stall. + */ +static char *gsp_phys_rmcall_gr_win_patch; +module_param_named(sema_gl_win_patch, gsp_phys_rmcall_gr_win_patch, charp, + 0644); +MODULE_PARM_DESC(sema_gl_win_patch, + "Hex addr:data pairs (comma-separated) patched into a captured 3D window method before replay (default empty)"); + +static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) +{ + const char *p = gsp_phys_rmcall_gr_win_patch; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0, d = 0; + u32 digits = 0; + bool have_data = false; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | h; + p++; + digits++; + } + if (!digits) + break; + if (*p == ':' || *p == '=') { + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (d <= 0xfffffffu) + d = (d << 4) | h; + p++; + have_data = true; + } + } + n++; + if (have_data && v == addr) { + *out_data = d; + return true; + } + } + return false; +} + static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + u32 *out_n, u32 *n_dropped, const struct gsp_phys_gr_submit *req, u32 *n_shadow) { - u32 off = 0, reloc = 0; + u32 off = 0, wout = 0, reloc = 0, dropped = 0; - memcpy(out, in, (size_t)n * 4); + *out_n = n; + *n_dropped = 0; while (off < n) { - u32 hdr = out[off]; + u32 hdr = in[off]; u32 op = (hdr >> 29) & 0x7u; u32 cnt = (hdr >> 16) & 0xfffu; u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + if (gsp_phys_rmcall_gr_win_drop_match(addr) || + gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { + dropped++; + off += span; + continue; + } + memcpy(&out[wout], &in[off], (size_t)span * 4); + { + /* + * SESSION 153-CONTINUED-14: live data patch (BEGIN + * OP sweep etc.). Header-only forms rewrite the 13-bit + * data field; count-carrying forms rewrite the first + * payload word. + */ + u32 pv; + + if (gsp_phys_rmcall_gr_win_patch_find(addr, &pv)) { + static u32 patch_log; + u32 old; + + if (span > 1u) { + old = out[wout + 1]; + out[wout + 1] = pv; + } else { + old = (out[wout] >> 16) & 0x1fffu; + out[wout] = (out[wout] & ~0x1fff0000u) | + ((pv & 0x1fffu) << 16); + } + if (patch_log < 32) { + patch_log++; + pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", + addr, off, old, pv); + } + } + } + if ((op == 1 || op == 3 || op == 5) && cnt && off + 1u + cnt <= n) { - u32 *d = &out[off + 1]; + u32 *d = &out[wout + 1]; u32 lo_i = 0; bool pair = false; @@ -19375,19 +19859,272 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, addr < 0x4000 && (addr & 0x3fu) == 0x14 && cnt >= 2) { pair = true; + } else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) && + cnt >= 3) { + /* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_ + * A_FORMAT(j)+BASE(j) run [fmt, upper, lower] + * (j*0x10 stride) - the vertex data the draw + * reads. Observed [00001000 1 2150b790]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr == 0x2380 && cnt >= 3) { + /* SET_CONSTANT_BUFFER_SELECTOR_A/B: + * [size, upper, lower]; observed + * [00010000 1 2150a500]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 2) { + /* SET_COLOR_TARGET_A/B(j) offset pair: + * [upper, lower, w, h, fmt]; the surface the + * draw rasterizes into (must live in guest + * RAM for the present lane to see pixels). + * Observed [1 222fa200 40 4 f7 0]. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x218 && cnt >= 4 && + (d[2] & ~0xffu) == 0u && + (d[3] & 0xfffu) == 0u) { + /* SESSION 153-CONTINUED-27: the TWOD twin + * of the colour target - INC 0x218 cnt>=4 + * covers 0x218..0x224 = [w, h, upper, + * lower] (the emulator's SET_DST sequence; + * the glxgears client window fills the X + * screen through it). Shape-checked so a + * non-surface run can never be rewritten. */ + lo_i = 2; + pair = true; + } + if (pair) { + u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu); + bool ok = gsp_phys_rmcall_gr_3d_reloc( + &d[lo_i], &d[lo_i + 1], req, + n_shadow); + static u32 pair_log; + + if (ok) + reloc++; + if (pair_log < 48) { + pair_log++; + pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n", + addr, pva, ok ? "" : + "MISS ", + ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu)); + } } - if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i], - &d[lo_i + 1], - req, n_shadow)) - reloc++; + wout += span; off += 1u + cnt; continue; } off++; /* header-only / resync, like the service decoder */ + wout++; } + *out_n = wout; + *n_dropped = dropped; return reloc; } +/* + * SESSION 153-CONTINUED-27: alias-build for a big colour target. Consumes + * ceil(npages/16) never-used scratch-PT fetch lines and writes one PTE per + * surface page, so the whole surface is reachable through one CONTIGUOUS + * alias VA (lines are monotonic; consecutive lines are VA-contiguous + * inside a pool block - both pool blocks are checked). Every page's HPA + * comes from the launch's own page set when present, else from the + * service's trusted resolver - the identical vetted translation the + * shadow relocation uses. Lines are consumed even on failure (they are + * never reused, so a half-written line can never be shadowed later); + * the caller then falls back to the existing per-page path. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_gr_surface_alias_locked(const struct gsp_phys_gr_submit *req, + u64 va, u64 npages, u64 *alias_out) +{ + u64 map_pages = npages + GSP_PHYS_GR_SURF_SLACK_PAGES; + u32 nlines; + u32 base_line; + u32 i, j; + u64 first_va, last_va; + + if (map_pages > 4096u) + map_pages = 4096u; + nlines = (u32)((map_pages + + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES - 1) / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES); + base_line = boot_stage.rmcall_guest_pt_next_line; + if (!nlines || base_line + nlines > GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return false; + first_va = gsp_phys_rmcall_guest_pt_win_va( + base_line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)(base_line % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + last_va = gsp_phys_rmcall_guest_pt_win_va( + (base_line + nlines - 1u) / + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)((base_line + nlines - 1u) % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + /* one contiguous run: never straddle a pool block boundary */ + if (last_va != first_va + ((u64)(nlines - 1u) << 16)) + return false; + /* + * Consume the lines before writing: a failure mid-way leaves the + * partially written lines consumed (never reused), so the monotonic + * line invariant - a cached line can never shadow a later mapping - + * holds unconditionally. + */ + boot_stage.rmcall_guest_pt_next_line = base_line + nlines; + for (i = 0; i < nlines; i++) { + u32 line = base_line + i; + u32 page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u32 lip = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u8 *pt; + + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return false; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return false; + for (j = 0; j < GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; j++) { + u64 k = (u64)i * GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES + + j; + u64 pv, hpa = 0; + u32 pi; + + if (k >= map_pages) + break; + pv = va + (k << 12); + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == pv && + req->pages[pi].hpa) { + hpa = req->pages[pi].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(pv, &r) || + !r.hpa) { + /* + * A declared page must resolve; a + * slack-tail page past the guest + * allocation gets the dummy so the + * engine's overdraw vanishes. + */ + if (k < npages) + return false; + hpa = gsp_phys_gr_surf_dummy_dma_locked(); + if (!hpa) + return false; + } else { + hpa = r.hpa; + } + } + put_unaligned_le64( + gsp_phys_rmcall_pte(hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + lip + + j) * + 8); + } + dma_wmb(); + } + *alias_out = first_va; + return true; +} + +/* + * SESSION 153-CONTINUED-27: pre-pass over a raw 3D window. Scans the + * colour-target runs (C597 INC 0x800+j*0x40 [upper, lower, w, h, fmt, + * blksz] and - same surface, shape seen on the TWOD subchannel - INC + * 0x218 cnt>=4 [w, h, upper, lower]) and aliases each big surface once + * per boot so the relocation pass can rewrite its operands to the alias + * (r992/r993 evidence: the X screen 0x120400000, 1280x720 fmt 0xe6, is + * the draw's render target AND the TWOD fill's destination). Small + * surfaces keep the historical one-pair shadow relocation. Called with + * gsp_phys_rm_lock held. + */ +static void gsp_phys_gr_surface_alias_scan_locked( + const u32 *win, u32 n, const struct gsp_phys_gr_submit *req) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + span = 1u + cnt; + } else { + off++; + continue; + } + if (op == 1 && ((addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 4) || + (addr == 0x218 && cnt >= 4))) { + const u32 *d = &win[off + 1u]; + u64 va; + u32 w, h, fmt; + u64 bytes, npages; + u32 sdi; + u64 alias = 0; + + if (addr == 0x218) { + va = ((u64)(d[2] & 0xffu) << 32) | + (u64)(d[3] & 0xfffff000u); + w = d[0]; + h = d[1]; + fmt = 0xe6u; /* TWOD run carries fmt on + * its own IMMD; the surface + * class here is 32bpp */ + } else { + va = ((u64)(d[0] & 0xffu) << 32) | + (u64)(d[1] & 0xfffff000u); + w = d[2]; + h = d[3]; + fmt = d[4]; + } + if (!va || (va & 0xfffu) || fmt != 0xe6u || + !w || !h || w > 8192u || h > 8192u) + goto next; + bytes = (u64)w * 4u * (u64)h; + npages = (bytes + GSP_PHYS_PAGE_SIZE - 1) >> 12; + if (npages < 2u || npages > 4096u) + goto next; + for (sdi = 0; sdi < gsp_phys_gr_surf_n; sdi++) + if (gsp_phys_gr_surf[sdi].va == va) + goto next; /* already aliased */ + if (gsp_phys_gr_surface_alias_locked(req, va, npages, + &alias)) { + if (gsp_phys_gr_surf_n < + ARRAY_SIZE(gsp_phys_gr_surf)) { + gsp_phys_gr_surf[gsp_phys_gr_surf_n].va = + va; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .alias = alias; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .npages = npages; + gsp_phys_gr_surf_n++; + } + pr_info("vgsp-gr: SURFACE alias va=0x%llx pages=%llu (%ux%u fmt=0x%x) -> 0x%llx (contiguous shadow run; operands rewritten)\n", + va, npages, w, h, fmt, alias); + } else { + pr_info("vgsp-gr: SURFACE alias FAILED va=0x%llx (%ux%u fmt=0x%x): operands fall to the per-page path\n", + va, w, h, fmt); + } + } +next: + off += span; + } +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -19435,8 +20172,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; bool sem_gated = false; - static u32 sked_prev[11]; - static u32 sked_sample[11]; + static u32 sked_prev[14]; + static u32 sked_sample[14]; static u32 sked_tick; static bool sked_first; /* @@ -20177,6 +20914,48 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, memcpy(sh_page + (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-10: a stream3d (3D + * method-window) submission maps the engine's + * VAs straight at the GUEST page HPA + * (gsp_phys_rmcall_gr_map_shadow_locked + * installs the guest PTE - there is no + * shadow-chunk copy in that path), so the + * engine's I-fetch reads the guest page, not + * sh_page. The GLX program page is never + * written by the guest itself (the driver's + * uploads never ran), and the r945 evidence is + * exactly that: "Graphics Exception: Shader + * Program Header 11/18 Error" on the draw + * whose inline prog-hex blob had only reached + * this shadow copy. Author the inline bytes + * into the guest page too, emulating the + * upload that never executed. + */ + if (stream3d && pg->hpa) + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + stream3d && pg->hpa ? + 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + } copied++; } /* @@ -20600,14 +21379,66 @@ gr_page_unmapped: * held, before the method block is built. */ if (stream3d) { - u32 nrel = gsp_phys_rmcall_gr_3d_patch_window( - req->stream, req->stream_n, stream_patch, req, - &n_shadow); + u32 win_n = req->stream_n; + u32 ndrop = 0; + u32 nrel; + + /* + * SESSION 153-CONTINUED-27: a big colour target (the X + * screen) is aliased as ONE contiguous shadow run BEFORE + * the relocation pass runs, so the pass rewrites its + * operands to the alias (see the table above). + */ + gsp_phys_gr_surface_alias_scan_locked(req->stream, + req->stream_n, req); + nrel = gsp_phys_rmcall_gr_3d_patch_window( + req->stream, req->stream_n, stream_patch, &win_n, + &ndrop, req, &n_shadow); win3d = stream_patch; - win3d_n = req->stream_n; - pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n", - nrel, win3d_n, n_shadow); + win3d_n = win_n; + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u)\n", + nrel, win3d_n, n_shadow, ndrop); + if (ndrop) { + static u32 gr3d_drop_log; + + if (gr3d_drop_log < 32) { + gr3d_drop_log++; + pr_info("vgsp-gr: 3D win-drop h=0x%x push=0x%llx raw=%u kept=%u dropped=%u list=[%s]\n", + req->h_object, req->push_va, + req->stream_n, win3d_n, ndrop, + gsp_phys_rmcall_gr_win_drop ? + gsp_phys_rmcall_gr_win_drop : ""); + } + } + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 4) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } } words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, @@ -20615,6 +21446,10 @@ gr_page_unmapped: ch_sem_va, want_ch, win3d, win3d_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ + WRITE_ONCE(gsp_phys_gr_last_words, words); + WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); + WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); @@ -20997,10 +21832,20 @@ gr_page_unmapped: sked_sample[8] = ioread32(bar0 + 0x00409b00u); sked_sample[9] = ioread32(bar0 + 0x00409b04u); sked_sample[10] = ioread32(bar0 + 0x00409400u); + /* + * SESSION 153-CONTINUED-14: FECS/FE ctxsw handshake + * lanes. fe0 = FECS_CTXSW_STATUS_FE_0 (0x409c00), + * fehost = FECS_HOST_INT_STATUS (0x409c18), fecsirq + * = FECS_IRQSTAT (0x409008). If the FECS dies in a + * switch, it should die visibly here first. + */ + sked_sample[11] = ioread32(bar0 + 0x00409c00u); + sked_sample[12] = ioread32(bar0 + 0x00409c18u); + sked_sample[13] = ioread32(bar0 + 0x00409008u); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x\n", sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], @@ -21008,7 +21853,8 @@ gr_page_unmapped: sked_sample[4], sked_sample[5], sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], - sked_sample[10]); + sked_sample[10], sked_sample[11], + sked_sample[12], sked_sample[13]); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; @@ -21021,6 +21867,50 @@ gr_page_unmapped: else w0 = READ_ONCE(*(u32 *)sem_va); } + /* + * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D + * stream launch - where is the front end while a DRAW window + * does not retire? One shot per launch at ~150 ms and ~1.2 s + * (5 ms poll), bounded per boot. pbfetch inside the module + * pushbuffer page = the walk/exec is stuck in our block; + * pbfetch past it = the whole stream was decoded and the + * stall is in execution/completion. + */ + if (stream3d && bar0 && + (sked_tick == 30 || sked_tick == 240)) { + static u32 winprobe_log; + u32 pi; + + if (winprobe_log < 10) { + winprobe_log++; + pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, + req->push_va, words, sub.pb_dma, + ioread32(bar0 + 0x00400504u), + ioread32(bar0 + 0x00400704u), + ioread32(bar0 + 0x00400708u)); + for (pi = 0; pi < 12; pi++) { + u32 b = 0x40000u + pi * 0x2000u; + u32 intr0 = ioread32(bar0 + b + 0x108u); + u32 gpf = ioread32(bar0 + b + 0x050u); + u32 pbf = ioread32(bar0 + b + 0x054u); + u32 pbfhi = ioread32(bar0 + b + 0x058u); + u32 hdrsh = ioread32(bar0 + b + 0x118u); + u64 fetch64 = + ((u64)(pbfhi & 0xffu) << 32) | + (u64)(pbf & ~3u); + + if (!intr0 && !gpf && !pbf && !hdrsh) + continue; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx hdr=%08x(%s m=0x%x cnt=%u sub=%u)\n", + pi, intr0, gpf, fetch64, hdrsh, + gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, + (hdrsh >> 13) & 7u); + } + } + } if (ch_rel_hpa) w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off)); @@ -21456,6 +22346,16 @@ gr_page_unmapped: ioread32(bar0 + 0x00405b00u), ioread32(bar0 + 0x00400100u), ioread32(bar0 + 0x0040013cu)); + if (bar0) + pr_info("vgsp-gr: FECS readback: CUR=0x%08x NEW=0x%08x ST1=0x%08x FE0=0x%08x HOSTINT=0x%08x IRQSTAT=0x%08x CPUCTL=0x%08x OS=0x%08x\n", + ioread32(bar0 + 0x00409b00u), + ioread32(bar0 + 0x00409b04u), + ioread32(bar0 + 0x00409400u), + ioread32(bar0 + 0x00409c00u), + ioread32(bar0 + 0x00409c18u), + ioread32(bar0 + 0x00409008u), + ioread32(bar0 + 0x00409100u), + ioread32(bar0 + 0x00409080u)); if (bar0) { u32 g; @@ -24462,6 +25362,91 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* + * gsp-phys-shared-bit0 (write-only, mode 0200): development-harness assist + * for the NVIDIA X driver's per-client shared-data slot. + * + * Every direct GLX client receives a shared memory page from the X driver + * (nvidia_drv.so creates a memfd and passes it over a unix socket) whose + * first words are {u32 zero, u32 cookie, u32 state, u32 zero, u32 hClient, + * u32 hObject}; glcore's device init reads the state word and skips its own + * class-c46f channel creation while bit 0 of it is set. In this emulated + * environment the X driver publishes state=3 (bits 0+1) and direct clients + * therefore never allocate a channel and never render. The value is written + * by the X driver before the fd reaches the client, so it cannot be + * corrected from the client side. + * + * Input: "0x<hex GPA>" (4 KB aligned) naming the guest page that carries the + * slot. The GPA must fall inside a range registered through + * gsp-phys-guest-ram; it is translated with gsp_phys_gpa_to_hpa_vgpu() + * (first open mdev) and the state word is cleared of bit 0 through the + * kernel direct map (the translator validates pfn_valid, so __va() is a + * valid kernel mapping for the shmem-backed guest RAM). As a sanity guard + * the write is only performed when the page's +0x10 word looks like an RM + * client handle (0xc1d0xxxx / 0xc1e0xxxx) or the token carries a "force" + * suffix. Old and new values are always logged. + * + * This is a diagnostic aid for the development VM, matching the spirit of + * the existing guest-GPA pin; it is not part of the product behaviour. + */ +static ssize_t gsp_phys_shared_bit0_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[48]; + u64 gpa, hpa = 0; + u32 *word; + u32 h1032; + bool force = false; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + { + char *p = strim(buf); + char *tok = strsep(&p, " \t"); + + if (p && strstr(p, "force")) + force = true; + if (kstrtoull(tok, 0, &gpa) || !PAGE_ALIGNED(gpa)) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: write ignored: expected \"0x<hex GPA>\" (4 KB aligned, optional 'force')\n"); + return -EINVAL; + } + } + + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); + if (ret) { + pr_err("vgsp-boot: gsp-phys-shared-bit0: GPA 0x%llx translation failed (%d: no open mdev / GPA not inside a gsp-phys-guest-ram range / not RAM)\n", + gpa, ret); + return -ENOENT; + } + + word = (u32 *)__va(hpa + 8); + h1032 = *(u32 *)__va(hpa + 0x10); + if (!force && (h1032 >> 16) != 0xc1d0 && (h1032 >> 16) != 0xc1e0) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx word=0x%x but +0x10=0x%x does not look like an RM client handle; refusing (append 'force' to override)\n", + gpa, *word, h1032); + return -EINVAL; + } + + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx hpa=0x%llx hClient=0x%x word=0x%x -> 0x%x\n", + gpa, hpa, h1032, *word, *word & ~1u); + *word &= ~1u; + return len; +} + +static const struct file_operations gsp_phys_shared_bit0_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_shared_bit0_write, + .llseek = no_llseek, +}; + + +/* * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is @@ -24670,6 +25655,36 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -24916,6 +25931,20 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-shared-bit0", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_shared_bit0_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The shared-slot bit0 assist is optional; the other + * triggers and the state snapshot stay registered. + */ + pr_warn("vgsp-boot: debugfs file gsp-phys-shared-bit0 creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-guest-ram", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_ram_fops); @@ -25382,8 +26411,9 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index dfdf1d9..14cf482 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -354,7 +354,7 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, */ #define GSP_PHYS_GR_MAX_PAGES 16u #define GSP_PHYS_GR_MAX_INLINE 8u -#define GSP_PHYS_GR_INLINE_BYTES 64u +#define GSP_PHYS_GR_INLINE_BYTES 256u #define GSP_PHYS_GR_QMD_WORDS 64u /* * IQ2M sentinel: written to the inline-QMD destination's word 63 after the @@ -372,8 +372,12 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, * r929: grown 256 -> 512 dwords so a whole GL 3D section push (the observed * X server one-shot is 1524 B = 381 dwords) survives in the captured raw * tail; the raw window is the source of the 3D method-stream replay. + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB) for the DIRECT GL + * client's 5016-7144 B render pushes (glcore); the raw window must carry + * the whole push for the section capture's `len == push_n*4` gate. Must + * equal SVC_GR_PUSH_WORDS (static_assert in gsp_sema_service.c). */ -#define GSP_PHYS_GR_RAW_WORDS 512u +#define GSP_PHYS_GR_RAW_WORDS 2048u /* * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region. diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 7b7e230..99f394c 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -3137,12 +3137,78 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); + /* + * SESSION 153-CONTINUED-38: the USERD control page sits at + * userdMem.address + userdOffset[subDeviceId] (NV_CHANNEL_ALLOC_PARAMS + * userdOffset[NV_MAX_SUBDEVICES], @64; subDeviceId @136, the + * one-hot bit index is normally 0 but the X client-share channel + * carries 1). The GL client's kernel channels declare userdMem fb + * 0x18fe00 with userdOffset[0]=0x7fe00, so the service was + * probing/monitoring 0x18fe00 while the real control page + * (GPGet/GPPut) lives at 0x20fc00 - both the registered candidate + * and every FB observation were at the wrong offset, which is why + * the direct client's channels never resolved a USERD (CONT-37 + * alloc/free retry loop). Index by the subdevice. + */ + { + u32 sd = 0; + u64 uo = 0; + + if (params_size >= 140) + memcpy(&sd, params + 136, 4); + if (sd >= 8) + sd = 0; + if (params_size >= 64 + (sd + 1) * 8) { + memcpy(&uo, params + 64 + (u64)sd * 8, 8); + desc.userd_pa += uo; + } + } memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + GSPRM_CHAN_MEM_APERTURE, 4); memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8); memcpy(&desc.notifier_as, params + GSPRM_CHAN_NOTIFIER_MEM + GSPRM_CHAN_MEM_APERTURE, 4); + /* + * SESSION 153-CONTINUED-15 (draw attribution): every channel-class + * allocation is named with its wire shape before registration, so a + * channel the service silently refuses (gpfifo VA 0 / entries < 2) + * cannot hide behind an absent "registered" line. Bounded. + */ + { + static u32 chan_alloc_log; + + if (chan_alloc_log < 32) { + chan_alloc_log++; + pr_info("vgsp-rm: chan alloc hObject=0x%x gpfifo=0x%llx ent=%u eng=%u vas=0x%x cid=%u userd=0x%llx/%u notif=0x%llx/%u\n", + h_object, desc.gpfifo_va, desc.gpfifo_entries, + desc.engine_type, desc.h_vaspace, desc.cid, + desc.userd_pa, desc.userd_as, + desc.notifier_pa, desc.notifier_as); + } + } + /* + * SESSION 153-CONTINUED-38: name the per-subdevice userdOffset (the + * kernel channels' control page may sit inside the declared userdMem + * at a nonzero offset; the service currently assumes address+0). + * Bounded. + */ + { + static u32 uo_log; + + if (uo_log < 8 && params_size >= 80) { + u64 uo0 = 0; + u32 sd = 0; + + memcpy(&uo0, params + 64, 8); + if (params_size >= 140) + memcpy(&sd, params + 136, 4); + uo_log++; + pr_info("vgsp-rm: chan userdOff[0]=0x%llx subDevId=0x%x hObject=0x%x (userd memdesc=0x%llx/%u)\n", + uo0, sd, h_object, desc.userd_pa, + desc.userd_as); + } + } gsp_sema_service_channel_add(&desc); } @@ -3505,6 +3571,19 @@ MODULE_PARM_DESC(gsp_rm_binapi07_fwd, "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); /* + * SESSION 153-CONTINUED-17: reply word (little-endian u16) for BINAPI + * 0x20810107. The guest GL client reads reply byte[1] as a feature flag + * (glcore+0xFC74E1: setne <ctx>+0x203 after cmpb $0x0,reply[1]) and the + * flag gates method emission at glcore+0xB5B48B (see the control handler). + * 0x0101 sets both bytes; 0x0100 sets byte[1] only. Default 0 = the + * historical {0,0} reply. + */ +static uint gsp_rm_binapi07_rep; +module_param_named(binapi07_rep, gsp_rm_binapi07_rep, uint, 0644); +MODULE_PARM_DESC(binapi07_rep, + "reply word (LE u16) for BINAPI cmd 0x20810107; the GL client reads byte[1] as a feature flag (default 0)"); + +/* * session 75: reply words for the client-pending control 0x2080a097 (the * libnvidia-eglcore engine-readiness poll). Runtime-tunable so one boot can * sweep reply variants: the client reads +0x204, +0x488 and +0x48c back @@ -4381,6 +4460,42 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + case 0x20800afe: { /* NV2080_CTRL_CMD_INTERNAL_INIT_USER_SHARED_DATA + * + * The guest kernel driver hands the GSP-RM the physical + * address of the RUSD (RM user shared data) page it + * allocated; on real hardware the GSP registers the + * memdesc and keeps the page's sequence-guarded fields + * (clocks, BAR1, utilisation, ...) populated, and 0 + * de-initialises it. Nothing here performs that upkeep, + * and consumers treat a zeroed sequence word as "data + * invalid" and skip the fields, so the correct emulated + * behaviour is to acknowledge the registration, echo the + * request (the caller may read the address back) and log + * the address once so the page can be correlated with + * guest RAM during diagnosis. */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 8) { + u64 phys_addr; + + memcpy(&phys_addr, pp, 8); + memcpy(rp, pp, req.paramsSize); + rm->rusd_phys_addr = phys_addr; + rm->rusd_init_seen = true; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: INIT_USER_SHARED_DATA physAddr=0x%llx (RUSD registered; fields left at sequence 0 = 'data invalid', consumers skip)", + phys_addr); + out_status = 0; + out_params_size = req.paramsSize; + } else { + out_status = 0; + out_params_size = 0; + } + break; + } + case 0x2080a097: { /* client-pending engine-context query (session 75) * * libnvidia-eglcore issues this via NVOS54 and then @@ -4510,6 +4625,52 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, out_params_size = req.paramsSize; break; + case 0x20810107: { /* NV2081_BINAPI reserved-interface method 7 */ + /* + * SESSION 153-CONTINUED-17: the guest GL client + * (libnvidia-glcore 570.144, file offset 0xFC74A6) sends this + * 2-byte query right after allocating its BINAPI object and + * CONSUMES REPLY BYTE [1] AS A FEATURE FLAG: + * + * glcore+0xFC74D9: cmpb $0x0, 0x131(%rsp) + * glcore+0xFC74E1: setne 0x203(%r14) ; flag = replies[1] + * + * and the flag gates 3D method emission into the command + * buffer: + * + * glcore+0xB5B48B: mov 0x1248(%r14),%rax + * cmpb $0x0,0x203(%rax) + * je <skip> + * glcore+0xB5B49B: movabs $0xf020010e66,%rdi + * mov %rdi,(0x80(%r14)) ; emit + advance + * + * With the historical {0,0} generic reply the flag stays 0 and + * the client's submission pipeline never activates (no GL + * channel, no pushes, black window). binapi07_rep selects the + * reply word (LE u16) at runtime so one boot can test the + * feature without a rebuild: 0x0101 = both bytes set, + * 0x0100 = byte[1] only. Default 0 reproduces {0,0}. + */ + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 2 && req.paramsSize <= 16) { + memcpy(rp, pp, req.paramsSize); + rp[0] = gsp_rm_binapi07_rep & 0xff; + rp[1] = (gsp_rm_binapi07_rep >> 8) & 0xff; + out_params_size = req.paramsSize; + } else { + out_params_size = 0; + } + if (gsp_sema_dbg) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI 0x20810107 reply in=%u rep=0x%04x bytes=%02x %02x", + req.paramsSize, gsp_rm_binapi07_rep, + rp[0], rp[1]); + out_status = 0; + break; + } + case TU104_CTRL_I2C_TRANSACTION: /* I2C transactions have variable-length FINN-serialized * params: an all-zero reply fails the driver's DeserializeUp diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index a4b1afd..c0eecf8 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -184,6 +184,10 @@ struct gsp_rm_emu { u64 ctrl_fwd_fallback; /* forward attempt used emulation */ u64 ctrl_fwd_timeout; /* fallbacks caused by a backend timeout */ + /* NV2080_CTRL_CMD_INTERNAL_INIT_USER_SHARED_DATA bookkeeping */ + u64 rusd_phys_addr; /* last registered RUSD phys address */ + bool rusd_init_seen; /* a registration was acknowledged */ + /* * NV01_EVENT objects (class 0x5, allocated via class 0x79 in the * guest ABI). Each carries {hSrcResource, notifyIndex, data}; the diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 2ab1060..4d497d9 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -47,6 +47,7 @@ #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" #include "gsp_phys_booter.h" @@ -205,8 +206,28 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* + * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record pushbuffer VA. Constant across every measured boot (r908 + * documented the GL client ring pair {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every + * canonical push read named exactly this VA). Used to discriminate the + * kick ring page from shape lookalikes in the RAM ring scan - r978 + * latched a foreign table page (pa=0x104fa9000) whose entry pair passed + * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000) + * while the true ring (pa=0x1168ce000, found by scanning for the + * record's own entry pair) sat later in RAM scan order. + */ +#define SVC_GL_KICK_PUSH_VA 0x1203c0000ULL /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ -#define SVC_GR_USERD_WATCH 8 +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 /* * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM @@ -612,7 +633,38 @@ struct svc_channel { u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the GL client's channel + * allocation declares its USERD in the FB aperture (0x18fc00-class + * offsets) but the guest's GP_PUT writes land in the emulated BAR0 + * shadow at that offset. Watch that block across passes: a nonzero PUT + * (or a PUT step) is the client's own live write; adopt it as the + * channel's USERD in the BAR0 aperture so the pump iterates the ring. + * bar0_userd_at is the first-sighting stamp; a stable nonzero PUT is + * adopted after a short patience window so a single-submission client + * (waiting on that submission) cannot deadlock the watch. + */ + u64 bar0_userd_watch_pa; /* watched offset (0 = none) */ + u32 bar0_userd_watch_put; /* PUT at the first sighting */ + unsigned long bar0_userd_watch_at; /* jiffies at the first sighting */ + u8 bar0_userd_log; /* bounded per-channel log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ /* * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The @@ -666,6 +718,14 @@ struct svc_channel { */ bool gr_ram_ring; /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring + * liveness re-validation pace for the kick channel (h 0xbfef00xx). + * While the kick USERD stays unresolved this channel's ring latch is + * re-checked each pass (at most once a second) and re-armed when it + * no longer decodes entries - see svc_kick_ring_revalidate. + */ + unsigned long gr_kick_ring_reval_at; + /* * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the * registered/ring-derived candidates all fail; the scan validates the * hit against the ring entries the put claims, so a foreign channel's @@ -675,6 +735,19 @@ struct svc_channel { */ u8 userd_scans; /* + * SESSION 153-CONTINUED-37: time-paced re-run state for the FB USERD + * shape scan (r496b) and its pristine-block fallback. The old + * count-only cap spent every retry within ~64 ms of registration + * (pump pass cadence), while the X driver's pushbuffer test kicks its + * first GP_PUT hundreds of ms later and its VAS registers at ~1 s - + * so on unlucky timings the budget was gone before any candidate + * existed and the 10 s DMA test read GET=0 forever ("Failed to + * initialize DMA", r1013). One scan per 25 ms up to 512 keeps the + * discovery alive across the whole client window. + */ + unsigned long fb_userd_scan_at; /* jiffies of the last scan */ + u16 fb_userd_scan_n; /* scans spent since registration */ + /* * r828: the RC watchdog's notifier VA, captured from the class2d * SET_NOTIFY_A/B methods on subchannel 3 and zeroed at the NOTIFY * (see svc_notify2d_write). Survives the pass snapshot via the @@ -865,12 +938,27 @@ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session st /* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ #define SVC_MAX_GUESTS 8 +/* + * SESSION 153-CONTINUED-34: PCFG doorbell window in the emulated BAR0 shadow. + * Work-submission doorbells the guest pokes after writing a GPFIFO entry can + * land anywhere in the PCFG window (0x88000..0x89000), not only at the + * virtual-function doorbell the trap path used to dispatch (0x88704). Because + * the BAR0 shadow is direct-mapped, most writes never reach the trap path at + * all, so the service thread also watches the window for changed words and + * feeds each new value to gsp_sema_service_doorbell (token format unchanged; + * the call only wakes the pump earlier - the pump itself is per-pass). + */ +#define SVC_BAR0_PCFG_LO 0x88000u +#define SVC_BAR0_PCFG_HI 0x89000u +#define SVC_BAR0_PCFG_WORDS ((SVC_BAR0_PCFG_HI - SVC_BAR0_PCFG_LO) / 4u) struct svc_guest_sess { struct nv_vgpu_device *vgpu; /* NULL = free slot */ bool active; /* a channel of this guest is * registered until this guest's * last channel closes */ u64 last_build; /* per-guest svc_rebuild freshness */ + u32 pcfg_snap[SVC_BAR0_PCFG_WORDS]; + bool pcfg_have; /* snapshot taken for this guest */ }; static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; static DEFINE_SPINLOCK(svc_guest_lock); @@ -1171,6 +1259,10 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 kick_badpg; /* r977: kick-family push reads + * that lacked the canonical + * submit-record prefix (wrong + * page; forced the rescue) */ u64 rescue_map_ok; /* SESSION 148: content-verified * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ @@ -1512,6 +1604,17 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 +/* + * SESSION 153-CONTINUED-34: the emulated BAR0 (register) shadow as a third + * aperture. The GL client's channel allocation reports its USERD block in the + * FB aperture (NV_ADDR_FBMEM, offset 0x18fc00/0x18fe00/0xfbfc00...) but the + * writes the guest actually makes to its GP_PUT land in the emulated BAR0 + * shadow at that same offset (the vGPU guest driver programs its usermode + * block through the register window). Every other source reads the FB shadow + * at those offsets and finds nothing, so the pump never iterates the + * channel's GPFIFO and the channel retires with submissions=0. + */ +#define SVC_APERTURE_BAR0 3 /* * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom @@ -1523,7 +1626,31 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len */ static inline u32 svc_ap_of(int ap) { - return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + if (ap == SVC_APERTURE_FB) + return SVC_APERTURE_FB; + /* SESSION 153-CONT-34: the BAR0 selector must survive normalisation. */ + if (ap == SVC_APERTURE_BAR0) + return SVC_APERTURE_BAR0; + return SVC_APERTURE_SYS; +} + +/* + * SESSION 153-CONT-34: the emulated BAR0 shadow (the buffer the guest's + * register writes actually land in - it is mmap'd into QEMU and also the + * trap-path write target). NULL when the device has no emulation shadow. + */ +static void *svc_bar0_shadow(struct nv_vgpu_device *vgpu, size_t *size) +{ + struct nv_vgpu_bar_emul_ctx *bctx; + + if (!vgpu) + return NULL; + bctx = vgpu->bar_emul_ctx; + if (!bctx || !bctx->bar0_shadow) + return NULL; + if (size) + *size = bctx->bar0_shadow_size; + return bctx->bar0_shadow; } /* @@ -1543,6 +1670,16 @@ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(buf, base + addr, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(buf, base + addr, len); + return 0; + } return svc_read(vgpu, addr, buf, len); } @@ -1558,6 +1695,16 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(base + addr, buf, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(base + addr, buf, len); + return 0; + } return svc_write(vgpu, addr, buf, len); } @@ -1639,8 +1786,27 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int i; if (!desc || !desc->h_object || !desc->gpfifo_va || - desc->gpfifo_entries < 2) + desc->gpfifo_entries < 2) { + /* + * SESSION 153-CONTINUED-15 (draw attribution): a refused + * channel registration used to be silent, so a real client + * channel with an empty gpfifo VA looked identical to a + * channel that was never allocated. Name every refusal. + */ + static u32 refuse_log; + + if (refuse_log < 32) { + refuse_log++; + pr_info("sema-svc: chan add REFUSED h=0x%x gpfifo=0x%llx ent=%u eng=%u userd=0x%llx/%u\n", + desc ? desc->h_object : 0, + desc ? desc->gpfifo_va : 0, + desc ? desc->gpfifo_entries : 0, + desc ? desc->engine_type : 0, + desc ? desc->userd_pa : 0, + desc ? desc->userd_as : 0); + } return; + } if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; @@ -3452,6 +3618,7 @@ static void svc_pin_drop(u64 va) * every push was "implausible"). */ static inline bool svc_pool_sem_va(u64 va); +static bool svc_kick_family(const struct svc_channel *ch); static void svc_pool_pa_forget(u64 va); /* @@ -3488,6 +3655,9 @@ static u32 svc_rdw_log; * appeared. Each scan reports its decision, a scan that found nothing * reportable logs a sweep summary. */ static u32 svc_userdp_log; +/* SESSION 153-CONTINUED-44 (r1026): bounded witness for zero-put USERD + * candidate skips (svc_chan_userd; the screen-pattern false block). */ +static u32 svc_zero_put_skip_log; /* r829: bounded witness for the zero-entry holds (bad ring translations). */ static u32 svc_zero_entry_log; static u8 svc_read_last_src; @@ -5525,9 +5695,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_owed_resolved, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, + svc.kick_badpg, svc.pool_alt_applied, svc.pool_residue, svc_wait_held, svc_wait_pass, svc_wait_giveup, svc_wait_ovf); @@ -5616,6 +5787,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_LINE_COUNT 0x41c #define CE_M_PITCH_IN 0x410 #define CE_M_PITCH_OUT 0x414 +#define CE_M_SRC_ORIGIN_X 0x744 +#define CE_M_SRC_ORIGIN_Y 0x748 +#define CE_M_DST_ORIGIN_X 0x74c +#define CE_M_DST_ORIGIN_Y 0x750 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -5723,6 +5898,18 @@ struct svc_push_state { u32 pitch_in, pitch_out; u32 have_pitch_in, have_pitch_out; u32 line_count; + /* + * SESSION 153 CONT.25: C5B5 SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750). The X DDX's read/present paths + * position each strip with SRC_ORIGIN (x in bytes, y in rows for + * compSizeFactor == Bpp == 4; class C5B5 has no null remapper) + * instead of folding it into OFFSET_IN. The linear shadow holds + * the pixels linearly, so the effective operand becomes + * base + origin_y * pitch + origin_x. + */ + u32 src_origin_x, src_origin_y; + u32 dst_origin_x, dst_origin_y; + u32 have_src_origin, have_dst_origin; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -5746,6 +5933,66 @@ struct svc_push_state { u32 qmd[SVC_QMD_WORDS]; }; +/* + * SESSION 153 CONT.25: CE register state persists across pushes on the + * same channel/subchannel in hardware. The X DDX's read-image loop pushes + * the full setup once (SET_SRC_BLOCK_SIZE / OFFSET_IN / PITCH / + * LINE_LENGTH, a 112-byte push) and then per-strip DELTA pushes that only + * carry SRC_ORIGIN, SET_SEMAPHORE_A/PAYLOAD, OFFSET_OUT, (LINE_COUNT) and + * LAUNCH_DMA (a 52-byte push; raw witness: the delta launch decodes + * line=0/in=0/have_out=1/have_sem=1 on h=0x1000013). Decoding each push + * in isolation left line_length 0, so the transfer-shaped delta launch + * built NO op: every other 9-row strip never executed and the readback + * kept half the rows black (the visible even/odd banding). Remember the + * last full setup per channel and fill a delta launch's missing fields + * from it. + */ +struct svc_ce_saved { + u64 h_object; + u64 offset_in; + u32 pitch_in, pitch_out, line_length, line_count; + u32 src_phys_mode, dst_phys_mode; + u32 have_in, have_pitch_in, have_pitch_out; + u32 have_src_mode, have_dst_mode; +}; +#define SVC_CE_SAVED_N 8 +static struct svc_ce_saved svc_ce_saved[SVC_CE_SAVED_N]; +static u32 svc_ce_saved_next; +static u32 svc_cedelta_log; + +static struct svc_ce_saved *svc_ce_saved_slot(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_CE_SAVED_N; i++) + if (svc_ce_saved[i].h_object == h) + return &svc_ce_saved[i]; + return NULL; +} + +static void svc_ce_saved_store(const struct svc_push_state *st, u64 h) +{ + struct svc_ce_saved *s = svc_ce_saved_slot(h); + + if (!s) { + s = &svc_ce_saved[svc_ce_saved_next++ % SVC_CE_SAVED_N]; + memset(s, 0, sizeof(*s)); + s->h_object = h; + } + s->offset_in = st->offset_in; + s->pitch_in = st->pitch_in; + s->pitch_out = st->pitch_out; + s->line_length = st->line_length; + s->line_count = st->line_count; + s->src_phys_mode = st->src_phys_mode; + s->dst_phys_mode = st->dst_phys_mode; + s->have_in = st->have_in; + s->have_pitch_in = st->have_pitch_in; + s->have_pitch_out = st->have_pitch_out; + s->have_src_mode = st->have_src_mode; + s->have_dst_mode = st->have_dst_mode; +} + /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ @@ -5792,6 +6039,22 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc.op_tr_src = 1; return 0; } + /* + * SESSION 153-CONTINUED-44 (r1027): the CONT-43 walk-first for + * below-floor CE operands is REVERTED. With it live, the moment the + * client's channels actually ran (submissions 33646 via the slot + * hold + zero-put-skip), the guest kernel oopsed repeatedly + * (`clear_page_erms` page fault from `rm_ioctl [nvidia]`, then + * `_mapInsertBase` general protection, Xorg exited) - i.e. the + * walk-first resolutions for below-floor COPY operands clobbered + * guest kernel memory, the r365/r369 corruption class the fill + * guard exists for. The DDX screen-repaint poison that CONT-43 + * fixed is a FILL class and stays fixed by svc_fill_floor_walk in + * svc_fill_1d; the copy path returns to the historical order + * (pins -> scanned map -> walk -> discovered -> targeted scan), + * which is the order every pre-r1025 boot (including the working + * r1024) ran with. + */ if (!svc_map_lookup(va, &pa)) { *addr = pa; *aperture = CE_APERTURE_SYS; @@ -6156,6 +6419,37 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, done_out); } +/* + * SESSION 153-CONT-43: resolve a below-floor fill destination through the + * guest's published page directories before the pin/ledger/map chain. + * + * Below SVC_VAS_FLOOR the driver maps its ABI/MMIO window (the X server's + * screen VA 0x120400000 family, the client pushbuffer region, ...). The + * content-scanned map and the landing ledger are heuristics built for the + * above-floor client VAs; for these low VA numbers a page-table page + * discovered by the scanner can hold a false identity pair (va == pa), and + * the LOWFILL guard then refuses every fill that the pair shadows - the + * observed X server repaint death (`LOWFILL-REFUSED va=0x120458c00 + * pa=0x120458c00`, fill done=52800 rc=-2, replayed ~15 Hz forever). The + * guest's page tables are the truth for this region: resolve through them + * first. Returns true when a translation was applied. + */ +static bool svc_fill_floor_walk(struct nv_vgpu_device *vgpu, u64 va0, + u64 *addr, u32 *aperture) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (va0 >= SVC_VAS_FLOOR) + return false; + if (svc_walk_vaspaces(vgpu, va0 & ~0xfffULL, &wpa, &wap)) + return false; + *addr = wpa + (va0 & 0xfff); + *aperture = (wap == SVC_APERTURE_FB) ? CE_APERTURE_VID + : CE_APERTURE_SYS; + return true; +} + static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) { @@ -6215,6 +6509,27 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 ele pr_info("sema-lowfill-window va=0x%llx -> fb offset 0x%llx (vidmem window convention) elem=%u count=%u value=0x%llx\n", va0, addr, elem, count, value); } + } else if (svc_fill_floor_walk(vgpu, va0, &addr, &aperture)) { + /* + * SESSION 153-CONT-43: below-floor VA (the driver's + * ABI/MMIO mapping window). The guest's published + * page directory is the truth there; a pin or a stale + * landing-table entry can hold a false identity pair + * (va==pa), observed live as the X server's screen + * repaint dying ~44 rows in (`LOWFILL-REFUSED + * va=0x120458c00 pa=0x120458c00` repeating at + * ~15 Hz, the screen frozen on a stale fill + * artifact). The walk resolves the same VA through + * the guest's 2 MB big page to vidmem + * (pa=0x200000+off), which is what the driver meant. + */ + prov = 6; + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-walk va=0x%llx -> pa=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, addr, aperture, elem, count, + value); + } } else if (svc_vas_has(addr)) { shadow = true; } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { @@ -6568,7 +6883,7 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +#define SVC_GR_INLINE_BYTES 256 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -6594,8 +6909,18 @@ static struct svc_fwd_cand svc_fwd_cand; * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the * 3D stream replay needs the window verbatim, not just its tail. Must equal * GSP_PHYS_GR_RAW_WORDS (static_assert below). - */ -#define SVC_GR_PUSH_WORDS 512 + * + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB). The DIRECT GL + * client (glcore, channel family 0xbeef) emits its per-frame render pushes + * at 5016-7144 B (1254-1786 dwords) with the draw state (program binds, + * vertex-array bases, constant uploads) spread across the whole push; the + * 512-dword tail kept only the release block, so the section capture could + * never satisfy `len == push_n*4` and the verbatim 3D replay never marked + * (CONT-44: zero src=stream captures on the client channel). The window + * slice still has to fit the backend's method block; only the raw[] + * retention is widened here. + */ +#define SVC_GR_PUSH_WORDS 2048 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -6700,8 +7025,14 @@ struct svc_gr_cand_rec { * side by side in a ring; the accessor macro addresses the current slot, * so every existing use keeps working on the selected candidate, and * capture/consume select the slot instead of clobbering it. + * + * SESSION 153-CONTINUED-45: 4 -> 8 slots. The DIRECT GL client's draw + * sections are 1470-dword windows split into up to four <=448-dword chunks + * (see svc_gl_stream_chunks); with four slots a single big push could + * evict its own later chunks (or the X channel's pending candidate) before + * the pump drained them. */ -#define SVC_GR_CAND_RING 4 +#define SVC_GR_CAND_RING 8 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) @@ -6821,8 +7152,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -7060,6 +7391,50 @@ MODULE_PARM_DESC(sema_pub_pa, "Force the publish page PA for sema_pub_adv (0 = content-discover per boot)"); /* + * SESSION 153-CONTINUED-36: the GL shared-state keeper (automates the + * slot-word handshake that gsp-phys-shared-bit0 previously did by hand). + * + * Every direct GLX client receives a shared page from the X driver whose + * word at +8 is a state field; glcore's device init reads it and SKIPS its + * own class-0xc46f channel creation while bit 0 is set. The emulated X + * driver publishes 3 (bits 0+1), so direct clients never allocate a + * channel and never render (witness: no c46f alloc in the RM log, glClear + * a no-op, err@ReadPixels=0x502). Clearing bit 0 while the client inits + * lets the channel alloc proceed (r999-r1002 by hand, r1012 by the + * debugfs write). The page moves per boot and is named here by guest PA; + * the keeper clears bit 0 on every service pass (up to ~1 kHz while the + * pump is active) so the client cannot re-publish 3 fast enough to block + * a later client's init. The PA is discovered by content (cookie word + * 0xc0b43fec at +4 / word5 0xcaf00006) with the host helper slot_hold.py + * or by hand, then written to this parameter; witness is bounded. + */ +static u64 gsp_sema_shared_hold_pa; +module_param_named(sema_shared_hold_pa, gsp_sema_shared_hold_pa, ullong, 0644); +MODULE_PARM_DESC(sema_shared_hold_pa, + "Guest PA of the GL shared-state page; clear bit 0 of its +8 word every pass (0 = off)"); +static u32 svc_shared_hold_log; + +static void svc_shared_hold(struct nv_vgpu_device *vgpu) +{ + u32 w[2]; + + if (!gsp_sema_shared_hold_pa) + return; + if (svc_read(vgpu, gsp_sema_shared_hold_pa + 8, w, sizeof(w))) + return; + if (!(w[0] & 1u)) + return; + w[0] &= ~1u; + if (svc_write(vgpu, gsp_sema_shared_hold_pa + 8, w, 4)) + return; + svc_shared_hold_log++; + if (svc_shared_hold_log <= 16 || + (svc_shared_hold_log % 4096) == 0) + pr_info("sema-shared: held pa=0x%llx word -> 0x%x (n=%u)\n", + gsp_sema_shared_hold_pa, w[0], svc_shared_hold_log); +} + +/* * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload. @@ -7071,6 +7446,7 @@ struct svc_gl_up { u8 data[SVC_GR_INLINE_BYTES]; }; +#define SVC_GL_PROG_MAX 8 struct svc_gl_sec { bool seen; u32 subch; @@ -7080,6 +7456,18 @@ struct svc_gl_sec { * host-sem/2D ops whose VAs are not mapped in the * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ u64 rel_va; /* last VALID release decoded on the channel */ @@ -7109,9 +7497,146 @@ struct svc_gl_sec { */ u8 blob[SVC_GR_INLINE_BYTES]; u32 blob_len; + /* + * SESSION 153-CONTINUED-18: the kick notification this push carried + * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the + * C597 bind that opened this section). Executed at flush time by + * svc_kick_notify_exec when both the target and the NOTIFY are in + * this same push (the short variant carries the NOTIFY without the + * arm pair; it never executes and stays witness-only). + */ + u64 kick_notify_va; + u32 kick_notify_val; + bool kick_notify_seen; }; static struct svc_gl_sec svc_gl; /* service thread only */ /* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* + * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. + * + * A SET_OBJECT bind is sticky on real hardware: the class stays bound to + * the channel subchannel across pushes until another bind replaces it. The + * GLX server's C597 SET_OBJECT arrives once (the driver's init push at + * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s) + * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour + * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the + * SET_OBJECT-only recognizer below never opens a section for them and the + * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597 + * bind is seen - including catches of the init push - and treat any later + * method run on that pair, on engine_type==0, as C597 class space. + */ +static u64 svc_gl_att_h; /* channel whose subch carries the C597 bind */ +static u32 svc_gl_att_subch; +static u32 svc_gl_draw_log; /* bounded draw-section-open witness */ +/* + * SESSION 153-CONTINUED-13: the C597 attachment is per-channel. The single + * (h_object, subch) slot was overwritten by the RM channel's periodic + * C597-bind pushes (h=0xbfef0002, observed every ~7 s on this boot), after + * which the GLX client's draw pushes on 0x1000013 failed the BEGIN + * recogniser's `svc_gl_att_h == ch->h_object` test and no draw section was + * ever opened (a boot-level, order-dependent race: the r954 boot decoded + * no such pushes). A small table (first match wins, dedup on note) keeps + * every channel's bind sticky independently, so the draw channel's + * attachment cannot be stolen. + */ +#define SVC_GL_ATT_MAX 8 +static u64 svc_gl_att_tab_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_tab_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + /* refresh LRU position: move to slot 0 */ + if (i) { + memmove(&svc_gl_att_tab_h[1], + &svc_gl_att_tab_h[0], + (size_t)i * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], + &svc_gl_att_tab_sub[0], + (size_t)i); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + } + return; + } + if (!svc_gl_att_tab_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) { + /* table full: evict the oldest (last) slot */ + free = SVC_GL_ATT_MAX - 1; + } + memmove(&svc_gl_att_tab_h[1], &svc_gl_att_tab_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], &svc_gl_att_tab_sub[0], + (size_t)free); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + /* keep the legacy single slot pointing at the newest note (logging) */ + svc_gl_att_h = h; + svc_gl_att_subch = subch; +} + +static bool svc_gl_att_has(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) + return true; + return false; +} + +static u32 svc_gl_att_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return svc_gl_att_tab_sub[i]; + return 0xffu; +} + +static void svc_gl_att_clear(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + u32 n = SVC_GL_ATT_MAX - 1 - i; + + if (n) + memmove(&svc_gl_att_tab_h[i], + &svc_gl_att_tab_h[i + 1], + (size_t)n * sizeof(u64)); + if (n) + memmove(&svc_gl_att_tab_sub[i], + &svc_gl_att_tab_sub[i + 1], + (size_t)n); + svc_gl_att_tab_h[SVC_GL_ATT_MAX - 1] = 0; + svc_gl_att_tab_sub[SVC_GL_ATT_MAX - 1] = 0; + if (svc_gl_att_h == h) + svc_gl_att_h = svc_gl_att_tab_h[0]; + return; + } +} +/* * SESSION 153 (G-B gate: the client's own C597 draws must open sections). * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which * returns true when the push VA sits in the GR candidate ring - even when @@ -7247,39 +7772,74 @@ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ static char *sema_gl_prog_hex; module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); MODULE_PARM_DESC(sema_gl_prog_hex, - "explicit program bytes (hex) for the synthesized GL launch (<=64 B)"); + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; static u32 svc_gl_prog_len; -static bool svc_gl_prog_parsed; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; -static void svc_gl_prog_apply_param(void) +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) { - const char *h = sema_gl_prog_hex; u32 n, i; - if (svc_gl_prog_parsed) - return; - svc_gl_prog_parsed = true; + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; if (!h || !h[0]) return; n = (u32)strlen(h); if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { - pr_warn("sema-gl: prog-hex param ignored (bad length %u)\n", - n); + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); return; } for (i = 0; i < n; i += 2) { unsigned int b; if (sscanf(h + i, "%2x", &b) != 1) { - pr_warn("sema-gl: prog-hex param bad at %u\n", i); + pr_warn("sema-gl: %s param bad at %u\n", tag, i); return; } - svc_gl_prog[i / 2] = (u8)b; + dst[i / 2] = (u8)b; } - svc_gl_prog_len = n / 2; - pr_info("sema-gl: prog-hex param: %u bytes stored\n", - svc_gl_prog_len); + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); } /* @@ -7320,6 +7880,63 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static u32 svc_gl_np_log; /* CONT.45 bounded no-param stream witness */ +static u32 svc_gl_mark_log; /* CONT.45 bounded 3D-mark-failure witness */ +/* + * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. + * + * The GL client's per-frame kick push on the kick-family channel + * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}] + * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA + * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16 + * status at +0xE). On real hardware the 2D engine writes a completed + * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) - + * that write is the client's per-frame completion edge; the lane used to + * log "flush skip" and drop it, leaving the member all-zero forever. This + * executor authors the NvNotification through the same trusted VA walk + * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies + * by readback. sema_kick_notify=0 disables (A/B); the witness line + * "sema-kick: exec ..." is bounded (first 48 + every 512th). + */ +static bool gsp_sema_kick_notify = true; +module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644); +MODULE_PARM_DESC(sema_kick_notify, + "Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)"); +static u32 svc_kick_log; /* bounded kick-notify witness */ +static u64 svc_kick_count; /* executor invocations */ +/* + * r982: the notify page a pending-marker hit proved, per knowledge the + * exec picked up. Kept in the service's own scope (not the channel + * snapshot): the kick channel is the only consumer and the entry is + * re-validated by a read before every use, so a stale PA can only cost + * one failed probe, never a write into freed memory. + */ +static struct { + u64 h; /* channel handle that proved the page */ + u64 pa; /* PA of the marker-proven notify page */ +} svc_kick_pa_cache[4]; + +static void svc_kick_pa_cache_store(u64 h, u64 pa) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (svc_kick_pa_cache[i].h == h) { + svc_kick_pa_cache[i].pa = pa; + return; + } + } + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (!svc_kick_pa_cache[i].h) { + svc_kick_pa_cache[i].h = h; + svc_kick_pa_cache[i].pa = pa; + return; + } + } + svc_kick_pa_cache[0].h = h; + svc_kick_pa_cache[0].pa = pa; +} + static bool gsp_sema_gl_draw_synth; module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); MODULE_PARM_DESC(sema_gl_draw_synth, @@ -7335,7 +7952,13 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, { u32 i, free = SVC_GL_HOLD_MAX; - if (!h || !prog || !param || !rel) + /* + * SESSION 153-CONTINUED-13: rel == 0 is admitted (a section that has + * real operands but flushed before the channel's release was decoded + * stashes them here); a later call with rel != 0 completes the entry + * and a rel == 0 call must NEVER wipe a stored release. + */ + if (!h || !prog || !param) return; for (i = 0; i < SVC_GL_HOLD_MAX; i++) { if (svc_gl_hold_tab[i].h == h) { @@ -7351,8 +7974,10 @@ static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, svc_gl_hold_tab[free].prog_va = prog; svc_gl_hold_tab[free].param_va = param; svc_gl_hold_tab[free].param_size = psize; - svc_gl_hold_tab[free].rel_va = rel; - svc_gl_hold_tab[free].rel_payload = rpay; + if (rel) { + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; + } } static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, @@ -7419,13 +8044,46 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) break; memcpy(&v[k], data + 4 * k, 4); } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. */ if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && - count >= 2 && !svc_gl.prog_va) { - svc_gl.prog_va = ((u64)(v[0] & 0xff) << 32) | - (u64)(v[1] & 0xfffffffc); + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } return; } switch (addr) { @@ -7510,12 +8168,555 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) } } return; + case 0x17c8: /* SET_INDEX_BUFFER_A/B: A=ADDRESS_UPPER(7:0), + * B=ADDRESS_LOWER(31:0). SESSION 153-CONTINUED-46: + * the indexed draws (glxgears' glDrawElements) fetch + * their indices from this VA; r1039 evidence: the + * client's 3D window binds it + * ("200205f2 ... data=00000000 0450fcc8" -> VA + * 0x0450fcc8) but the candidate page set held only the + * program page (n_pages=1) while X's retiring windows + * carry 3-6 pages - the index page is never mapped in + * the replay channel. Collect it like every other + * address operand. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x1c20: /* SET_VERTEX_STREAM_A_FORMAT/BASE(j) for j >= 2: + * same [fmt, BASE_UPPER, BASE_LOWER] shape as 0x1c00/ + * 0x1c10; the old switch only matched j=0/1. + */ + case 0x1c30: + case 0x1c40: + case 0x1c50: + case 0x1c60: + case 0x1c70: + case 0x1c80: + case 0x1c90: + case 0x1ca0: + case 0x1cb0: + case 0x1cc0: + case 0x1cd0: + case 0x1ce0: + case 0x1cf0: + case 0x1c00: /* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run + * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex + * stream the draw reads positions/colours from. + * Observed 0x1c00 [00001000 00000001 2150b790] and + * 0x1c10 [00001008 00000001 215257c0] (SESSION + * 153-CONTINUED-10 draw-push decode). + */ + case 0x1c10: + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x2380: /* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper, + * lower]. Observed [00010000 00000001 2150a500]. + */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x0800: /* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER, + * OFFSET_LOWER, w, h, fmt, ...] - the surface the + * draw rasterizes into. Observed [00000001 222fa200 + * 00000040 00000004 000000f7 00000000]. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; default: return; } } /* + * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect + * the C597 operand fields of every method run on the section's subchannel. + * Used when a draw section opens at its BEGIN marker (0x1618): the state + * methods that precede the marker (vertex stream binds, colour target, + * constant buffer binds) were decoded before the section existed, so their + * VAs must be re-collected or the candidate's page set starves and the + * replay's shadow relocation leaves them pointing at unreachable guest + * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main + * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only. + */ +static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, + u32 att_subch) +{ + u32 o = start; + + while (o + 4 <= end && o + 4 <= len) { + u32 hdr, op, count, subch, addr; + + memcpy(&hdr, buf + o, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + if ((op == 1 || op == 3 || op == 5) && count && + o + 4 + count * 4 <= len) { + if (subch == att_subch) + svc_gl_field(addr, count, buf + o + 4, + len - o - 4); + o += 4 + count * 4; + continue; + } + o += 4; + } +} + +/* + * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame + * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write + * is a completed NvNotification: nonzero timestamp at +0x0 (ns since + * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0 + * at +0xE. Resolution is the release path's trusted walk + * (svc_lookup_trusted -> svc_write_ap), then readback verification. + */ +static bool svc_kick_family(const struct svc_channel *ch); +static u64 svc_push_pa_ceiling(void); +static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 val, + u64 push_va) +{ + u64 pa = 0, ts = ktime_get_real_ns(); + int ap = SVC_APERTURE_SYS; + u8 old[16], nw[16], rb[16]; + bool ok = false, changed = false; + u32 o0 = 0, n0 = 0; + int rc = -ENOENT; + const char *rsrc = "trusted"; + u32 kc = 0; + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + memcpy(nw + 8, &val, sizeof(val)); + if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { + int r1, r2; + + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + rsrc = "trusted-u"; + } + svc_cur_vas = save; + } + /* + * r978: r977's exec fires (27 calls) but authors the word in + * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads, + * while r973 - the boot where the kick consumption visibly + * advanced the client - wrote the RING-ADJACENT sysmem slot + * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring + + * 0x1000 + 0xb0; the r975 probe measured the identical rule, + * pa=0x104b640b0 over ring 0x104b63000). Both walk answers + * miss that page this boot, so before falling back to the + * shadow: (a) the general lookup (the discovered rm_mem map - + * r979's sema_probe proved it resolves this VA to SYS after + * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin + * for this VA (the guest pin stream is the client's own + * statement of the mapping), then (c) for a kick-family + * channel the ring-relative completion slot, computed from + * the live ring pin each call. + */ + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 mpa = 0; + int map_ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &mpa, &map_ap) && + svc_ap_of(map_ap) == SVC_APERTURE_SYS) { + pa = mpa; + ap = map_ap; + rsrc = "map"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + svc_ap_of(iap) == SVC_APERTURE_SYS) { + pa = ipa; + ap = iap; + rsrc = "ipin"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch && + svc_kick_family(ch)) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, + &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 cand = (ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE + + (va & (SEMA_PAGE_SIZE - 1)); + + if (cand < svc_push_pa_ceiling()) { + pa = cand; + ap = SVC_APERTURE_SYS; + rsrc = "ring+10b0"; + } + } + } + /* + * r981: the client's pending notification marker. r979/r980 + * measured the REAL slot's +0xC u32 reading 0x80000000 + * while the client waited (and 0 once completed; the write + * below clears it) - on the page the discovered map names, + * ring + 0x20b0 on both boots, one page ABOVE the r973-era + * ring + 0x10b0 heuristic the exec had been writing + * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs + * 0x12e0790b0 on r980 - the completion landed on the wrong + * page both boots). When the chosen candidate does not show + * the marker, probe the ring-relative alternatives and use + * the one that does; the readback below still verifies the + * write. A slot already completed (marker 0) keeps the + * resolved candidate unchanged. + * + * r982: remember the page a pending-marker hit named, per + * handle. r981's boot: exec n=1 found the marker page + * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the + * marker is 0 afterwards so every later exec fell back to + * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes) + * while the map/trusted stages were still catching up. The + * cached page is retried at the top of the next call (read + * first, used only while the read succeeds), so all later + * completions stay on the page the marker proved. + */ + if (ch && svc_kick_family(ch)) { + /* Retry the page a marker hit proved, if still readable. */ + for (kc = 0; kc < 4; kc++) { + u32 probe_buf = 0; + + if (svc_kick_pa_cache[kc].h != ch->h_object || + !svc_kick_pa_cache[kc].pa) + continue; + if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa, + SVC_APERTURE_SYS, &probe_buf, + sizeof(probe_buf)) && + probe_buf != 0xffffffffu) { + pa = svc_kick_pa_cache[kc].pa; + ap = SVC_APERTURE_SYS; + rsrc = "kickcache"; + } + break; + } + } + if (ch && svc_kick_family(ch)) { + u32 pend = 0; + + if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap), + &pend, sizeof(pend)) && + pend != 0x80000000u) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, + &ring_pa, &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 base = ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1)); + u64 cands[2]; + int i; + + cands[0] = base + + 2 * SEMA_PAGE_SIZE + off; + cands[1] = base + + SEMA_PAGE_SIZE + off; + for (i = 0; i < 2; i++) { + u32 m = 0; + + if (cands[i] >= + svc_push_pa_ceiling()) + continue; + if (!svc_read_ap(vgpu, + cands[i] + 0xc, + SVC_APERTURE_SYS, + &m, sizeof(m)) && + m == 0x80000000u) { + pa = cands[i]; + ap = SVC_APERTURE_SYS; + rsrc = i ? + "ring+10b0p" : + "ring+20b0p"; + svc_kick_pa_cache_store( + ch->h_object, + cands[i]); + break; + } + } + } + } else if (pend == 0x80000000u && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + /* + * The chosen candidate IS the marker page: + * prove it for the cache so later execs keep + * writing the same page (r981: after the + * single marker hit the fallback wrote two + * wrong ring-relative pages for 33 further + * execs). Only a SYS answer that showed the + * marker is cached; a zero read (already + * completed) caches nothing - unproven. + */ + svc_kick_pa_cache_store(ch->h_object, pa); + } + } + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); + if (!r1 && !r2 && + !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) { + rc = 0; + ok = true; + changed = memcmp(old, rb, sizeof(old)) != 0; + memcpy(&o0, old, 4); + memcpy(&n0, rb, 4); + } else { + /* CONT.18 probe: name the failing step. */ + rc = r1 ? (r1 < 0 ? r1 : -100) : + (r2 ? (r2 < 0 ? r2 : -101) : -102); + } + } + svc_kick_count++; + if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { + svc_kick_log++; + pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", + ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val, + (unsigned long long)ts, o0, n0, changed ? 1 : 0, + (unsigned long long)svc_kick_count, rc); + } + /* + * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting + * client's completion edge - post the channel event so its OS event + * fires (r934's lesson on the notify2d path: without the event the + * GL client's kick submit wait never completes and the app retries + * on a ~16-22 s timer; r972 measured exactly that cadence with the + * event missing here). Only for a real channel (the dev probe + * passes ch=NULL). + */ + if (ok && ch) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return ok; +} + +/* + * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify + * executor on an explicit VA against the service's own device. The + * execution MUST run in a kernel-thread context: vfio_iommu_type1's + * dma_rw only copies when current->mm == dma->mm or the caller is a + * kthread (kthread_use_mm); a sysfs write from a shell process has + * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The + * workqueue item below is a kthread - the same context class as the + * sema service thread that runs the real kick lane. + * echo 0x12000e0b0 > .../parameters/sema_kick_probe + */ +static u64 svc_kick_probe_va; +static void svc_kick_probe_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn); + +static void svc_kick_probe_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (vgpu) + svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0); +} + +static int nv_vgpu_sema_kick_probe_set(const char *val, + const struct kernel_param *kp) +{ + if (!val || kstrtoull(val, 0, &svc_kick_probe_va)) + return -EINVAL; + if (svc_kick_probe_va) + schedule_work(&svc_kick_probe_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = { + .set = nv_vgpu_sema_kick_probe_set, +}; + +module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200); +MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)"); + +/* + * SESSION 153-CONTINUED-44 (r1026): host-driven present blit. + * + * The client's rendered frame provably exists in guest RAM (its readback + * staging) while the X server scans the emulated screen surface in vidmem + * (base 0x200000, pitch 5120 for the 1280x720 4-bpp screen), so the window + * shows only the DDX's cleared fill artifact. This is a bounded, explicit + * present emulation: the host names the guest-PA staging of the CLIENT'S + * OWN GPU-rendered frame, and the kernel blits those rows into the scanned + * surface - never synthesizing pixels. The host drives it with: + * + * echo <src_gpa>,<src_pitch>,<w>,<h>[,<dst_off>,<dst_pitch>] \ + * > .../parameters/sema_screen_blit + * + * (all values 0x-hex or decimal; src_pitch and w/h in bytes/pixels, bpp=4; + * defaults dst_off=0x200000, dst_pitch=5120). Bounded: w,h <= 4096, + * pitch <= 65536, total <= 64 MiB, one row buffer. Runs on a workqueue + * (vfio_dma_rw for the guest read, bars[1] shadow for the vidmem write), + * logs one line with the byte/row counts. + */ +#define SVC_SCREEN_BLIT_DEF_OFF 0x200000ULL +#define SVC_SCREEN_BLIT_DEF_PITCH 5120u +static u64 svc_screen_blit_gpa; +static u32 svc_screen_blit_spitch, svc_screen_blit_w, svc_screen_blit_h; +static u64 svc_screen_blit_doff; +static u32 svc_screen_blit_dpitch; +static void svc_screen_blit_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_screen_blit_work, svc_screen_blit_work_fn); + +static void svc_screen_blit_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + u8 *row, *base; + size_t size; + u32 y, w4 = svc_screen_blit_w * 4; + u64 bytes = 0; + int i, rc = 0; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!vgpu) { + pr_info("sema-screen-blit: no vgpu (no channel registered yet)\n"); + return; + } + base = vgpu->bars[1].vaddr; + size = svc_fb_shadow_size(vgpu); + if (!base || svc_screen_blit_doff + (u64)svc_screen_blit_dpitch * + svc_screen_blit_h > size) { + pr_info("sema-screen-blit: bounds (off=0x%llx pitch=%u h=%u size=%zu)\n", + svc_screen_blit_doff, svc_screen_blit_dpitch, + svc_screen_blit_h, size); + return; + } + row = kmalloc(svc_screen_blit_spitch, GFP_KERNEL); + if (!row) { + pr_info("sema-screen-blit: no memory for row (pitch=%u)\n", + svc_screen_blit_spitch); + return; + } + for (y = 0; y < svc_screen_blit_h; y++) { + rc = svc_read(vgpu, svc_screen_blit_gpa + (u64)y * + svc_screen_blit_spitch, row, + svc_screen_blit_spitch); + if (rc) + break; + memcpy(base + svc_screen_blit_doff + (u64)y * + svc_screen_blit_dpitch, row, w4); + bytes += w4; + } + kfree(row); + pr_info("sema-screen-blit: src=0x%llx spitch=%u w=%u h=%u -> off=0x%llx dpitch=%u rows_done=%u bytes=%llu rc=%d\n", + svc_screen_blit_gpa, svc_screen_blit_spitch, + svc_screen_blit_w, svc_screen_blit_h, + svc_screen_blit_doff, svc_screen_blit_dpitch, y, bytes, rc); +} + +static int nv_vgpu_sema_screen_blit_set(const char *val, + const struct kernel_param *kp) +{ + char buf[128]; + int nf = 0; + u64 v[6] = {0}; + + if (!val || strlen(val) >= sizeof(buf)) + return -EINVAL; + strscpy(buf, val, sizeof(buf)); + { + char *p = buf; + + while (nf < 6 && p && *p) { + char *c = strchr(p, ','); + + if (c) + *c++ = '\0'; + if (kstrtoull(p, 0, &v[nf])) + return -EINVAL; + nf++; + p = c; + } + } + if (nf < 4) + return -EINVAL; + svc_screen_blit_gpa = v[0]; + svc_screen_blit_spitch = (u32)v[1]; + svc_screen_blit_w = (u32)v[2]; + svc_screen_blit_h = (u32)v[3]; + svc_screen_blit_doff = nf > 4 ? v[4] : SVC_SCREEN_BLIT_DEF_OFF; + svc_screen_blit_dpitch = nf > 5 ? (u32)v[5] : + SVC_SCREEN_BLIT_DEF_PITCH; + if (!svc_screen_blit_gpa || !svc_screen_blit_w || + !svc_screen_blit_h) + return -EINVAL; + if (svc_screen_blit_w > 4096 || svc_screen_blit_h > 4096 || + svc_screen_blit_spitch > 0x10000 || + svc_screen_blit_spitch < svc_screen_blit_w * 4 || + svc_screen_blit_dpitch > 0x10000 || + svc_screen_blit_dpitch < svc_screen_blit_w * 4 || + (u64)svc_screen_blit_spitch * svc_screen_blit_h > (64u << 20)) + return -EINVAL; + schedule_work(&svc_screen_blit_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_screen_blit_ops = { + .set = nv_vgpu_sema_screen_blit_set, +}; + +module_param_cb(sema_screen_blit, &nv_vgpu_sema_screen_blit_ops, NULL, 0200); +MODULE_PARM_DESC(sema_screen_blit, + "Blit a guest-RAM frame into the scanned vidmem surface: src_gpa,src_pitch,w,h[,dst_off,dst_pitch] (session 153 CONT.44, dev present emulation)"); + + +/* * Close the open section: synthesize the QMD from the collected fields and * capture it as the forwarding candidate. The QMD word layout mirrors * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 @@ -7524,6 +8725,217 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * (synthesized - no QMD buffer exists in guest memory) so the backend uses * its qmdwin/scratch path. */ + +/* + * SESSION 153-CONTINUED-45 (task 1): big-window verbatim replay. + * + * The direct GL client (glcore, channel family 0xbeef) renders through + * 5016-7144 B pushes whose C597 section window is 1253-1470 dwords - far + * beyond the backend's 448-dword method-block window (any window slice goes + * verbatim through the module pushbuffer page). The engine executes 3D + * methods from ANY number of sequential ring entries with channel state + * persisting across them (that is how real drivers split work), and the + * forward pump already forwards ONE candidate per pass FIFO per channel + * (svc_gr_request picks THIS channel's oldest unconsumed candidate). So the + * window is split into method-aligned chunks of <=448 dwords, each enqueued + * as its own candidate in ring order; successive passes replay them in + * order and the physical channel accumulates the section's state, exactly + * as if the one big window had executed. + * + * Boundary rule: a chunk ends at the last METHOD boundary that fits the + * budget - methods are never split across chunks (a run's data words would + * otherwise be orphaned). The QMD/rel/inline records are cloned from the + * section's base candidate; only stream_off/stream_n differ, so every + * chunk launch carries the same release identity and program images. + */ +#define SVC_GL_STREAM_CHUNK_MAX 6 +/* + * Words per chunk: 384 is the largest window the physical engine is + * PROVEN to retire (r997: `3D stream marked h=0x1000013 + * push=0x120010004 window=384 words` -> `GUEST GR LAUNCH OK`), + * while the client's 394-word first window schedule-stalled its + * replay (r1030: grfifo frozen at 1, fecs idle, timeout + software + * release). Stay under the proven bound. + * SESSION 153-CONTINUED-45: runtime-settable for the bisect - the + * /sys/module/vgsp/parameters/sema_gl_chunk_words knob overrides it so a + * live client's window can be cut to 64/128/256 words to find the first + * length (or starting method) that schedules and retires. + */ +static unsigned svc_gl_chunk_words_param; +module_param_named(sema_gl_chunk_words, svc_gl_chunk_words_param, uint, 0644); +MODULE_PARM_DESC(sema_gl_chunk_words, + "3D stream chunk size in words for the chunked verbatim replay (0 = built-in 384)"); +#define SVC_GL_STREAM_CHUNK_WORDS 384u + +static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, + u32 at_off, u32 max_words) +{ + u32 off = start; + u32 limit = start + max_words * 4u; + u32 last = start; + + while (off + 4 <= at_off && off + 4 <= len) { + u32 hdr, op, cnt, nxt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + if (op == 4 || op == 7 || !cnt) + nxt = off + 4; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u <= len && + off + 4 + cnt * 4u <= at_off) + nxt = off + 4 + cnt * 4u; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u > at_off) + /* + * SESSION 153-CONTINUED-45: a run whose data the + * window edge would cut MUST NOT be a boundary - + * ending a chunk on its header alone makes the + * PBDMA fault mid-method (r1031 PB-FAULT-DUMP P0 + * hdr=20164010 cnt=22, its 22 data words absent) + * and the whole replay dies silently. Stop before + * it; the orphaned head is left out of the replay. + */ + break; + else + nxt = off + 4; /* resync word, as the decoders */ + if (nxt > limit) + break; + last = nxt; + off = nxt; + } + if (last == start) { + /* One method alone overruns the budget: take what fits so the + * window makes progress instead of wedging. */ + last = start + max_words * 4u; + if (last > at_off) + last = at_off; + if (last <= start) + last = start + 4u <= at_off ? start + 4u : at_off; + } + return last; +} + +static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, + u32 len, u32 at_off) +{ + u32 cur = svc_gl.start_off; + u32 n = 0, pending = 0, ri; + + if (!svc_gr_cand.valid) + return; + /* + * Backpressure: one frame's window is self-contained (it re-emits the + * whole state), so while the previous frame's chunks are still being + * replayed there is no reason to queue another window - it would only + * evict in-flight chunks and execute a torn state sequence. Count + * this channel's unconsumed candidates (including the base just + * captured); at >= 8 (a full frame's 4 chunks plus slack) the new + * capture is dropped whole: the base candidate is invalidated so the + * pass-end request cannot forward it down the compute path either. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].h_object == ch->h_object) + pending++; + if (pending >= SVC_GR_CAND_RING) { + if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: stream chunks backpressure h=0x%x push=0x%llx pending=%u -> capture dropped\n", + ch->h_object, svc_gr_cand.push_va, pending); + } + svc_gr_cand.valid = false; + return; + } + while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { + u32 cw = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; + u32 end; + + if (cw > 448u) + cw = 448u; + end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + struct svc_gr_cand_rec *c = &svc_gr_cand; + + if (n) { + u32 ri, found = SVC_GR_CAND_RING; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + found = rp; + break; + } + } + if (found == SVC_GR_CAND_RING) { + pr_info("sema-gl: stream chunks: no free ring slot after %u chunks h=0x%x push=0x%llx (window truncated)\n", + n, ch->h_object, svc_gr_cand.push_va); + break; + } + svc_gr_cand_ring[found] = svc_gr_cand; + c = &svc_gr_cand_ring[found]; + } + c->valid = true; + c->offered = false; + c->stream_3d = true; + c->stream_off = cur / 4u; + c->stream_n = (end - cur) / 4u; + n++; + cur = end; + } + if (n) { + pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", + ch->h_object, svc_gr_cand.push_va, + (at_off - svc_gl.start_off) / 4u, n); + } + /* + * SESSION 153-CONTINUED-46: the first chunk's full method stream. + * Every beef chunk launch stalls the physical front-end (grfifo + * frozen, fecs_cur=0, PBDMA stopped mid-block at a + * SET_VERTEX_ATTRIBUTE_A run while X windows retire) and the window + * tail (`3DSTREAM ... tail=`) alone cannot name the stalling method. + * Dump the first chunk's words (up to 96) as operand lines so the + * exact method list is on record; bounded per boot. + */ + { + static u32 chunkdump_log; + + if (chunkdump_log < 4 && svc_gl.start_off + 8u <= at_off) { + u32 cur2 = svc_gl.start_off, end2, k2, wcnt; + u32 cw2 = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : + SVC_GL_STREAM_CHUNK_WORDS; + + chunkdump_log++; + if (cw2 > 448u) + cw2 = 448u; + end2 = svc_gl_chunk_boundary(buf, len, cur2, at_off, + cw2); + wcnt = (end2 - cur2) / 4u; + if (wcnt > 96u) + wcnt = 96u; + pr_info("sema-gl: CHUNK1 h=0x%x push=0x%llx off=0x%x words=%u (dump up to 96)\n", + ch->h_object, svc_gr_cand.push_va, cur2, + (end2 - cur2) / 4u); + for (k2 = 0; k2 + 4 <= wcnt; k2 += 4) + pr_info("sema-gl: CHUNK1[%02u] %08x %08x %08x %08x\n", + k2, + get_unaligned_le32(buf + cur2 + 4u * k2), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 1)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 2)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 3))); + for (; k2 < wcnt; k2++) + pr_info("sema-gl: CHUNK1[%02u] %08x\n", k2, + get_unaligned_le32(buf + cur2 + 4u * k2)); + } + } +} + static void svc_gl_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, const struct svc_push_state *st, const u8 *buf, @@ -7531,7 +8943,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 at_off) { struct svc_push_state s2; - struct gsp_bridge_inline pr; + struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; u32 n_recs = 0, k; /* * SESSION 149 R16 (M3): the inline attachment's source. Priority: @@ -7597,14 +9009,147 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { - if (gsp_sema_dbg && svc_gl_skip_log < 16) { + /* + * SESSION 153-CONTINUED-45 (task 1): the + * DIRECT GL client's (glcore) per-frame draw + * sections bind a program (SET_PIPELINE_ + * PROGRAM_ADDRESS) and carry the bracketing + * release, but no tex-header pool - the + * sticky pool was bound in an earlier push. + * The verbatim 3D replay path never reads + * param_va (the synthesized compute fields + * are what stream_3d bypasses), so a + * prog+rel section whose window fits the + * replay method block is capturable as a + * real stream window with param=0. Bounded + * to the GL stream lane and the fit so the + * historical skip still guards everything + * else; CONT-44 evidence: every client draw + * flush-skipped here with prog/rel real + * (`flush skip h=0xbeef0100 ... prog=0x4390900 + * param=0x0 rel=0x519c0a0`). + */ + bool stream_noparam = false; + + /* + * SESSION 153-CONTINUED-45: the push page is + * cyclically zeroed/rewritten by the client + * (watched live: real -> ZERO -> new content + * on a ~6 s cadence); a capture landing in the + * zero window decodes garbage (prog found + * deeper, window head all zeros) and its + * replay makes the PBDMA fault on the zero + * lead (r1031 PB-FAULT-DUMP P0). Only capture + * when the window head is a real word - a + * later pass/entry on the filled page captures + * the frame. + */ + if (at_off > svc_gl.start_off && + svc_gl.start_off + 4u <= len) { + u32 w0h; + + memcpy(&w0h, buf + svc_gl.start_off, 4); + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (page mid-write; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + } + /* + * SESSION 153-CONTINUED-46: the admission + * bound uses the BUILT-IN 384-word chunk + * budget, never the sema_gl_chunk_words + * bisect knob. r1033 evidence: with the knob + * at 64 the bound shrank to 6*64=384 words + * and the client's real 1253-word windows + * were rejected outright ("flush skip ... + * win=1253 words (no operands, no hold)") - + * the knob is meant to cut the replayed + * chunks smaller, not to stop the capture. + * With the full bound every real window is + * admitted and chunked (the knob then + * truncates the replayed words, logged). + */ + if (svc_gl.prog_va && svc_gl.rel_va && + !svc_gl.param_va && + gsp_sema_gl_xlate && !ch->engine_type && + op_src[0] != 'd' && + at_off > svc_gl.start_off && + (at_off - svc_gl.start_off) / 4u <= + SVC_GL_STREAM_CHUNK_MAX * + SVC_GL_STREAM_CHUNK_WORDS) + stream_noparam = true; + if (!stream_noparam) { + /* + * SESSION 153-CONTINUED-13: the release may be + * decoded AFTER this section in ring order + * (the r955 init: the 24 B release pushes sit + * behind the 1524 B init push in the ring, so + * every replay pass of the section flushed + * before svc_gl.rel_va was known). When the + * operands are real but only the release is + * missing, stash them for the channel so the + * next section/pass (by then the constant + * release stream has been decoded) captures + * with src=hold instead of losing the init + * capture - without which the guest's GLX + * init completion degrades and the client + * never emits its draw pushes at all. + */ + if (svc_gl.prog_va && svc_gl.param_va && + !svc_gl.rel_va) + svc_gl_hold_store(ch->h_object, + svc_gl.prog_va, + svc_gl.param_va, + svc_gl.param_size, + 0, 0); + /* + * SESSION 153-CONTINUED-18 (task 1): replace + * the old "flush skip" drop with NOTIFY + * EXECUTION when this push carried the kick + * member (SET_NOTIFY_A/B + NOTIFY in the same + * push). The section has no draw operands - + * it is the client's completion kick - so the + * GPU's notification write is the only work + * it asks for. + */ + if (gsp_sema_kick_notify && + svc_gl.kick_notify_seen && + svc_gl.kick_notify_va && + svc_kick_notify_exec(vgpu, ch, + svc_gl.kick_notify_va, + svc_gl.kick_notify_val, + push_va)) { + /* authored once; a replay pass must not + * re-author an unchanged member. */ + svc_gl.kick_notify_seen = false; + svc_gl.kick_notify_va = 0; + goto out; + } + if (gsp_sema_dbg && svc_gl_skip_log < 2048) { svc_gl_skip_log++; - pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx win=%u words (no operands, no hold)\n", ch->h_object, push_va, svc_gl.prog_va, svc_gl.param_va, - svc_gl.rel_va); + svc_gl.rel_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u); } goto out; + } else if (gsp_sema_dbg && svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: stream capture no-param h=0x%x push=0x%llx prog=0x%llx rel=0x%llx win=%u words (real draw operands; pool param bound elsewhere)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.rel_va, + (at_off - svc_gl.start_off) / 4u); + } } } } @@ -7748,27 +9293,41 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } } memset(blob, 0, sizeof(blob)); - blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, - sizeof(blob)); - if (!blob_rc) { - u32 z = 0; - - for (k = 0; k < 16; k++) - z |= blob[k]; - if (z) - blob_trusted = true; - } - if (blob_trusted) { - blob_n = sizeof(blob); - blob_src = "trusted-prog"; - } else if (svc_gl_prog_len) { + /* + * SESSION 153-CONTINUED-10: the explicit operator program override + * (sema_gl_prog_hex) wins over the trusted page read. The GLX + * server's per-frame CONTENT pushes bind their own program slots + * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated + * nonzero bytes at that offset; the R16 trusted-read priority + * accepts any nonzero 16 bytes as "the real program" and attached + * garbage, and the engine rejected the draw with "Graphics + * Exception: Shader Program Header 11/18 Error" (r944 fault + * evidence, push 0x120014578). An unset override keeps the + * historical read priority unchanged. + */ + if (svc_gl_prog_len) { blob_n = svc_gl_prog_len; memcpy(blob, svc_gl_prog, blob_n); blob_src = "prog-hex"; - } else if (svc_gl.blob_len) { - blob_n = svc_gl.blob_len; - memcpy(blob, svc_gl.blob, blob_n); - blob_src = "lid-fallback"; + } else { + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } } /* * r929 (Path A): a stream-derived capture is a 3D method window - the @@ -7776,12 +9335,69 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * uploads/binds, so the synthesized compute attachments (the inline * program record, and with it the prog-hex fallback) do not apply. * Draw-synth/hold captures keep the old compute path unchanged. - */ - if (blob_n && op_src[0] != 's') { - memset(&pr, 0, sizeof(pr)); - pr.dst_va = svc_gl.prog_va; - pr.len = blob_n; - memcpy(pr.data, blob, blob_n); + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); n_recs = 1; } /* @@ -7790,11 +9406,11 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * replays of the same (push_va, content hash). */ svc_gl_cap_store(push_va, svc_push_hash(buf, len)); - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u, op_src); + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); /* * SESSION 153: remember real operands for this channel so a later * draw-only section on the same channel can still capture. A @@ -7820,7 +9436,7 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, get_unaligned_le32(blob + 8), get_unaligned_le32(blob + 12)); } - svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, push_va); /* * r929 (Path A): a stream-derived capture (op_src=stream: the section @@ -7838,8 +9454,18 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, * 448-word cap leaves headroom for the backend's 512-word method * block (2 bind words + the window + the 8-word release tail). * Draw-synth/hold captures are never marked. - */ - if (op_src[0] == 's' && svc_gr_cand.valid && + * + * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX + * server's per-frame CONTENT pushes carry the draw window but no + * program/param operands (the class bind is sticky; no per-push + * SET_OBJECT), so their sections capture with the channel's real + * operands adopted from the hold table (filled by the init + * capture). The window itself is still the REAL push bytes sliced + * from its own first C597 state method to its END - exactly the + * raw 3D method window the r929 replay lane wants; the synthesized + * compute fields are what stream_3d bypasses. + */ + if ((op_src[0] == 's' || op_src[0] == 'h') && svc_gr_cand.valid && len == svc_gr_cand.push_n * 4 && svc_gl.start_off + 8u <= at_off && (at_off - svc_gl.start_off) / 4u <= 448u) { @@ -7849,6 +9475,34 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", ch->h_object, push_va, svc_gr_cand.stream_n, svc_gl.start_off); + } else if ((op_src[0] == 's' || op_src[0] == 'h') && + svc_gr_cand.valid) { + /* + * SESSION 153-CONTINUED-45: a REAL-operand section whose + * window cannot ride the single 448-dword block - split it + * into method-aligned chunks, each a verbatim replay in ring + * order (channel state persists across launches). The raw[] + * window must still hold the WHOLE push (the slices index + * into it), which is why the per-record raw bound was + * widened; if the raw geometry does not hold, the bounded + * diagnostic below names the failing condition (the next + * bound to size). + */ + if (len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off) { + svc_gl_stream_chunks(ch, buf, len, at_off); + } else if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark failed h=0x%x push=0x%llx win_start=0x%x at=0x%x win=%u words raw_ok=%d fit_ok=%d (rawlen=%u pushn=%u len=%u)\n", + ch->h_object, push_va, svc_gl.start_off, + at_off, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u, + len == svc_gr_cand.push_n * 4, + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u, + len / 4u, svc_gr_cand.push_n, len); + } } out: /* @@ -11025,6 +12679,14 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, static struct { u64 va_page; u64 pa_page; + /* + * SESSION 153-CONTINUED-38: the VAS the page was derived for. The + * counter VA window (0x20400fXXX) is a fixed per-client layout, so + * two clients (the RC scrubber and the DDX) register the SAME VA and + * a VA-keyed cache/derive hands the second one the first one's page; + * that is the slot-ownership collision CONT-37 measured. 0 = any. + */ + u32 h_vas; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; static u32 svc_cnt_der_log; @@ -11039,7 +12701,9 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) bool fresh = true; for (i = 0; i < svc_cnt_der_n; i++) { - if (svc_cnt_der[i].va_page == page) { + if (svc_cnt_der[i].va_page == page && + (!svc_cur_vas || !svc_cnt_der[i].h_vas || + svc_cnt_der[i].h_vas == svc_cur_vas)) { pa = svc_cnt_der[i].pa_page | off; fresh = false; break; @@ -11060,7 +12724,27 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) * it (r194: the counter page 0x20400f000 never resolved; its * walk hit probed unreadable and targeted was never reached). */ - if (!svc_walk_vaspaces(vgpu, page, &p, &a) && + /* + * SESSION 153-CONTINUED-38: the channel's OWN page tables + * answer first. The global walk sources can only name one + * page for the shared counter VA (0x20400fXXX is a fixed + * per-client layout), so a second client's slot derivation + * collided with the first client's page and its registration + * was rejected forever (CONT-37: "slot owned by another + * channel"). The per-channel walk (svc_cur_vas, set by the + * pump for the channel being serviced) names the page the + * client itself polls. + */ + if (!ok && svc_cur_vas && + !svc_chanvas_lookup(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "chanvas"; + } + if (!ok && !svc_walk_vaspaces(vgpu, page, &p, &a) && a == SVC_APERTURE_SYS && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, @@ -11106,6 +12790,7 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) svc_cnt_der[svc_cnt_der_n].va_page = page; svc_cnt_der[svc_cnt_der_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der[svc_cnt_der_n].h_vas = svc_cur_vas; svc_cnt_der_n++; } *pa_out = pa; @@ -11113,6 +12798,56 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) } /* + * SESSION 153-CONTINUED-38: counter-slot disambiguation for a channel with + * no usable hVASpace (kernel channels register vas=0x0, so svc_cur_vas + * cannot answer). Two clients' channels can share a rank - the rank formula + * then names the SAME slot VA in each client's own address space - and the + * global walk sources answer with only one client's page, so the second + * client's autoreg collided with the first client's registered slot and was + * rejected forever (CONT-37: the GL client's beef0100 rank=2 vs the X + * client's bfef0002, both 0x2132acfd0). Walk EVERY registered VAS and + * return the first page that is readable AND different from avoid_pa: that + * is the second client's own page for the same VA. + */ +static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va, + u64 avoid_pa, u64 *pa_out) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + int i, n = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + u64 p = 0; + int a = SVC_APERTURE_SYS; + u32 probe; + + if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + page, local[i].ap, &p, &a, false)) + continue; + if (a != SVC_APERTURE_SYS) + continue; + p = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if ((p & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (avoid_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + if (svc_read_ap(vgpu, p, SVC_APERTURE_SYS, &probe, + sizeof(probe))) + continue; + *pa_out = p; + return 0; + } + return -ENOENT; +} + +/* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips its GR * channels in ascending handle order onto DESCENDING 16-byte slots, so the @@ -11227,6 +12962,47 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (collision) { + /* + * SESSION 153-CONTINUED-38: the slot VA is shared by + * rank across clients (each client's rank-r channel + * polls 0x20400fec0+(TOTAL-1-r)*0x10 in its OWN VAS), + * so a collision with another client's registered slot + * means the global derive answered with the OTHER + * client's page. Re-resolve through every registered + * VAS and keep the first readable page that differs - + * that page is this client's own slot. Only when no + * such page exists is the collision real and the + * registration must stay pending. + */ + u64 pa2 = 0; + + if (!svc_counter_derive_not(vgpu, va, pa, &pa2)) { + bool coll2 = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && + svc.chan[j].h_object != + ch->h_object && + svc.chan[j].counter_pa == + (pa2 & ~0x3ULL)) { + coll2 = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!coll2) { + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (own-VAS page; contender owner page 0x%llx kept)\n", + ch->h_object, ch->gr_index, + pa2 & ~0x3ULL, pa); + pa = pa2 & ~0x3ULL; + collision = false; + } + } + } if (!collision) { /* * Persist the registration into the authoritative @@ -11243,6 +13019,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, * hung. The authoritative store makes the * registration durable and the log one-shot. */ + spin_lock_irqsave(&svc.chan_lock, flags); ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && @@ -11252,8 +13029,8 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); } - spin_unlock_irqrestore(&svc.chan_lock, flags); if (collision) { pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", ch->h_object, ch->gr_index, pa); @@ -12019,16 +13796,34 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, if (!ch->notify2d_va) return false; if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { - if (gsp_sema_dbg && svc_notify2d_log < 16) { + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { svc_notify2d_log++; pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", ch->h_object, ch->notify2d_va); } return false; } - if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) - return false; - svc_notify2d_done++; + /* + * SESSION 153-CONTINUED-18: read the first word for the witness + * (the kick member's timestamp slot: 0 = never GPU-written; the + * completed write makes it nonzero - see svc_kick_notify_exec). + */ + { + u32 o0 = 0; + + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0))) + o0 = 0xffffffff; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return false; + svc_notify2d_done++; + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n", + ch->h_object, ch->notify2d_va, pa, o0); + } + } /* * r834: pin the page that accepted the ack; the same stale-map * defense as the push-side pin in the decode. @@ -12045,14 +13840,294 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, */ if (completion) gsp_rm_post_channel_event(vgpu, ch->h_object); - if (gsp_sema_dbg && svc_notify2d_log < 16) { - svc_notify2d_log++; - pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n", - ch->h_object, ch->notify2d_va, pa); - } return true; } +/* ------------------------------------------------------------------ */ +/* FERMI_TWOD (0x902d) executor */ +/* ------------------------------------------------------------------ */ +/* + * SESSION 153-CONTINUED-26: the X DDX paints its surfaces (the screen + * 0x120400000 1280x720 X8R8G8B8 and the cursor 0x120370000 256x256 + * A8R8G8B8) exclusively through subchannel-3 FERMI_TWOD methods: the + * SET_DST_* group, then RENDER_SOLID_PRIM (mode RECTS, color format, + * solid color, RENDER_SOLID_PRIM_POINT_X_Y packed pairs). Without this + * executor every 2D surface paint was a no-op and the surfaces the CE + * readback reads stayed unwritten (the screen byte pattern is all + * 0x00000000 - "the X server's drawing never executes"). + * + * Method map: cl902d.h (FERMI_TWOD_A); encodings: NVA06F_DMA_* + * (1 INC / 3 NON_INC / 4 IMMD_DATA, data in header bits 28:16 / 5 + * ONE_INC). Live-decoded per-frame dialog (guest RAM dump at + * 0x8d30c5f8): NON_INC 0x100 cnt17 (MME/global-render-enable window), + * INC 0x280 clip {0,0,0x7fff,0x7fff}, IMMD 0x204 layout, INC 0x214 + * pitch, INC 0x218 {w,h,upper,lower}, IMMD 0x200 fmt, IMMD 0x2ac + * operation, INC 0x580 {mode,fmt,color}, NON_INC 0x5e0 {packed x/y + * point pairs}, IMMD 0x10c NOTIFY, IMMD 0x100 NOP. + * + * The DST VA is (upper<<32)|lower resolved through the SAME operand + * machinery every other engine op uses (svc_fill), so the 2D painting + * and the DDX's own CE readback of the same VA always agree by + * construction. + */ +struct svc_twod_state { + u64 h_object; + u32 dst_fmt, dst_layout, dst_bsize, dst_pitch, dst_w, dst_h; + u64 dst_off; + u32 have_dst; + u32 operation, rop; + u32 prim_mode, prim_fmt, prim_color; + u32 pt_x, pt_y, have_pt; + u64 fills, fill_bytes, misses; + u64 last_fill_hash, last_fill_va; + u32 fill_skips; + u32 dst_log, fill_log, unk_log; +}; + +#define SVC_TWOD_STATES 4 +static struct svc_twod_state svc_twod_tab[SVC_TWOD_STATES]; + +unsigned gsp_sema_twod = 1; +module_param_named(sema_twod, gsp_sema_twod, uint, 0644); +MODULE_PARM_DESC(sema_twod, + "Execute FERMI_TWOD (0x902d) fills on subchannel 3 (default 1)"); + +static struct svc_twod_state *svc_twod_get(struct svc_channel *ch) +{ + u32 i, free_i = SVC_TWOD_STATES; + + for (i = 0; i < SVC_TWOD_STATES; i++) { + if (svc_twod_tab[i].h_object == ch->h_object) + return &svc_twod_tab[i]; + if (!svc_twod_tab[i].h_object && free_i == SVC_TWOD_STATES) + free_i = i; + } + if (free_i == SVC_TWOD_STATES) + free_i = 0; /* bounded table: steal slot 0 */ + memset(&svc_twod_tab[free_i], 0, sizeof(svc_twod_tab[0])); + svc_twod_tab[free_i].h_object = ch->h_object; + return &svc_twod_tab[free_i]; +} + +static u32 svc_twod_bpp(u32 fmt) +{ + switch (fmt) { + case 0xe8: case 0xf8: case 0xe9: case 0xfb: case 0xfc: case 0xee: + return 2; + case 0xf3: case 0xf4: case 0xf7: case 0x1c: + return 1; + default: + return 4; + } +} + +/* One RECTS primitive: fill the (xa,ya)-(xb,yb) axis-aligned rectangle. */ +static void svc_twod_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_twod_state *tw, u32 xa, u32 ya, u32 xb, + u32 yb, u64 push_va) +{ + u32 bpp = svc_twod_bpp(tw->dst_fmt); + u32 pitch, x0, x1, y0, y1, w, h; + u64 done = 0; + int rc; + + if (!tw->have_dst) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x push=0x%llx SKIP (no DST set)\n", + ch->h_object, push_va); + } + return; + } + /* + * Pitch: the PITCH layout carries it explicitly (the cursor dialog + * pushes 0x400 for a 256-wide 4-bpp surface). The BLOCKLINEAR + * screen dialog sets only w/h; the DDX's own CE readback of the + * same surface rides pitch = w*bpp (5120 = 1280*4), so the linear + * approximation uses w*bpp there - painting and readback agree. + */ + pitch = (tw->dst_layout == 1 && tw->dst_pitch) ? tw->dst_pitch : + tw->dst_w * bpp; + if (!bpp || !pitch || !tw->dst_w || !tw->dst_h) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x SKIP (bpp=%u pitch=%u w=%u h=%u)\n", + ch->h_object, bpp, pitch, tw->dst_w, + tw->dst_h); + } + return; + } + x0 = xa < xb ? xa : xb; + x1 = xa < xb ? xb : xa; + y0 = ya < yb ? ya : yb; + y1 = ya < yb ? yb : ya; + /* Surface-bound clamp (the DDX keeps CLIP at 0x7fff). */ + if (tw->dst_w >= 4 && tw->dst_w <= 0x8000) { + if (x0 >= tw->dst_w) + return; + if (x1 > tw->dst_w) + x1 = tw->dst_w; + } + if (tw->dst_h >= 4 && tw->dst_h <= 0x8000) { + if (y0 >= tw->dst_h) + return; + if (y1 > tw->dst_h) + y1 = tw->dst_h; + } + w = x1 - x0; + h = y1 - y0; + if (!w || !h) + return; + /* + * Replay dedupe: the DDX re-submits the same dialog push buffer + * for every repaint (the pump re-decodes ring entries), and the + * first pass of a reused slot is often a replay (allow=0). The + * fill is idempotent, so re-running is safe - but re-running on + * every replay pass would write megabytes per second for zero + * change. Execute once per (push VA, content hash); a changed + * push (new points/colors/surface) hashes differently and runs. + */ + if (tw->fills && push_va == tw->last_fill_va && + tw->last_fill_hash == svc_pass_hash) { + tw->fill_skips++; + return; + } + tw->last_fill_va = push_va; + tw->last_fill_hash = svc_pass_hash; + /* + * Row-wise fill through the shared operand machinery: the same + * resolution chain (shadow -> pin -> landing -> walk -> scan) the + * CE pitched fill uses, so the operand guards (LOWFILL / + * FILL-PT / FILL-HOLD) apply to 2D painting as well. + */ + rc = svc_fill(vgpu, tw->dst_off + (u64)y0 * pitch + (u64)x0 * bpp, + tw->prim_color, bpp, w, 0, CE_APERTURE_SYS, &done, + h, pitch); + tw->fills++; + if (!rc) + tw->fill_bytes += (u64)w * bpp * h; + else + tw->misses++; + if (tw->fill_log < 256) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x va=0x%llx lay=%u fmt=0x%x pitch=%u rect=%u,%u-%u,%u color=0x%x bpp=%u rows=%u done=%llu rc=%d push=0x%llx\n", + ch->h_object, tw->dst_off, tw->dst_layout, + tw->dst_fmt, pitch, x0, y0, x1, y1, tw->prim_color, + bpp, h, done, rc, push_va); + } +} + +static void svc_twod_method(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 m, u32 v, bool exec, + u64 push_va) +{ + struct svc_twod_state *tw = svc_twod_get(ch); + + switch (m) { + case 0x200: + tw->dst_fmt = v; + break; + case 0x204: + tw->dst_layout = v; + break; + case 0x208: + tw->dst_bsize = v; + break; + case 0x214: + tw->dst_pitch = v; + break; + case 0x218: + tw->dst_w = v; + break; + case 0x21c: + tw->dst_h = v; + break; + case 0x220: + tw->dst_off = (tw->dst_off & 0xffffffffULL) | + ((u64)(v & 0x1ffffff) << 32); + break; + case 0x224: + tw->dst_off = (tw->dst_off & ~0xffffffffULL) | v; + tw->have_dst = 1; + if (tw->dst_log < 64) { + tw->dst_log++; + pr_info("sema-twod-dst: h=0x%x va=0x%llx pitch=%u w=%u h=%u fmt=0x%x lay=%u bsz=0x%x\n", + ch->h_object, tw->dst_off, tw->dst_pitch, + tw->dst_w, tw->dst_h, tw->dst_fmt, + tw->dst_layout, tw->dst_bsize); + } + break; + case 0x2ac: + tw->operation = v; + break; + case 0x2a0: + tw->rop = v; + break; + case 0x580: + tw->prim_mode = v; + break; + case 0x584: + tw->prim_fmt = v; + break; + case 0x588: + tw->prim_color = v; + break; + case 0x5e0: { + u32 x = v & 0xffff, y = (v >> 16) & 0xffff; + + if (tw->prim_mode == 4 /* RECTS: point pairs */) { + if (!tw->have_pt) { + tw->pt_x = x; + tw->pt_y = y; + tw->have_pt = 1; + } else { + tw->have_pt = 0; + if (exec && gsp_sema_twod) + svc_twod_fill(vgpu, ch, tw, + tw->pt_x, tw->pt_y, + x, y, push_va); + } + } else if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x (point mode=%u)\n", + ch->h_object, m, v, tw->prim_mode); + } + break; + } + default: + if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x\n", + ch->h_object, m, v); + } + break; + } +} + +/* + * Dispatch a whole method run's data words. `inc` selects the NVA06F + * DMA semantics: INC_METHOD/ONE_INC (op 1/5) advance the method address + * per word (a contiguous register run); NON_INC (op 3) does NOT - every + * data word goes to the SAME method, which is how the DDX streams the + * RECTS point list into RENDER_SOLID_PRIM_POINT_X_Y (0x5e0; the 2-point + * rect and the 4-point double-band fills are one NON_INC run each, and + * its 17-word 0x100 run is a NO_OPERATION padding block). + */ +static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 addr, bool inc, u32 count, const u8 *data, + u32 avail, bool exec, u64 push_va) +{ + u32 k; + + for (k = 0; k < count && (u64)(k + 1) * 4 <= avail; k++) { + u32 v; + + memcpy(&v, data + 4 * k, 4); + svc_twod_method(vgpu, ch, inc ? addr + 4 * k : addr, v, exec, + push_va); + } +} + /* * SESSION 148 perf (r862/r864): the RC watchdog's class2d notify cycle * keep-alive that needs no push decode and no captured VA. Under the @@ -12300,6 +14375,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* SESSION 153-CONTINUED-10: draw-window tracker. First method of + * the current C597 run on the attached subchannel after the last + * foreign-subchannel (2D/present) run; see the BEGIN recognizer + * below. Per pass, like the rest of the decode state. */ + u32 gl_win_start = 0; /* m164: fresh CLASS-2 QMD witness for this decode pass. */ svc_gr_qmd_seen = 0; @@ -12430,6 +14510,55 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-12: attach on the draw marker itself. + * + * The section recogniser below needs the sticky C597 bind + * (svc_gl_att_h) that the boot-class GLX init push carries. On a + * boot where the class is bound channel-side (inside the emulated + * RM channel setup) and no decoded push re-emits SET_OBJECT C597, + * every draw-shaped push is dropped and no 3D replay ever runs. + * Adopt the first trusted push that carries an NVC597_BEGIN IMMD + * (op 4, addr 0x1618, subchannel 0) as the attached channel; the + * per-pass gl_win_start tracker then runs over that same push from + * its start, exactly as it does after an explicit bind, so the + * window keeps the pre-BEGIN state methods (vertex stream bases, + * colour target, constant-buffer selectors). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && trusted && + !svc_gl_att_has(ch->h_object, 0)) { + u32 o2 = 0; + + while (o2 + 4 <= len) { + u32 h2, op2, cnt2, sc2, ad2; + + memcpy(&h2, buf + o2, 4); + op2 = (h2 >> 29) & 0x7; + cnt2 = (h2 >> 16) & 0xfff; + sc2 = (h2 >> 13) & 0x7; + ad2 = (h2 & 0x1fff) << 2; + if (op2 == 4 && ad2 == 0x1618 && sc2 == 0) { + svc_gl_att_note(ch->h_object, 0); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: attach-on-BEGIN h=0x%x push=0x%llx off=0x%x (no explicit C597 bind seen)\n", + ch->h_object, push_va, o2); + } + break; + } + if (op2 == 4 || op2 == 7) { + o2 += 4; + continue; + } + if ((op2 == 1 || op2 == 3 || op2 == 5) && cnt2 && + cnt2 <= 0xfff && o2 + 4 + cnt2 * 4 <= len) { + o2 += 4 + cnt2 * 4; + continue; + } + o2 += 4; + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -12441,6 +14570,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on + * the attached C597 channel. The per-frame CONTENT pushes + * interleave the 2D/present ops (subchannel 3) and the 3D + * block (subchannel 0); the replayable 3D window starts at + * the first subchannel-0 method after the last subchannel-3 + * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are + * excluded: they name client semaphore VAs (0x1_2000c000) + * the replay channel cannot translate (the r929 v1 fault + * class), so they must stay outside the sliced window. + */ + if (gsp_sema_gl_xlate && !ch->engine_type) { + u32 as = svc_gl_att_subch_of(ch->h_object); + + if (as != 0xffu) { + if (subch != as) + gl_win_start = 0; + else if (!gl_win_start && + !(op == 1 && addr >= 0x1b00 && + addr <= 0x1b0c)) + gl_win_start = off; + } + } + + /* * fail4: name every big method-data run and every run whose * address is the LOAD_INLINE_DATA method (any op/subch - the * encoding of the 64 KB H2D upload is what we are pinning). @@ -12676,6 +14829,120 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); } + /* + * SESSION 153-CONTINUED-10 (task 1): the client's per-frame + * CONTENT pushes carry real C597 draws but no per-push + * SET_OBJECT (the class bind is sticky), so the section + * recognizer above never opened for them and the 3D replay + * lane never saw a draw. Recognise the draw marker itself: + * NVC597_BEGIN (0x1618, an IMMD method - its immediate is + * the topology in the count field) on the attached + * subchannel of the attached channel opens the section. + * SESSION 153-CONTINUED-26: the glxgears-era client draw + * pushes carry no 0x1618 marker - their first run is the + * C597 GL render-target setup (INC 0x840, {1, rtVA, w, h, + * fmt, blksz}) followed by the 0x1b0 draw list - so the RT + * setup opens the section too. Both shapes are gated on + * the sticky attachment, the channel class, and a closed + * section. + * The window starts at gl_win_start (the first C597 state + * method after the last foreign-subchannel run); the state + * methods before the marker are re-scanned so their operand + * VAs (vertex stream bases 0x1c00/0x1c10, colour target + * 0x0800, constant buffer selectors 0x2380) enter the + * candidate's page set - without them the backend's shadow + * relocation cannot rewrite the window and the engine + * faults on an unreachable guest VA (the r929 v2 class). + * The window then runs from gl_win_start to the pass end + * (the section close at the END method / push tail), which + * svc_gl_flush slices into the verbatim 3D replay. + */ + if (gsp_sema_gl_xlate && !ch->engine_type && + ((op == 4 && addr == 0x1618) || + (op == 1 && addr == 0x840 && count >= 4)) && + !svc_gl.seen && + svc_gl_att_has(ch->h_object, subch)) { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + u32 wstart = gl_win_start ? gl_win_start : off; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = wstart; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + /* + * SESSION 153-CONTINUED-26: when the RT-setup run is + * the opener, its address word IS the draw's render + * target - put it into the candidate page set, or the + * forwarded replay's shadow relocation cannot rewrite + * the RT VA and the physical launch faults/times out + * (r990: `GR forward failed rc=-110` on every client + * draw while the X section's 7-pointer capture ran). + * Must run AFTER the section reset above. + */ + if (op == 1 && addr == 0x840 && off + 12 <= len) { + /* + * SESSION 153-CONTINUED-27: the run is + * [upper, lower, w, h, fmt, blksz] - the VA + * is ((upper & 0xff) << 32) | lower exactly + * like every other pair recognizer in + * svc_gl_field. Reading only the low word + * added 0x20400000 for a surface whose true + * VA is 0x120400000 (the X screen), so the + * backend's shadow relocation MISSed it and + * the physical replay wrote the raw VA with + * no PTE chain (r992 FAULT_PDE @ +0xa000). + */ + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_gl_add_ptr(((u64)(v0 & 0xff) << 32) | + (u64)(v1 & 0xfffffffc)); + } + if (wstart < off) + svc_gl_scan_window(wstart, off, buf, len, + subch); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n", + ch->h_object, push_va, subch, wstart, + off, len, trusted); + } + } + + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + svc_gl_att_has(ch->h_object, subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD (0x902d) IMMD_DATA + * methods on subchannel 3 (SET_DST_FORMAT 0x200, layout, + * SET_OPERATION 0x2ac, NOTIFY 0x10c, ...). The value rides + * in the header's bits 28:16. State only; no execution. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 4 && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_method(vgpu, ch, addr, + (hdr >> 16) & 0xfff, + allow_ops || trusted, push_va); + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -12773,6 +15040,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl.start_off = off; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* SESSION 153-CONTINUED-10: the bind is sticky - + * remember it so the class-space recognizer + * (BEGIN 0x1618, below) accepts this channel's + * later per-push-CONTENT pushes that carry no + * SET_OBJECT of their own. + * SESSION 153-CONTINUED-13: per-channel table - + * a foreign channel's periodic C597 bind must + * not steal this channel's attachment. */ + svc_gl_att_note(ch->h_object, subch); /* * SESSION 153: witness every C597 section open * (bounded) - it is the recognizer-side proof @@ -12786,9 +15062,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->h_object, push_va, subch, trusted); } - } else if (svc_gl.seen) { - svc_gl_flush(vgpu, ch, &st, buf, len, - push_va, trusted, replay, off); + } else { + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + /* A different class bound over the attached + * subchannel replaces the C597 bind: drop the + * attachment so the class-space recognizer + * cannot misread methods of the new class. + * SESSION 153-CONTINUED-13: clears only this + * channel's entry. */ + svc_gl_att_clear(ch->h_object, subch); } } else if (gsp_sema_gl_xlate && svc_gl.seen && subch == svc_gl.subch) { @@ -12848,6 +15133,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD NON_INC/ONE_INC runs + * on subchannel 3. The dialogs' whole body is NON_INC (the + * 17-word context window and the point pairs); decode the + * data words as cl902d methods before the generic + * whole-run skip below consumes them. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && (op == 3 || op == 5) && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_run(vgpu, ch, addr, op == 5 ? true : false, + count, buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -12880,6 +15179,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memcpy(&v4, buf + off + 20, 4); /* + * SESSION 153-CONTINUED-26: FERMI_TWOD INC runs on + * subchannel 3 (the RENDER_SOLID_PRIM mode/format/color + * triplet at 0x580, the DST group, clip). The three + * notify shapes (0x100 cnt<=1, 0x104, 0x108, 0x10c) stay + * with the watchdog handler below; everything else in the + * cl902d space is decoded here and must not fall into the + * CE method chain. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 1 && + addr >= 0x0100 && addr <= 0x1400 && + !(addr == 0x0100 && count <= 1) && + addr != 0x0104 && addr != 0x0108 && addr != 0x010c) { + svc_twod_run(vgpu, ch, addr, true, count, + buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + off += 4 + count * 4; + continue; + } + + /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of @@ -12951,6 +15271,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (count >= 2) ch->notify2d_va |= (u64)v1; /* + * SESSION 153-CONTINUED-18 (task 1): when + * this push also carries the C597 kick + * section, remember the arm so the section + * flush can execute the NOTIFY write (the + * 2D ack below is the watchdog's zero-ack; + * the kick member needs the completed + * NvNotification authored). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl.kick_notify_va = + ch->notify2d_va; + /* * r834: this method stream reached us through * a REAL read (svc_read_last_pa); pin its page * so the watchdog's push translation cannot go @@ -12971,6 +15303,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case SVC_2D_NOTIFY: if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + /* + * SESSION 153-CONTINUED-18: mark the + * kick section's NOTIFY (payload 0) so + * svc_gl_flush executes the completion + * write at the armed VA - the executor + * authors the NvNotification the 2D + * zero-ack alone does not. + */ + if (gsp_sema_gl_xlate && svc_gl.seen) { + svc_gl.kick_notify_val = v0; + svc_gl.kick_notify_seen = true; + } if (svc_notify2d_write(vgpu, ch, true)) { /* @@ -13558,12 +15902,118 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.pitch_out = v0 & 0xffffff; st.have_pitch_out = 1; break; + /* + * SESSION 153 CONT.25: SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750), pushed as a count-2 (X,Y) + * or count-4 (X,Y,X,Y) INC run (nv_xaa_kepler-style + * SetSrcOrigin: x already scaled by compSizeFactor == Bpp + * for this class). No other class uses these method + * offsets (cl902d/cla06f clear). + */ + case CE_M_SRC_ORIGIN_X: + case CE_M_SRC_ORIGIN_Y: + case CE_M_DST_ORIGIN_X: + case CE_M_DST_ORIGIN_Y: + { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + st.src_origin_x = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + st.src_origin_y = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + st.dst_origin_x = vv[k]; + st.have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + st.dst_origin_y = vv[k]; + st.have_dst_origin = 1; + break; + } + } + } + break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); u32 n_ops_before = n_ops; bool dma_served = false; + /* + * SESSION 153 CONT.25: remember a full setup (this is + * the push that carried LINE_LENGTH + OFFSET_IN) as + * the channel's persistent CE state, then fill a + * transfer-shaped DELTA launch (the driver's per-strip + * follow-up: no LINE_LENGTH/OFFSET_IN of its own, + * OFFSET_OUT and the semaphore release present - the + * witnessed 52-byte pushes on h=0x1000013) from it so + * it builds its op instead of being silently dropped. + */ + if (allow_ops && st.line_length && st.have_in) + svc_ce_saved_store(&st, ch->h_object); + if (transfer != 0 && allow_ops && st.have_out && + st.have_sem && + (!st.line_length || !st.have_in)) { + struct svc_ce_saved *s = + svc_ce_saved_slot(ch->h_object); + + if (s && s->line_length && s->have_in) { + if (!st.line_length) { + st.line_length = + s->line_length; + st.line_count = + s->line_count; + } + if (!st.have_in) { + st.offset_in = + s->offset_in; + st.have_in = 1; + } + if (!st.have_pitch_in && + s->have_pitch_in) { + st.pitch_in = s->pitch_in; + st.have_pitch_in = 1; + } + if (!st.have_pitch_out && + s->have_pitch_out) { + st.pitch_out = s->pitch_out; + st.have_pitch_out = 1; + } + if (!st.have_src_mode && + s->have_src_mode) { + st.src_phys_mode = + s->src_phys_mode; + st.have_src_mode = 1; + } + if (!st.have_dst_mode && + s->have_dst_mode) { + st.dst_phys_mode = + s->dst_phys_mode; + st.have_dst_mode = 1; + } + if (gsp_sema_dbg && + svc_cedelta_log < 64) { + svc_cedelta_log++; + pr_info("sema-cedelta: h=0x%x push=0x%llx dma=0x%08x filled line=%u lc=%u in=0x%llx pi=%u po=%u sx=%u sy=%u (engine state inherited)\n", + ch->h_object, push_va, + dma, st.line_length, + st.line_count, + st.offset_in, + st.pitch_in, + st.pitch_out, + st.src_origin_x, + st.src_origin_y); + } + } + } + if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; @@ -13665,6 +16115,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->type = 0; m->dst = st.offset_out; + /* SESSION 153 CONT.25: blocklinear + * destination strip position. */ + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } /* * r367: the driver's own memset push * (channel_utils.c) proves the @@ -13813,6 +16276,38 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; /* + * SESSION 153 CONT.25: a BLOCKLINEAR + * operand positions each strip with + * SRC/DST_ORIGIN (x in bytes, y in + * rows) instead of folding it into + * OFFSET_IN/OUT. The linear shadow + * holds the pixels linearly, so the + * effective operand is + * base + origin_y*pitch + origin_x. + */ + if (st.have_src_origin) { + u32 sp = (st.have_pitch_in && + st.pitch_in) ? + st.pitch_in : + st.line_length; + + m->src += + (u64)st.src_origin_y * + sp + + st.src_origin_x; + } + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } + /* * r402: a 2D copy (cudaMemcpy2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes with @@ -13868,7 +16363,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (gsp_sema_dbg && (st.have_pitch_out || st.have_pitch_in) && - svc.op2d_raw_log < 12) { + svc.op2d_raw_log < 32) { u32 dl = len < 220 ? len : 220; svc.op2d_raw_log++; @@ -13883,8 +16378,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.pitch_in, m->dst, m->src, (u32)m->bytes, dma); - pr_info("sema-op2d-raw: raw %*phN\n", - dl, buf); + /* %*phN caps at 64 bytes; + * print the push whole in + * 64-byte halves. */ + pr_info("sema-op2d-raw: raw[0] %*phN\n", + dl > 64 ? 64 : dl, + buf); + if (dl > 64) + pr_info("sema-op2d-raw: raw[1] %*phN\n", + dl - 64, + buf + 64); } n_ops++; ops_seq++; @@ -14334,7 +16837,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * in one op"). Bounded. */ if (lm->type == 1) - pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%llu rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u so=%u/%u do=%u/%u\n", ch->h_object, lm->dst, lm->src, lm->bytes, lm->rows, lm->dpitch, @@ -14344,7 +16847,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_pitch_out, st.pitch_out, st.have_pitch_in, - st.pitch_in); + st.pitch_in, + st.have_src_origin, + st.src_origin_y, + st.have_dst_origin, + st.dst_origin_y); } } @@ -15803,6 +18310,52 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) } } +/* + * SESSION 153-CONT-34: watch the emulated BAR0 shadow's PCFG window + * (0x88000..0x89000) for doorbell writes the guest made directly to the + * mmap'd shadow (those never reach the trap path). Each changed word is fed + * to the doorbell path, whose only effect is waking the pump earlier - the + * pump itself stays per-pass. The first call for a guest snapshots the window + * without firing, so pre-existing register values (the emulation's own init + * writes) cannot ring. + */ +static u32 svc_bar0_pcfg_log; + +static void svc_bar0_pcfg_scan(struct nv_vgpu_device *vgpu) +{ + struct svc_guest_sess *sess; + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u32 i; + + if (!base || bsz < SVC_BAR0_PCFG_HI) + return; + sess = svc_guest_get(vgpu, true); + if (!sess) + return; + for (i = 0; i < SVC_BAR0_PCFG_WORDS; i++) { + u32 v; + + memcpy(&v, base + SVC_BAR0_PCFG_LO + i * 4, 4); + if (!sess->pcfg_have) { + sess->pcfg_snap[i] = v; + continue; + } + if (v == sess->pcfg_snap[i]) + continue; + sess->pcfg_snap[i] = v; + if (!v) + continue; /* a clear, not a doorbell */ + gsp_sema_service_doorbell(v); + if (svc_bar0_pcfg_log < 32) { + svc_bar0_pcfg_log++; + pr_info("vgsp-sema: PCFG doorbell (BAR0 shadow scan) off=0x%x val=0x%x\n", + SVC_BAR0_PCFG_LO + i * 4, v); + } + } + sess->pcfg_have = true; +} + static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; @@ -16443,6 +18996,79 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_read_last_src = (u8)src; svc_read_last_pa = pa; /* + * SESSION 153-CONTINUED-46: zero-content rescue for the + * direct GL client's push pages. r1032/r1033 evidence: + * the beef family's 5-7 KB draws routinely read back + * ALL ZERO from the trusted chain (latch/map/walk) - + * "sema-pushdec: h=0xbeef0100 va=0xcae2894 len=7148 + * rc=0 src=4 pa=0x13d181000 ... first8=00000000 + * 00000000" - and the capture is then skipped as a + * zero-head window, so no 3D replay ever launches + * (42 such skips on one boot). This is the r817 class + * of stale/wrong kernel-buffer translations: the + * driver's OWN PROMOTE_CTX statement holds the real + * backing. When the first chunk of a big GL push reads + * all zero from the trusted chain (src >= 2; the + * host-verified injected pins are ground truth and + * stay authoritative), probe the promote booking: a + * DIFFERENT page with non-zero content is used in + * place, the learned cache is healed with it, and the + * read is tagged src=9 so the pushdec witness shows + * the rescue. Bounded per boot. + */ + if (src >= 2 && first && len >= 512u && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + !memchr_inv(p, 0, chunk)) { + u64 ppa = 0; + int pap = SVC_APERTURE_SYS; + static u32 pushzero_log; + + if (!svc_promo_lookup(page_va, &ppa, &pap) && + ((ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + u64 rpa = ppa + + (va & (SEMA_PAGE_SIZE - 1)); + int rrc = svc_read_ap(vgpu, rpa, pap, + p, chunk); + + if (!rrc && memchr_inv(p, 0, chunk)) { + pa = rpa; + svc_read_last_src = 9; + svc_read_last_pa = rpa; + svc_lat_store(page_va, ppa, + pap); + if (pushzero_log < 32) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + ch->h_object, + (unsigned long long)va, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } else { + /* promo page unusable: put the + * zero state back exactly. */ + memset(p, 0, chunk); + if (!rrc && pushzero_log < 32) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + ch->h_object, + (unsigned long long)va, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } + } else if (pushzero_log < 32) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + ch->h_object, + (unsigned long long)va, src, + (unsigned long long)pa); + } + } + /* * r820: name whether the trusted answer came from a * driver-declared PROMOTE_CTX buffer; the decode uses * it for the zero-NOP acceptance (see the flag). @@ -16499,8 +19125,8 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu, * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ -static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, - u32 entries, u32 *put, u32 *get) +static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get, bool loose_get) { u32 p = 0, g = 0; @@ -16512,19 +19138,187 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, return -ENOENT; *put = p; *get = g; - if (p > entries || g > p) + /* + * SESSION 153-CONTINUED-38: the kernel channels' control page can + * carry an uninitialised GP_GET (measured 0x201400 at fb 0x18fe00+0x88 + * while GPPut was a live 3) - GP_GET is GPU-owned ("read only" in the + * Kepler control page) and the driver does not zero it, exactly the + * CE-layout lesson (ISSUE-0002 r47: "the GET word is GPU-owned and its + * initial content is not coherent"). The pump tracks its own position + * in last_get and rewrites GP_GET as it consumes, so `loose_get` + * callers (large GR kernel rings only) accept the block on the PUT + * alone. + */ + if (p > entries || (!loose_get && g > p)) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: callers decide */ return 0; } +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + return svc_userd_probe_ex(vgpu, pa, as, entries, put, get, false); +} + +/* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's + * channel allocation declares its USERD in the FB aperture at a 0x18fc00/ + * 0x18fe00-class offset, but the guest's GP_PUT/GP_GET writes for those + * channels land in the EMULATED BAR0 register shadow at that same offset + * (the vGPU guest driver programs its usermode/userd block through the + * register window; the FB shadow at the same offset holds nothing). Every + * existing source reads the FB/SYS shadows or scans guest RAM, so the block + * is invisible: svc_chan_userd returns -ENOENT, the pump never iterates the + * channel's GPFIFO ring and the channel retires with submissions=0. + * + * Watch policy (same spirit as the RAM-scan liveness latch): read the pair + * at bar0_shadow + userd_pa. A nonzero, coherent PUT is the guest's own + * write - the shadow is zeroed at device creation and no other writer + * targets that offset - so: + * - PUT >= 2 (two publishes already): adopt immediately; + * - PUT == 1 with a different value later (movement): adopt; + * - PUT == 1 stable past SVC_BAR0_USERD_PATIENT_MS: adopt anyway, so a + * client waiting on its single first submission cannot deadlock the + * watch (the pump then consumes that very submission); + * - unreadable/zero/incoherent: keep waiting (other sources unchanged). + * Adoption sets the BAR0 aperture so every later USERD read/write + * (svc_read_ap/svc_write_ap) and the pump's GP_GET write-back hit the same + * block the guest is updating. The walk seed mirrors the RAM-scan latch: + * the newest entries the client is waiting on, bounded to entries-1 slots. + */ +#define SVC_BAR0_USERD_PATIENT_MS 3000u + +static bool svc_bar0_userd_try(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 *put, u32 *get) +{ + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u64 off = ch->userd_pa; + u32 p = 0, g = 0, span; + + if (!base || !off) + return false; + if (off + SVC_USERD_GP_PUT + 4 > bsz) + return false; + /* + * Only the FB-declared family is retried here: SYS-declared offsets + * are real guest PAs the ordinary paths already read, and the CUDA/ + * kick channels resolve through their own sources before this point. + */ + if (ch->userd_as != SVC_APERTURE_FB) + return false; + + memcpy(&p, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&g, base + off + SVC_USERD_GP_GET, 4); + + if (gsp_sema_dbg && (p || g) && ch->bar0_userd_log < 8) { + ch->bar0_userd_log++; + pr_info("sema-chan: BAR0 userd probe h=0x%x rank=%u off=0x%llx put=%u get=%u\n", + ch->h_object, ch->engine_type ? ch->ce_index + : ch->gr_index, + off, p, g); + } + /* Idle, incoherent or an implausible register-looking value. */ + if (!p || g > p || p > 0x200000u) + return false; + + if (ch->bar0_userd_watch_pa != off) { + ch->bar0_userd_watch_pa = off; + ch->bar0_userd_watch_put = p; + ch->bar0_userd_watch_at = jiffies; + if (p >= 2) + goto adopt; + return false; /* first sighting, p == 1: patience */ + } + if (p != ch->bar0_userd_watch_put) + goto adopt; /* movement: live writer */ + if (time_after(jiffies, ch->bar0_userd_watch_at + + msecs_to_jiffies(SVC_BAR0_USERD_PATIENT_MS))) + goto adopt; /* stable nonzero past the patience window */ + return false; + +adopt: + ch->userd_ok = true; + ch->userd_as = SVC_APERTURE_BAR0; + if (ch->gpfifo_entries > 1 && p) { + span = p < ch->gpfifo_entries ? p : ch->gpfifo_entries - 1; + ch->last_get = (p % ch->gpfifo_entries + + ch->gpfifo_entries - span) % ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->bar0_userd_watch_pa = 0; + *put = p; + *get = g; + pr_info("sema-chan: BAR0 USERD h=0x%x eng=%u ADOPT off=0x%llx put=%u get=%u ap=bar0 (walk seed last_get=%u; guest GP_PUT lives in the emulated BAR0 shadow)\n", + ch->h_object, ch->engine_type, off, p, g, + (u32)ch->last_get); + return true; +} + /* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out, u32 *put_out, u32 *get_out); /* + * SESSION 153-CONTINUED-37: pristine-block USERD fallback for a GR channel + * whose ring does not translate yet. + * + * Measured (r1013 fresh boot): the X driver's pushbuffer-test channel + * (class 0xc46f, 8192 entries, VA 0x120090000) reports its USERD as the FB + * memdesc base 0x30000, but the guest's GPPut write lands in emulated vidmem + * at 0xa0000 - the params' base is relative to a client pool the emulation + * cannot see (same class of problem the r496b sweep solves once the ring + * resolves). The block is pristine in the test window: the only non-zero + * bytes in [0x0, 0x100) are the GP_PUT dword at +0x8c (GP_GET at +0x88 is + * still 0, because the driver zeroes it before the NOP test). When the + * channel's VAS page directory has not registered yet, the r496b sweep + * cannot run (it validates a hit by reading the ring entries the put + * claims), so this bounded shape scan keeps the discovery alive: pending + * only (get < put), pristine page, unclaimed by any other channel, then + * adopt it exactly like the ring-validated sweep does. The r496b sweep + * still runs later and may re-adopt a ring-validated block; both paths + * claim their latch, so a foreign channel cannot take it in between. + */ +static bool svc_fb_userd_pristine_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *off_out, + u32 *put_out, u32 *get_out) +{ + u8 *base = vgpu->bars[1].vaddr; + u64 lim, off; + + if (!base || !ch->gpfifo_entries) + return false; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x2000000ULL); + for (off = 0; off + 0x100 <= lim; off += 0x1000) { + u32 hp, hg; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + if (!hp || hp > ch->gpfifo_entries) + continue; + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + /* Only a posed kick (get < put) - an idle/stale block stays + * with its owner. */ + if (hg >= hp) + continue; + if (memchr_inv(base + off, 0, SVC_USERD_GP_GET) || + memchr_inv(base + off + SVC_USERD_GP_GET + 8, 0, + 0x100 - SVC_USERD_GP_GET - 8)) + continue; + if (svc_fb_claim_foreign(vgpu, ch->h_object, 0, off)) + continue; + *off_out = off; + *put_out = hp; + *get_out = hg; + return true; + } + return false; +} + +/* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one * candidate: for the combined GPFIFO+USERD allocations the control block sits @@ -16541,6 +19335,12 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * aperture stays the channel's own; a wrong second aperture only costs a * failed probe (the values must still read as a coherent put/get pair). */ +/* + * SESSION 153-CONTINUED-38: the shared entry-window gate used by every GR + * ring/entry validator (defined next to svc_gr_ring_candidate below). + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva); + static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { @@ -16695,6 +19495,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * at the layout offset. */ if (ch->engine_type != 0) { + /* + * SESSION 153-CONT-34: before giving up on a CE channel, try + * the registered offset read as a BAR0-shadow USERD - the GL + * client's copy/2D channels declare the same FB offsets as its + * GR channel and their GP_PUT writes land in the same shadow. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; svc_userd_sweep_sched(ch); return -ENOENT; } @@ -16726,10 +19534,42 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } for (i = 0; i < nc; i++) { + bool loose = !ch->engine_type && ch->gpfifo_entries > 1024; + if (!cand[i]) continue; - if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, - put, get)) { + if (!svc_userd_probe_ex(vgpu, cand[i], as[i], + ch->gpfifo_entries, put, get, loose)) { + /* + * SESSION 153-CONTINUED-44 (r1026): a GR kernel + * channel's candidate read of (put == 0, arbitrary + * get) is not a block the client publishes to. The + * cleared-screen fill pattern (alternating + * 0xff000000 / 0x00000000 pixels) yields exactly + * (put=0, get=0xff000000) at every 8-byte aligned + * offset, and the loose large-ring acceptance + * (g > p allowed) latched it: r1025's every + * beef0100 cycle adopted fb 0x20fc00 with put=0 + * forever (`submissions=0`, client recycles at + * 12 s) while the pre-walk-first boot's FB shape + * scan adopted the block carrying the client's + * real puts (`put=3/14/6`, submissions 29->208). + * Skip zero-put candidates for GR channels: the + * resolution re-runs every pass, and the FB/RAM + * shape scans (which require put >= 1) then adopt + * the block the client actually publishes on its + * first push. CE channels keep their layout rules + * (idle 0/0 acceptance is a documented CE case). + */ + if (!ch->engine_type && !*put) { + if (svc_zero_put_skip_log < 32) { + svc_zero_put_skip_log++; + pr_info("sema-chan: userd candidate 0x%llx/%u skipped for h=0x%x (put=0 get=%u; not a published block)\n", + cand[i], as[i], + ch->h_object, *get); + } + continue; + } /* ISSUE-0002 r46: name a fallback latch (CE wrong block). */ if (gsp_sema_dbg && ch->engine_type && ch->ce_index < 32 && @@ -16758,10 +19598,48 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget * gate stays open while candidates are under watch, so the * liveness confirmation cannot be starved by slice accounting. - */ - if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. + */ + /* + * SESSION 153-CONTINUED-38: kernel-VA rings (the direct GL client's + * 0x4000000-class channels) resolve through the channel's own VAS + * walk, never the guest-RAM content scan, so gr_ram_ring stays false + * and the RAM USERD discovery below would never run for exactly the + * channels that need it (CONT-37: beef0100-0102 ok=0, sub=0, then + * the alloc/free retry loop). Anchor the walk-resolved ring as a pin + * - only after entry0 decodes plausibly, so a stale walk answer is + * not cemented - and let the scan run on either anchor. The scan's + * entry validation still proves ownership against the ring. + */ + if (!ch->engine_type && ring_pa && !ch->gr_ram_ring && !ch->userd_ok) { + u8 re[SVC_GPFIFO_ENTRY_SIZE]; + u32 r0, r1; + u64 rva; + u32 rlen; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, re, sizeof(re))) { + memcpy(&r0, re, 4); + memcpy(&r1, re + 4, 4); + rva = ((u64)(r1 & 0xff) << 32) | + (u64)(r0 & 0xfffffffc); + rlen = ((r1 >> 10) & 0x1fffff) << 2; + if (!(rva & 0xfffff) && rlen >= 4 && rlen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, rva)) { + svc_pin_store(ch->gpfifo_va, ring_pa, ring_ap); + svc_lat_store(ch->gpfifo_va, ring_pa, ring_ap); + } + } + } + if (!ch->engine_type && (ch->gr_ram_ring || ring_pa) && !ch->userd_ok && (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || - ch->gr_ram_userd_watch_n)) { + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -16832,7 +19710,11 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * bounds the channel's life. */ if (!ch->engine_type && ring_pa && - ch->userd_scans < (ring_pa ? 64u : 8u)) { + ch->fb_userd_scan_n < 512 && + (!ch->fb_userd_scan_at || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { u64 off, lim; u8 *base = vgpu->bars[1].vaddr; u64 best = 0, best_dist = ~0ULL; @@ -16840,6 +19722,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool best_pending = false; u32 gate_n = 0, vfail_n = 0; + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; ch->userd_scans++; lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); if (base && lim) { @@ -16855,7 +19739,11 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); - if (!hp || hp > ch->gpfifo_entries || hg > hp) + /* CONT-38: large GR kernel rings may carry an + * uninitialised GPU-owned GP_GET; the PUT alone + * gates there (see svc_userd_probe_ex). */ + if (!hp || hp > ch->gpfifo_entries || + (hg > hp && ch->gpfifo_entries <= 1024)) continue; gate_n++; /* Validate: every claimed entry decodes to a @@ -16887,13 +19775,12 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * Every entry the put claims must name a * VA in the ring's allocation window - * pushbuffers below the ring, the - * progress tracker above it. + * progress tracker above it; the + * kernel-VA family is accepted in its + * own low window (CONT-38). */ - if (!eva || - eva + SVC_GR_RAM_WIN / 2 < - ch->gpfifo_va || - eva >= ch->gpfifo_va + - SVC_GR_RAM_WIN / 2 || + if (!eva || !svc_gr_entry_win_ok( + ch->gpfifo_va, eva) || elen < 4 || elen > 0x100000) { ok = false; rck = (int)k; @@ -16971,6 +19858,48 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } } + /* + * SESSION 153-CONTINUED-37: the channel's ring VA does not translate + * yet (the client's VAS page directory registers ~1 s after channel + * creation; until then svc_lookup cannot resolve it and the + * ring-validated r496b sweep above never runs). The driver's DMA + * test only needs the block it is polling: a freshly kicked pristine + * FB block (see svc_fb_userd_pristine_scan). Adopt it so the pump + * starts iterating entries as soon as the ring walk lands, inside + * the driver's 10 s window. + */ + if (!ch->engine_type && !ring_pa && + ch->fb_userd_scan_n < 512 && + (!ch->fb_userd_scan_at || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { + u64 off = 0; + u32 p = 0, g = 0; + + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; + if (svc_fb_userd_pristine_scan(vgpu, ch, &off, &p, &g)) { + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_fb_claim_add(ch->h_object, 0, off); + *put = p; + *get = g; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by pristine-FB scan at fb 0x%llx (put=%u get=%u; ring 0x%llx not translated yet)\n", + ch->h_object, ch->gr_index, off, p, g, + ch->gpfifo_va); + return 0; + } + } + /* + * SESSION 153-CONT-34: last resort for the GL client family - retry the + * registered offset as a BAR0-shadow USERD. Every other source (layout, + * ring-derived, RAM scan, FB sweep) has already missed the block whose + * GP_PUT writes land in the emulated register window. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; return -ENOENT; } @@ -17284,6 +20213,29 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, latch_off, latch_upa, latch_put, latch_get); } +/* + * SESSION 153-CONTINUED-41 (mission: advance the client's completed side + * host-side): the direct GL client's kernel channels (h=0xbeef01xx) arm + * their OWN completion sentinel. Glcore's submit pump writes 0xffff into + * a u16 at notifier-page+0x0e+i*16 (i=0..5) and spins until the word + * changes (glcore+0xB1A313 scan loop; live: armed at +0x2e during the + * yield wait, SESSION 40/41). On real hardware the RM completion + * overwrites that word; the emulation never did, so every ladder attempt + * timed out (~11 s), the client fell back to no-op frames and GP_PUT was + * never stored. Proof by host-side emission: blindly writing 0 over the + * armed word took a live client from ~23.4 s/frame to ~11.7 s/frame. + * The channel notifier proper starts at `base` (page+0x60 for this + * family), so the client's six 16-byte records are just below it; clear + * any that read 0xffff. Only the client family is touched - the X DDX's + * bfef-family member page uses a different all-zero wait discipline and + * a 0xffff there means a delivered error, so it must not be wiped. + */ +static bool gsp_sema_client_slot = true; +module_param_named(sema_client_slot, gsp_sema_client_slot, bool, 0644); +MODULE_PARM_DESC(sema_client_slot, + "Clear the GL client's armed 0xffff completion slots (beef-family notifier pages)"); +static u32 svc_client_slot_n; + static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -17295,6 +20247,32 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, if (!ch->notifier_pa) return; + if (gsp_sema_client_slot && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u64 page = base & ~((u64)SEMA_PAGE_SIZE - 1); + int i; + + for (i = 0; i < 6; i++) { + u64 pa = page + 0x0e + (u64)i * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > base) + break; + if (svc_read_ap(vgpu, pa, ch->notifier_as, + &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (!svc_write_ap(vgpu, pa, ch->notifier_as, + &z, sizeof(z))) { + svc_client_slot_n++; + if (gsp_sema_dbg && svc_client_slot_n <= 64) + pr_info("sema-chan: client-slot cleared h=0x%x slot=%d pa=0x%llx\n", + ch->h_object, i, pa); + } + } + } + /* * P6c C3: while a delivered fault notifier is latched the pump must * not clear the error slot. The latch clears when the guest ack reads @@ -17395,6 +20373,89 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-42 (mission: converge the client-slot clear so a + * ladder attempt COMPLETES instead of recycling). + * + * CONT-41 found the protocol: glcore's submit pump arms a 0xffff sentinel in + * one of six u16 records just below the beef-family channel-notifier base + * (page+0x0e+i*16) and treats ANY other value as the completion. The kernel + * clear landed inside svc_notify_complete, but that hook only runs on passes + * where a completion is being processed - a client whose sentinel is armed + * while the pump has nothing else to complete still sat out its ~11 s + * deadline (r1023: attempt 2/3 had the clears logged and still retired + * submissions=0; only attempt 1 - cleared while its pushes were live - + * submitted). This sweep runs once per service pass for every live + * beef-family channel and clears ANY armed 0xffff in the six records, so + * every ladder phase releases within a pass regardless of what else the + * pump is doing. The bfef-family (X DDX member page) keeps its 0xffff + * delivered-error semantics and is not touched. + * + * Bounded: 16 channels per pass, first 96 clear events logged with the slot + * index (arm-phase tracing), then every 4096th. The log's slot index is + * what identifies which phase the client is in when it arms each record. + */ +static u32 svc_client_slot_log_n; +static unsigned long svc_client_slot_at; + +static void svc_client_slot_sweep(struct nv_vgpu_device *vgpu) +{ + struct { + u32 h; + u64 base; + u32 as; + } tab[16]; + unsigned long flags; + int i, n = 0; + + if (!gsp_sema_client_slot) + return; + /* ~1 pass per tick while the pump runs tight; plenty for an ~11 s wait. */ + if (time_before(jiffies, svc_client_slot_at)) + return; + svc_client_slot_at = jiffies + msecs_to_jiffies(2); + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS && n < ARRAY_SIZE(tab); i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || !c->notifier_pa) + continue; + if ((c->h_object & 0xfffff000u) != 0xbeef0000u) + continue; + tab[n].h = c->h_object; + tab[n].base = c->notifier_pa; + tab[n].as = c->notifier_as; + n++; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + for (i = 0; i < n; i++) { + u64 page = tab[i].base & ~((u64)SEMA_PAGE_SIZE - 1); + int k; + + for (k = 0; k < 6; k++) { + u64 pa = page + 0x0e + (u64)k * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > tab[i].base) + break; + if (svc_read_ap(vgpu, pa, tab[i].as, &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (svc_write_ap(vgpu, pa, tab[i].as, &z, sizeof(z))) + continue; + svc_client_slot_n++; + svc_client_slot_log_n++; + if (gsp_sema_dbg && (svc_client_slot_log_n <= 96 || + (svc_client_slot_log_n % 4096) == 0)) + pr_info("sema-chan: client-slot cleared(sweep) h=0x%x slot=%d pa=0x%llx n=%u\n", + tab[i].h, k, pa, svc_client_slot_log_n); + } + } +} + +/* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in @@ -17504,6 +20565,30 @@ static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) } /* + * SESSION 153-CONTINUED-38: kernel-VA ring entry window. The direct GL + * client's kernel channels (h 0xbeef01xx) place their GPFIFO ring and + * pushbuffers in the driver's LOW aperture (0x4000000-class, + * SVC_LOW_REL_VA_LO) instead of the userspace GPU-VA window (>= 4 GiB, + * SEMA_VA_LO) every GR gate below assumed, so their entries could never + * validate and the channels never resolved a USERD (the CONT-37 alloc/free + * retry loop). An entry VA in the low family is this channel's push only + * when it sits in the same contiguous allocation as the ring VA; accept it + * within a 256 MB window around the ring, never outside the low family. + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva) +{ + if (eva + SVC_GR_RAM_WIN / 2 >= ring_va && + eva < ring_va + SVC_GR_RAM_WIN / 2) + return true; + if (eva >= SVC_LOW_REL_VA_LO && + eva < SVC_LOW_REL_VA_LO + 0x10000000ULL && + eva + 0x10000000ULL > ring_va && + eva < ring_va + 0x10000000ULL) + return true; + return false; +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -17526,8 +20611,11 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; - if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || - *len < 4 || *len > 0x10000) + if ((*va & 0xfffff) || *len < 4 || *len > 0x10000) + return false; + /* SESSION 153-CONTINUED-38: low-aperture kernel-VA ring entries. */ + if ((*va < SEMA_VA_LO || *va >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, *va)) return false; /* * SESSION 132: a candidate whose entry0 push provably reads all-zero @@ -18590,6 +21678,22 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, continue; if (len1 < 4 || len1 > 0x100000) continue; + /* + * r979: kick-family ring discrimination. Shape alone + * admitted a foreign table page ahead of the real ring + * on r978 (pa=0x104fa9000, entries naming a bogus + * 0x12048b824/0x40000 push) while the true ring's + * entry pair names the client's per-frame record VA + * (SVC_GL_KICK_PUSH_VA, the r908-documented GL + * pushbuffer, constant across r908-r978). A + * kick-family candidate must name the record; the + * lookalike class then never latches. Non-kick + * channels keep the shape gate unchanged. + */ + if (svc_kick_family(ch) && + va0 != SVC_GL_KICK_PUSH_VA && + va1 != SVC_GL_KICK_PUSH_VA) + continue; *pa_out = gpa + o; return true; } @@ -18627,6 +21731,99 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance). + * + * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring + * found by the guest-RAM content scan) posts a per-frame completion kick + * the client's flush loop paces on. Its USERD is a sysmem control block + * the driver allocates far from the ring - r971 measured the ring at pa + * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the + * same nomination was +0.6..+2.3 GB across r966-r971) - while the general + * RAM USERD scan's allocation-locality gate only admits candidates within + * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939 + * nine-decoy class). That gate rejected the channel's real USERD on every + * boot since the gate landed, the scan retired ("RAM scan found no + * candidate in 20 slice(s)"), the pump never iterated the kick entries + * and the kick-notify lane never ran. For this family the ring-entry + * match (every PUT-claimed ring entry must decode to a pushbuffer VA + * inside the ring's own allocation window) is the ownership proof; the + * locality distance is not - so the gate is waived for these channels and + * the distance is logged instead. The pump's speculative-latch + * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final + * judge, and the rejected-latch memory prevents a frozen block from ever + * pinning the scan. + */ +static bool svc_kick_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu; +} + +/* + * Re-validate the kick channel's ring latch each pass (bounded to once a + * second per channel). The RAM content scan latches a ring page once; if + * the driver later zeroes or reuses that page (the r966-r971 class the + * design brief describes: the USERD scan then validates every candidate + * against a dead ring and adopts none), the latch is dropped here so the + * content scan re-runs and re-latches the channel's current ring page. + * The stale entry pair is logged on re-arm so the transition is visible + * in the record. A live ring (a first entry that still decodes to a + * window-bounded pushbuffer VA) is a cheap no-op. + */ +static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + unsigned long flags; + int i; + + if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok) + return; + if (!time_after(jiffies, ch->gr_kick_ring_reval_at)) + return; + ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000); + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + u64 eva; + u32 elen; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x100000) + return; /* ring page live; nothing to do */ + } + /* + * The latched page no longer decodes entries: drop the one-shot + * latch state so the RAM ring content scan re-runs. The pin is left + * in place - a stale pin only shadows VA reads, while the content + * scan validates against raw RAM and re-pins on a hit. + */ + clear_bit(ch->gr_index, &svc_gr_ring_done); + clear_bit(ch->gr_index, &svc_pb_gr_done); + ch->gr_ram_cursor = 0; + ch->gr_ram_passes = 0; + ch->gr_ram_logged = false; + ch->gr_ram_ring = false; + pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n", + ch->h_object, ch->gpfifo_va, e0, e1); + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + svc.chan[i].gr_ram_ring = ch->gr_ram_ring; + svc.chan[i].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). * Such a client's USERD is a system-memory object like its ring, so the @@ -18666,6 +21863,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, u64 live_pa = 0; u32 live_put = 0, live_get = 0, live_prev = 0; int ring_ap = SVC_APERTURE_SYS; + bool kick; u32 wi; int i; unsigned long flags; @@ -18684,6 +21882,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, if (!buf) return false; } + kick = svc_kick_family(ch); + /* + * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch + * before validating candidates against it (svc_kick_ring_revalidate + * re-latches a ring page the driver has zeroed since the latch; see + * its comment). + */ + svc_kick_ring_revalidate(vgpu, ch); (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); /* * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. @@ -18710,9 +21916,13 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * mapped by NO process, and never advanced on any pass * (two frozen snapshots that cost a 15 s reject/re-latch * cycle all boot). A control block this far from its own - * ring is not this channel's USERD. + * ring is not this channel's USERD. SESSION 153-CONT-19: + * waived for the kick family - its real USERD is measured + * 0.6-2.3 GB from the ring, and the ring-entry match (not + * proximity) is the ownership proof there (see + * svc_kick_family). */ - if (ring_pa && + if (ring_pa && !kick && (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) || wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) { ch->gr_ram_userd_watch_n--; @@ -18737,10 +21947,47 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } last = ch->gr_ram_userd_watch_put[wi]; if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } wi++; continue; } - if (hp != last + 1) { + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { /* * SESSION 154: a jump or a reset is the churn * signature (the r916-r919 lookalike pools move by @@ -18755,9 +22002,37 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; ch->gr_ram_userd_watch_hits[wi] = ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; continue; } ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } /* * SESSION 154: ONE observed +1 step adopts. The live block * advances ~2-8 s per step (measured) while the client's @@ -18814,9 +22089,26 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * latches read 12/8 and 16/16 and every previously * measured lookalike counted 16 or more. Also accept * a free-running put (the old put <= entries bound - * would reject the live block). + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. + */ + /* + * SESSION 153-CONTINUED-38: the cap is also lifted for + * LARGE rings (the kernel channels' 32768-entry + * GPFIFOs): their put is a real free-running count and + * can sit far above 8 while the client waits (measured + * put=16 on beef0100). The remaining gates (near-empty + * page, entry validation against the ring, and the + * pump's confirm/reject) carry the ownership proof. */ - if (!hp || hp > 8 || hg > hp) + if (!hp || + hp > (kick || entries > 1024 ? 0x100000 : 8) || + (hg > hp && entries <= 1024)) continue; { u32 z = 0, zi; @@ -18842,8 +22134,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * above. Excludes the r936-r939 nine-decoy pair * (~460 MB from the ring) before it can occupy a * watch slot and burn a reject/re-latch cycle. + * SESSION 153-CONT-19: waived for the kick family + * (h 0xbfef00xx) - its real USERD is measured far + * outside this window, so admission there is by the + * ring-entry match alone (the entry loop below) and + * the pump's +1/+2-step confirmation; the distance + * is logged at the latch. */ - if (ring_pa && + if (ring_pa && !kick && (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) || gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5))) continue; @@ -18870,10 +22168,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, (u64)(a0 & 0xfffffffc); elen = ((a1 >> 10) & 0x1fffff) << 2; if (!eva || - eva + SVC_GR_RAM_WIN / 2 < - ch->gpfifo_va || - eva >= ch->gpfifo_va + - SVC_GR_RAM_WIN / 2 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva) || elen < 4 || elen > 0x100000) { ok = false; break; @@ -18915,6 +22210,64 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, hp == ch->gr_ram_userd_reject_put) continue; /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* * SESSION 154 (M-GL1.2): a candidate that passed the * shape gate, the entry validation, the page-density * gate and the put cap is latched SPECULATIVELY. The @@ -18942,6 +22295,23 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && !ch->gr_ram_userd_watch_n) { ch->gr_ram_userd_logged = true; @@ -18974,6 +22344,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_rejects; svc.chan[i].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[i].gr_ram_userd_watch[wj] = ch->gr_ram_userd_watch[wj]; @@ -18981,6 +22355,8 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, ch->gr_ram_userd_watch_put[wj]; svc.chan[i].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } break; } @@ -19020,6 +22396,27 @@ static const u8 svc_gr_anchor_sig[16] = { }; #define SVC_GR_ANCHOR_SWEEPS 3 +/* + * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record is a fixed method stream opening with the SET_SUBDEVICE_MASK + * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the + * bytes r975's anchor pin answered and r973 decoded to n=512 execs). + * Every boot r966-r976 read that VA through some untrusted translation + * and, because a plausible-looking garbage page (r976: first8=9864be60 + * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the + * generic method-stream parse, the entry was consumed with the wrong + * bytes and the kick lane stayed silent for the whole session. For + * this family the canonical prefix IS the content identity: a read + * that lacks it came from a stale/foreign page no matter how + * plausible it parses, and must take the rescue chain (VAS walk -> + * content anchor -> pin). + */ +static bool svc_kick_canonical(const u8 *buf, size_t len) +{ + return len >= sizeof(svc_gr_anchor_pre) && + !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre)); +} + static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, u32 len) { @@ -22862,7 +26259,7 @@ static void svc_fwd_hold_direct_note(void) * A bounded fallback run (sweep, when no forward is owed) keeps the list * from stalling when the launch was refused. */ -#define SVC_GR_COPY_PEND 8 +#define SVC_GR_COPY_PEND 64 struct svc_gr_copy_pend { bool valid; u32 h_object; @@ -22882,6 +26279,55 @@ static u32 svc_gr_copy_pend_next; static u64 svc_gr_copy_run_total; static u64 svc_gr_copy_run_log; +/* + * SESSION 153 CONT.25: recently-executed copy identities. The sweep + * re-decodes the ring continuation every pass, so an entry it just ran + * would be re-added (the valid-dedup above only covers live entries) and + * re-run forever - with a bounded per-pass run count that pins the runner + * on the same first slots. The content hash changes whenever the driver + * rewrites the push (new frame: the semaphore payload advances), so a + * (push_va, hash, src, dst, bytes, rows) match means "this exact strip + * already ran this frame". + */ +#define SVC_GR_COPY_DONE_N 512 +struct svc_gr_copy_done_rec { + u64 push_va, hash, src, dst; + u32 bytes, rows; +}; +static struct svc_gr_copy_done_rec svc_gr_copy_done[SVC_GR_COPY_DONE_N]; +static u32 svc_gr_copy_done_next; + +static bool svc_gr_copy_done_lookup(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_DONE_N; i++) + if (svc_gr_copy_done[i].push_va == push_va && + svc_gr_copy_done[i].hash == hash && + svc_gr_copy_done[i].src == src && + svc_gr_copy_done[i].dst == dst && + svc_gr_copy_done[i].bytes == bytes && + svc_gr_copy_done[i].rows == rows) + return true; + return false; +} + +static void svc_gr_copy_done_store(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + struct svc_gr_copy_done_rec *d = + &svc_gr_copy_done[svc_gr_copy_done_next++ % + SVC_GR_COPY_DONE_N]; + + d->push_va = push_va; + d->hash = hash; + d->src = src; + d->dst = dst; + d->bytes = bytes; + d->rows = rows; +} + static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, u64 src, u64 dst, u32 bytes, u32 rows, u32 dpitch, u32 spitch, u32 dma, u32 remap) @@ -22890,6 +26336,8 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, if (!bytes || !rows || !src || !dst) return; + if (svc_gr_copy_done_lookup(push_va, hash, src, dst, bytes, rows)) + return; /* this exact strip already ran for this content */ for (i = 0; i < SVC_GR_COPY_PEND; i++) { struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; @@ -22939,17 +26387,33 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, * when no forward is owed. The executor is the same svc_copy() the CE/GCE * op path uses (handles vidmem shadow sources and sysmem destinations). */ +/* + * SESSION 153 CONT.25: bounded per-call run count for the pending + * copy-out list. The X DDX readback keeps at most two strips in flight + * (the two scratch halves), so running the whole queue in one sweep pass + * overruns the server's consumption and every frame reads the newest + * strips only. Pace at a few per pass; 0 = unlimited (historical). + */ +static uint gsp_sema_gr_copy_run_max; +module_param_named(sema_gr_copy_run_max, gsp_sema_gr_copy_run_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_copy_run_max, + "Max pending GRCE copy-outs executed per sweep pass (0 = unlimited)"); + static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) { - u32 i; + u32 i, ran = 0; for (i = 0; i < SVC_GR_COPY_PEND; i++) { + if (gsp_sema_gr_copy_run_max && + ran >= gsp_sema_gr_copy_run_max) + break; struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; u64 done = 0; int rc; if (!p->valid) continue; + ran++; rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, p->rows, p->dpitch, p->spitch, p->h_object, 0, @@ -22957,6 +26421,8 @@ static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) svc_gr_copy_run_total++; if (rc == 0 || rc == SVC_COPY_FORWARDED) { p->valid = false; + svc_gr_copy_done_store(p->push_va, p->hash, p->src, + p->dst, p->bytes, p->rows); if (svc_gr_copy_run_log < 64) { svc_gr_copy_run_log++; pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", @@ -28540,6 +32006,9 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch u32 have_in = 0, have_out = 0; u32 pitch_in = 0, pitch_out = 0; u32 line_len = 0, line_count = 0; + u32 src_origin_x = 0, src_origin_y = 0; + u32 dst_origin_x = 0, dst_origin_y = 0; + u32 have_src_origin = 0, have_dst_origin = 0; u32 last_dma = 0, last_remap = 0; u32 cp_subch = ~0u; bool used_tail = false; @@ -28853,6 +32322,38 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch ((vv[k] & 0xffffu) << 16); } + } else if (addr >= CE_M_SRC_ORIGIN_X && + addr <= CE_M_DST_ORIGIN_Y && n) { + /* + * SESSION 153 CONT.25: the X DDX readback + * pair positions each 6-row strip with + * SRC_ORIGIN (x bytes / y rows) instead of + * folding it into OFFSET_IN - the same + * methods the main walk now decodes. The + * strip y steps 6 per push (A: 120, B: 126, + * A: 132, ...) and the copy must read + * OFFSET_IN + origin_y*PITCH_IN. + */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + src_origin_x = vv[k]; + have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + src_origin_y = vv[k]; + have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + dst_origin_x = vv[k]; + have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + dst_origin_y = vv[k]; + have_dst_origin = 1; + break; + } + } } else if (addr == 0x300 && count) { u32 dma = w0; @@ -28860,28 +32361,104 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a * release-only / semaphore op - no data * movement, nothing to record). + * + * SESSION 153 CONT.25: remember a full setup + * (LINE_LENGTH + OFFSET_IN present) as the + * channel's persistent CE state and fill a + * DELTA launch (the X DDX readback pair's + * second push: no setup of its own, only + * SRC_ORIGIN + SEM_A/PAYLOAD + OFFSET_OUT + + * LAUNCH_DMA) from it, exactly like the main + * walk - without this every odd strip of the + * frame is dropped. */ last_dma = dma; + if (have_in && off_in && line_len) { + struct svc_push_state tst; + + memset(&tst, 0, sizeof(tst)); + tst.offset_in = off_in; + tst.pitch_in = pitch_in; + tst.pitch_out = pitch_out; + tst.line_length = line_len; + tst.line_count = line_count; + svc_ce_saved_store(&tst, + ch->h_object); + } + if ((dma & 0x3) && have_out && off_out && + (!have_in || !line_len)) { + struct svc_ce_saved *sv = + svc_ce_saved_slot( + ch->h_object); + + if (sv && sv->line_length && + sv->have_in) { + if (!line_len) { + line_len = + sv->line_length; + line_count = + sv->line_count; + } + if (!have_in) { + off_in = + sv->offset_in; + have_in = 1; + } + if (!pitch_in) + pitch_in = + sv->pitch_in; + if (!pitch_out) + pitch_out = + sv->pitch_out; + } + } if ((dma & 0x3) && have_in && have_out && line_len && line_count && off_in && off_out) { + /* + * SESSION 153 CONT.25: fold the + * per-strip SRC/DST_ORIGIN into the + * operands (bytes on X, rows on Y; + * the linear shadow makes + * base + y*pitch + x the truth). + * line_len is BYTES for C5B5 + * (LINE_LENGTH_IN semantics), not + * dwords: the old *4 inflated every + * strip 4x. + */ + u64 src_f = off_in, dst_f = off_out; + + if (have_src_origin) + src_f += + (u64)src_origin_y * + (pitch_in ? + pitch_in : + line_len) + + src_origin_x; + if (have_dst_origin) + dst_f += + (u64)dst_origin_y * + (pitch_out ? + pitch_out : + line_len) + + dst_origin_x; if (inline_only) svc_ahead_copy_in(vgpu, ch->h_object, push_va, svc_push_hash(buf, len), - off_in, off_out, - line_len * 4, + src_f, dst_f, + line_len, line_count, dma); else if (forward) { svc_gr_copy_pend_add(ch->h_object, push_va, svc_push_hash(buf, len), - off_in, off_out, - line_len * 4, + src_f, dst_f, + line_len, line_count, pitch_out ? pitch_out : - line_len * 4, + line_len, pitch_in, dma, last_remap); cp_subch = subch; @@ -29586,6 +33163,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type) { u32 pput = 0; + /* + * SESSION 153-CONTINUED-38: set the channel's VAS + * BEFORE the autoreg derive, not after the ring + * lookup. The counter slot must be disambiguated + * through THIS channel's page tables (two clients + * register the same counter VA; CONT-37 collision), + * and the ring lookup + USERD discovery below must + * also resolve through the channel's own VAS first. + */ + svc_cur_vas = ch->h_vaspace; svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); /* @@ -29776,8 +33363,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * probably locked" while every other channel kept running. * Writing the ack on every pass that sees work pending is * idempotent. + * + * SESSION 153-CONTINUED-19: NOT for the kick family. Their + * notify2d_va is the kick record's own NOTIFY target (the + * shared completion word the GL client polls), and the kick + * lane authors that word (a completed NvNotification with a + * nonzero timestamp, svc_kick_notify_exec). The per-pass ack + * zeroed it 14029 times on r972 - the last kick write + * survived at most one pass (~60 ms) before being wiped, so + * the client's poll kept seeing "not fired". The GPU never + * clears this notification; the client does (re-arm). */ - if (!ch->engine_type && ch->notify2d_va && put != get) + if (!ch->engine_type && ch->notify2d_va && put != get && + !svc_kick_family(ch)) svc_notify2d_write(vgpu, ch, false); /* * SESSION 148 (r864): registration-based keep-alive that @@ -29870,6 +33468,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + /* * SESSION 154 (M-GL1.2): speculative-latch confirmation for * the RAM USERD. The scan latches a gated candidate right @@ -29880,13 +33482,42 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * candidate. The live block advances 1->2->3 in 2-10 s * (measured); the client's own release waits are ~11 s, so * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). + */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; + /* + * SESSION 153-CONTINUED-38: the latch confirmation must also + * judge a block adopted through the RAM scan for a + * walk-resolved ring (gr_ram_ring false; gr_ram_userd_latch_at + * stamps the latch), or a wrong low-aperture latch would stick + * forever. */ - if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && - !ch->gr_ram_userd_confirmed) { + if (!ch->engine_type && ch->userd_ok && + (ch->gr_ram_ring || ch->gr_ram_userd_latch_at) && + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + if (!ch->gr_ram_userd_latch_at) ch->gr_ram_userd_latch_at = jiffies; - if (put == ch->gr_ram_userd_latch_put + 1 || - put == ch->gr_ram_userd_latch_put + 2) { + stepped = kfam ? + put > ch->gr_ram_userd_latch_put : + (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2); + if (stepped) { ch->gr_ram_userd_confirmed = true; pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", ch->h_object, ch->userd_pa, @@ -30163,8 +33794,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * backend is off), run any pending GRCE copies so * they cannot sit forever (the completion edge is * the primary path). + * + * SESSION 153 CONT.25: also run when a forward IS + * owed - the X DDX readback pair's copies have no + * forwarded launch behind them, and a stuck + * unoffered candidate (valid && !offered) blocked + * this fallback forever: 1443 copy pends, 0 runs, + * the server's per-copy completion wait timing out + * on every strip. The completion edge still runs + * them first when it fires; the runner retires + * each entry once, so running early is bounded. */ - if (gsp_sema_gr_xlate && !svc_gr_forward_pending()) + if (gsp_sema_gr_xlate) svc_gr_copy_pend_run(vgpu); } @@ -30480,6 +34121,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } bool injected = false; bool allow_ops = (ch->engine_type != 0); + bool kick_badpg = false; /* * A resolved buffer that does not start @@ -30579,7 +34221,48 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) len >= 4 && len <= 16 && !memchr_inv(pb, 0, len); - if (!svc_method_stream_plausible(pb, len) && + /* + * r977/r979: a read of the + * kick family's per-frame + * record (SVC_GL_KICK_PUSH_VA; + * r978 showed the channel also + * carries bulk pushes that + * must NOT be held to this + * content identity - a 256 KB + * push at 0x12048b824) whose + * bytes lack the canonical + * submit-record prefix is a + * wrong page no matter how + * plausible it parses (r976: + * src=3 window garbage + * 9864be60 ffffffff consumed + * as a valid push and the kick + * lane ran zero execs). Force + * the implausible path so the + * VAS walk / content anchor / + * pin-drop chain below + * re-anchors the page. + */ + kick_badpg = !zero_nop && + svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128 && + !svc_kick_canonical(pb, len); + if (kick_badpg) { + svc.kick_badpg++; + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + } + + if ((!svc_method_stream_plausible(pb, len) || + kick_badpg) && !zero_nop) { svc.false_trans++; if (injected) { @@ -30657,6 +34340,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool pl = !rr ? svc_method_stream_plausible(vb, len) : false; + /* + * r977: for a kick-family + * entry a plausible walk + * answer still has to carry + * the canonical prefix - a + * wrong-VAS garbage page + * parses plausibly (r976) and + * would re-shadow the real + * page here. + */ + if (pl && kick_badpg && + !svc_kick_canonical(vb, len)) + pl = false; + if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30670,6 +34367,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, vb, len); svc.pushpg_fallback++; ok = true; + /* + * r977: a kick-family + * entry whose page the + * walk re-anchored is + * pinned so later passes + * read the canonical page + * directly (src=2) instead + * of re-paying the walk + * while the content anchor + * catches up (r975 needed + * ~78 s for the anchor; the + * walk answered in seconds). + */ + if (kick_badpg && + wap == SVC_APERTURE_SYS && + svc_kick_canonical(vb, len)) + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), + wpa, + SVC_APERTURE_SYS); if (gsp_sema_dbg && svc_pushpg_fb_log < 64) { svc_pushpg_fb_log++; @@ -30760,7 +34476,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) va, pb, len) && svc_method_stream_plausible(pb, - len)) { + len) && + (!(svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128) || + svc_kick_canonical(pb, + len))) { ch->rescue_last = 0; ch->push_hold_until = 0; svc.gpfifo_rescued++; @@ -32611,6 +36332,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_userd_watch_n = ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { svc.chan[k].gr_ram_userd_watch[wj] = @@ -32619,6 +36344,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_watch_put[wj]; svc.chan[k].gr_ram_userd_watch_hits[wj] = ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; } } svc.chan[k].gr_ram_put_seeded = @@ -32647,7 +36374,13 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_refresh_at = ch->gr_ram_refresh_at; svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; svc.chan[k].userd_scans = ch->userd_scans; + svc.chan[k].fb_userd_scan_at = + ch->fb_userd_scan_at; + svc.chan[k].fb_userd_scan_n = + ch->fb_userd_scan_n; svc.chan[k].notify2d_va = ch->notify2d_va; svc.chan[k].zero_held = ch->zero_held; svc.chan[k].userd_reg = ch->userd_reg; @@ -34161,6 +37894,11 @@ static int svc_thread(void *arg) u64 done_before; vgpu = vgpus[g]; + svc_shared_hold(vgpu); /* CONT.36: GL slot-word keeper */ + /* CONT.42: release any armed client completion sentinel + * on every pass, not only when a completion is being + * processed (see svc_client_slot_sweep). */ + svc_client_slot_sweep(vgpu); svc_handle_probe(vgpu); svc_handle_walk_probe(vgpu); svc_handle_chan_dump(vgpu); @@ -34285,6 +38023,14 @@ static int svc_thread(void *arg) svc_pb_discover(vgpu); /* + * SESSION 153-CONT-34: doorbell writes the guest made + * directly into the mmap'd BAR0 shadow (PCFG window) - + * feed them to the doorbell path so the pump is woken + * without waiting for the next pass boundary. + */ + svc_bar0_pcfg_scan(vgpu); + + /* * r580: the completion hint was hoisted above the * pumped fast path (it must run under load too); it is * not repeated here. diff --git a/kernel/main.c b/kernel/main.c index 509c560..b5c7105 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -849,25 +849,36 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use if (copy_from_user(tmp, buf, count)) return -EFAULT; - if ((u64)pos == 0x88704 || (gsp_sema_dbg && bar0_log < 64)) { + if ((u64)pos >= 0x88000 && (u64)pos < 0x89000) { + /* + * SESSION 153-CONTINUED-34: ANY write in the PCFG + * window is a submission doorbell, not only the + * virtual-function doorbell at 0x88704 the old code + * dispatched. Feed every 4-byte write to the doorbell + * path so the pump is woken for the client's channel + * kicks no matter which register the driver poked + * (the service thread also watches the mmap'd shadow + * for writes that never trap). Logging stays bounded. + */ static u32 db_log; u64 v = 0; memcpy(&v, tmp, count); - - if ((u64)pos == 0x88704) { - if (gsp_sema_dbg && db_log < 512) { - pr_info("vgsp: DOORBELL write pos=0x%llx size=%zu val=0x%llx\n", - (u64)pos, count, v); - db_log++; - } - if (count == 4) - gsp_sema_service_doorbell((u32)v); - } else { - pr_info("vgsp: BAR0 write pos=0x%llx size=%zu val=0x%llx\n", + if (gsp_sema_dbg && db_log < 512) { + pr_info("vgsp: PCFG write pos=0x%llx size=%zu val=0x%llx\n", (u64)pos, count, v); - bar0_log++; + db_log++; } + if (count == 4) + gsp_sema_service_doorbell((u32)v); + } else if ((u64)pos == 0x88704 || + (gsp_sema_dbg && bar0_log < 64)) { + u64 v = 0; + + memcpy(&v, tmp, count); + pr_info("vgsp: BAR0 write pos=0x%llx size=%zu val=0x%llx\n", + (u64)pos, count, v); + bar0_log++; } ret = nv_vgpu_bar_emul_write(vgpu, 0, (u64)pos, tmp, count); |