diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 345 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 831 |
2 files changed, 1159 insertions, 17 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index afe92b6..4125192 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19425,6 +19425,64 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, } /* + * SESSION 153-CONTINUED-27: big-surface alias bookkeeping. A replayed + * 3D window's colour target (C597 INC 0x800+j*0x40 [upper, lower, w, h, + * fmt] and its TWOD twin INC 0x218 [w, h, upper, lower]) can be a surface + * far larger than the launch's 16-page set (the X screen 0x120400000, + * 1280x720x4 = 900 pages). Two earlier routes are dead ends for it: + * - one-page shadow relocation (r992: `pair addr=0x840 va=0x120400000 + * -> MISS` then FAULT_PDE @ +0xa000 - the raster walk crosses the + * single alias page); + * - Tier-1 windows at the surface's own VAs (CONT.27 r993: PDE_UPDATE + * for pdeIndex 2306/2307 status=0x0, yet GPC2/GPC5 still fault + * FAULT_PDE @ +0x4000/+0xa000 - same class as the r929 v2 FE faults; + * the GPC engines only see the scratch-PT pool). + * The surviving route is the proven one: a CONTIGUOUS run of never-used + * scratch-PT fetch lines (one line = 16 PTEs = 64 KB VA; lines are + * monotonic and consecutive lines are VA-contiguous inside each pool + * block), one PTE per surface page, so the engine reaches all 900 pages + * through one contiguous alias VA and the raster walk never crosses into + * an unmapped page. The pair operands are rewritten to <alias>|<offset> + * by gsp_phys_rmcall_gr_3d_reloc. The table is per-boot: the alias PTEs + * are never rewritten (fetch-line rule) and the screen's HPAs are fixed + * by its single allocation, so every later launch is a pure rewrite. + */ +struct gsp_phys_gr_surf_keep { + u64 va; /* guest surface base VA (page aligned) */ + u64 alias; /* engine-visible contiguous shadow alias base */ + u64 npages; /* pages aliased (declared + slack tail) */ +}; +static struct gsp_phys_gr_surf_keep gsp_phys_gr_surf[4]; +static u32 gsp_phys_gr_surf_n; + +/* + * SESSION 153-CONTINUED-27: slack tail for the alias. r994 measured the + * replay's colour writes landing past the declared w*h*4 footprint + * (1280x720x4 = 0x384000) by up to +0xE000 (11.2 rows) - FAULT_PTE on + * the last fetch line's unwritten PTE slots (0x5d6000..0x5e2000 with the + * alias at 0x250000). The tail pages are aliased too: a page that still + * resolves (inside the guest allocation's own padding - exactly where + * native overdraw would land) maps the real HPA; one that does not + * resolves to a dedicated zeroed module page so the stray writes vanish + * instead of faulting. The chunk is allocated lazily and lives until + * module unload (4 KB). + */ +#define GSP_PHYS_GR_SURF_SLACK_PAGES 32u +static struct gsp_phys_boot_chunk gsp_phys_gr_surf_dummy; + +static u64 gsp_phys_gr_surf_dummy_dma_locked(void) +{ + struct gsp_phys_boot_chunk *d = &gsp_phys_gr_surf_dummy; + + if (!d->pages) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, d, 1u)) + return 0; + gsp_phys_chunk_zero(d); + } + return (u64)gsp_phys_chunk_page_dma(d, 0); +} + +/* * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, * program, tex pools) are installed in the ladder page tree by the Tier-1 @@ -19447,6 +19505,25 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, if (!va || !page) return false; + /* + * SESSION 153-CONTINUED-27: a big surface (the X screen) is aliased + * as ONE contiguous run by the pre-pass; rewrite the operand to the + * alias base | offset. A one-page shadow relocation of it would + * break the raster walk at the first page crossing (r992), and the + * raw VA has no PTE chain the GPC engines can see (r993). + */ + for (i = 0; i < gsp_phys_gr_surf_n; i++) { + if (va >= gsp_phys_gr_surf[i].va && + va < gsp_phys_gr_surf[i].va + + (gsp_phys_gr_surf[i].npages << 12)) { + sh = gsp_phys_gr_surf[i].alias + + (va - gsp_phys_gr_surf[i].va); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } for (i = 0; i < req->n_pages; i++) { if ((req->pages[i].guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || @@ -19531,6 +19608,48 @@ static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) } /* + * SESSION 153-CONTINUED-27: auto-drop of the software-emulated lanes from + * every verbatim 3D replay. Three method triggers in a captured window + * cause the physical engine to repeat work the sema service already + * executes in software - and, on the replay channel, with the WRONG + * context state (no previous-window sticky registers): + * - 0x1b0 LAUNCH_DMA (C597 CE lane): triggers both the CE copy and the + * LOAD_INLINE_DATA upload. The service authors every LID payload into + * the guest pages itself (svc_gl_lid/svc_grce_inline_write) and + * emulates the CE strips (svc_op2d); physically re-running them with + * an unset source/context walks unbounded memory (r992-r996: the + * "march" - faults at whatever page the alias cap ended, e.g. + * 0x5d6000/0x5f4000, after having written every mapped page). + * - 0x5e0 (TWOD RECTS fill trigger): the second point pair of the + * client's screen clear. The service's FERMI_TWOD executor runs every + * fill (r986: 46x `sema-twod-fill ... done=3686400 rc=0`); the + * physical re-run uses the replay channel's unset dst PITCH and + * marches (same fault class). + * - 0x10c NOTIFY: the completion write; without the preceding + * SET_NOTIFY_A/B (which live in other pushes) the engine writes + * address 0 (r996: `faulted @ 0x0_00000000 FAULT_PTE VIRT_WRITE`). + * The kick-notify lane + the backend's release tail author every + * completion the client actually waits on. + * All three are pure triggers - their setup methods stay in the window, + * so dropping the run can never misalign display state. The emulated + * lanes are the live ones (the r986 pixel proof: the screen paints via + * svc_twod fills, and every CE strip retires rc=0 in software). Default + * on; set 0 to replay the raw window unmodified. + */ +static bool gsp_phys_gr_win_drop_auto = 1; +module_param_named(sema_gl_win_drop_auto, gsp_phys_gr_win_drop_auto, bool, + 0644); +MODULE_PARM_DESC(sema_gl_win_drop_auto, + "Drop the software-emulated trigger methods (0x10c NOTIFY, 0x1b0 LAUNCH_DMA, 0x5e0 TWOD fill) from verbatim 3D replays (default 1)"); + +static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) +{ + if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) + return false; + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u; +} + +/* * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch * the DATA of one method inside a replayed 3D window before it reaches the * engine. Format: comma-separated `addr:data` hex pairs, addr = method @@ -19627,7 +19746,8 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, if ((op == 1 || op == 3 || op == 5) && cnt && off + 1u + cnt <= n) span = 1u + cnt; - if (gsp_phys_rmcall_gr_win_drop_match(addr)) { + if (gsp_phys_rmcall_gr_win_drop_match(addr) || + gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { dropped++; off += span; continue; @@ -19703,6 +19823,18 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, * Observed [1 222fa200 40 4 f7 0]. */ lo_i = 0; pair = true; + } else if (op == 1 && addr == 0x218 && cnt >= 4 && + (d[2] & ~0xffu) == 0u && + (d[3] & 0xfffu) == 0u) { + /* SESSION 153-CONTINUED-27: the TWOD twin + * of the colour target - INC 0x218 cnt>=4 + * covers 0x218..0x224 = [w, h, upper, + * lower] (the emulator's SET_DST sequence; + * the glxgears client window fills the X + * screen through it). Shape-checked so a + * non-surface run can never be rewritten. */ + lo_i = 2; + pair = true; } if (pair) { u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | @@ -19735,6 +19867,205 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, return reloc; } +/* + * SESSION 153-CONTINUED-27: alias-build for a big colour target. Consumes + * ceil(npages/16) never-used scratch-PT fetch lines and writes one PTE per + * surface page, so the whole surface is reachable through one CONTIGUOUS + * alias VA (lines are monotonic; consecutive lines are VA-contiguous + * inside a pool block - both pool blocks are checked). Every page's HPA + * comes from the launch's own page set when present, else from the + * service's trusted resolver - the identical vetted translation the + * shadow relocation uses. Lines are consumed even on failure (they are + * never reused, so a half-written line can never be shadowed later); + * the caller then falls back to the existing per-page path. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_gr_surface_alias_locked(const struct gsp_phys_gr_submit *req, + u64 va, u64 npages, u64 *alias_out) +{ + u64 map_pages = npages + GSP_PHYS_GR_SURF_SLACK_PAGES; + u32 nlines; + u32 base_line; + u32 i, j; + u64 first_va, last_va; + + if (map_pages > 4096u) + map_pages = 4096u; + nlines = (u32)((map_pages + + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES - 1) / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES); + base_line = boot_stage.rmcall_guest_pt_next_line; + if (!nlines || base_line + nlines > GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return false; + first_va = gsp_phys_rmcall_guest_pt_win_va( + base_line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)(base_line % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + last_va = gsp_phys_rmcall_guest_pt_win_va( + (base_line + nlines - 1u) / + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)((base_line + nlines - 1u) % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + /* one contiguous run: never straddle a pool block boundary */ + if (last_va != first_va + ((u64)(nlines - 1u) << 16)) + return false; + /* + * Consume the lines before writing: a failure mid-way leaves the + * partially written lines consumed (never reused), so the monotonic + * line invariant - a cached line can never shadow a later mapping - + * holds unconditionally. + */ + boot_stage.rmcall_guest_pt_next_line = base_line + nlines; + for (i = 0; i < nlines; i++) { + u32 line = base_line + i; + u32 page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u32 lip = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u8 *pt; + + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return false; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return false; + for (j = 0; j < GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; j++) { + u64 k = (u64)i * GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES + + j; + u64 pv, hpa = 0; + u32 pi; + + if (k >= map_pages) + break; + pv = va + (k << 12); + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == pv && + req->pages[pi].hpa) { + hpa = req->pages[pi].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(pv, &r) || + !r.hpa) { + /* + * A declared page must resolve; a + * slack-tail page past the guest + * allocation gets the dummy so the + * engine's overdraw vanishes. + */ + if (k < npages) + return false; + hpa = gsp_phys_gr_surf_dummy_dma_locked(); + if (!hpa) + return false; + } else { + hpa = r.hpa; + } + } + put_unaligned_le64( + gsp_phys_rmcall_pte(hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + lip + + j) * + 8); + } + dma_wmb(); + } + *alias_out = first_va; + return true; +} + +/* + * SESSION 153-CONTINUED-27: pre-pass over a raw 3D window. Scans the + * colour-target runs (C597 INC 0x800+j*0x40 [upper, lower, w, h, fmt, + * blksz] and - same surface, shape seen on the TWOD subchannel - INC + * 0x218 cnt>=4 [w, h, upper, lower]) and aliases each big surface once + * per boot so the relocation pass can rewrite its operands to the alias + * (r992/r993 evidence: the X screen 0x120400000, 1280x720 fmt 0xe6, is + * the draw's render target AND the TWOD fill's destination). Small + * surfaces keep the historical one-pair shadow relocation. Called with + * gsp_phys_rm_lock held. + */ +static void gsp_phys_gr_surface_alias_scan_locked( + const u32 *win, u32 n, const struct gsp_phys_gr_submit *req) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + span = 1u + cnt; + } else { + off++; + continue; + } + if (op == 1 && ((addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 4) || + (addr == 0x218 && cnt >= 4))) { + const u32 *d = &win[off + 1u]; + u64 va; + u32 w, h, fmt; + u64 bytes, npages; + u32 sdi; + u64 alias = 0; + + if (addr == 0x218) { + va = ((u64)(d[2] & 0xffu) << 32) | + (u64)(d[3] & 0xfffff000u); + w = d[0]; + h = d[1]; + fmt = 0xe6u; /* TWOD run carries fmt on + * its own IMMD; the surface + * class here is 32bpp */ + } else { + va = ((u64)(d[0] & 0xffu) << 32) | + (u64)(d[1] & 0xfffff000u); + w = d[2]; + h = d[3]; + fmt = d[4]; + } + if (!va || (va & 0xfffu) || fmt != 0xe6u || + !w || !h || w > 8192u || h > 8192u) + goto next; + bytes = (u64)w * 4u * (u64)h; + npages = (bytes + GSP_PHYS_PAGE_SIZE - 1) >> 12; + if (npages < 2u || npages > 4096u) + goto next; + for (sdi = 0; sdi < gsp_phys_gr_surf_n; sdi++) + if (gsp_phys_gr_surf[sdi].va == va) + goto next; /* already aliased */ + if (gsp_phys_gr_surface_alias_locked(req, va, npages, + &alias)) { + if (gsp_phys_gr_surf_n < + ARRAY_SIZE(gsp_phys_gr_surf)) { + gsp_phys_gr_surf[gsp_phys_gr_surf_n].va = + va; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .alias = alias; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .npages = npages; + gsp_phys_gr_surf_n++; + } + pr_info("vgsp-gr: SURFACE alias va=0x%llx pages=%llu (%ux%u fmt=0x%x) -> 0x%llx (contiguous shadow run; operands rewritten)\n", + va, npages, w, h, fmt, alias); + } else { + pr_info("vgsp-gr: SURFACE alias FAILED va=0x%llx (%ux%u fmt=0x%x): operands fall to the per-page path\n", + va, w, h, fmt); + } + } +next: + off += span; + } +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -20991,7 +21322,17 @@ gr_page_unmapped: if (stream3d) { u32 win_n = req->stream_n; u32 ndrop = 0; - u32 nrel = gsp_phys_rmcall_gr_3d_patch_window( + u32 nrel; + + /* + * SESSION 153-CONTINUED-27: a big colour target (the X + * screen) is aliased as ONE contiguous shadow run BEFORE + * the relocation pass runs, so the pass rewrites its + * operands to the alias (see the table above). + */ + gsp_phys_gr_surface_alias_scan_locked(req->stream, + req->stream_n, req); + nrel = gsp_phys_rmcall_gr_3d_patch_window( req->stream, req->stream_n, stream_patch, &win_n, &ndrop, req, &n_shadow); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3bb883d..4701331 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -5685,6 +5685,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_LINE_COUNT 0x41c #define CE_M_PITCH_IN 0x410 #define CE_M_PITCH_OUT 0x414 +#define CE_M_SRC_ORIGIN_X 0x744 +#define CE_M_SRC_ORIGIN_Y 0x748 +#define CE_M_DST_ORIGIN_X 0x74c +#define CE_M_DST_ORIGIN_Y 0x750 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -5792,6 +5796,18 @@ struct svc_push_state { u32 pitch_in, pitch_out; u32 have_pitch_in, have_pitch_out; u32 line_count; + /* + * SESSION 153 CONT.25: C5B5 SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750). The X DDX's read/present paths + * position each strip with SRC_ORIGIN (x in bytes, y in rows for + * compSizeFactor == Bpp == 4; class C5B5 has no null remapper) + * instead of folding it into OFFSET_IN. The linear shadow holds + * the pixels linearly, so the effective operand becomes + * base + origin_y * pitch + origin_x. + */ + u32 src_origin_x, src_origin_y; + u32 dst_origin_x, dst_origin_y; + u32 have_src_origin, have_dst_origin; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -5815,6 +5831,66 @@ struct svc_push_state { u32 qmd[SVC_QMD_WORDS]; }; +/* + * SESSION 153 CONT.25: CE register state persists across pushes on the + * same channel/subchannel in hardware. The X DDX's read-image loop pushes + * the full setup once (SET_SRC_BLOCK_SIZE / OFFSET_IN / PITCH / + * LINE_LENGTH, a 112-byte push) and then per-strip DELTA pushes that only + * carry SRC_ORIGIN, SET_SEMAPHORE_A/PAYLOAD, OFFSET_OUT, (LINE_COUNT) and + * LAUNCH_DMA (a 52-byte push; raw witness: the delta launch decodes + * line=0/in=0/have_out=1/have_sem=1 on h=0x1000013). Decoding each push + * in isolation left line_length 0, so the transfer-shaped delta launch + * built NO op: every other 9-row strip never executed and the readback + * kept half the rows black (the visible even/odd banding). Remember the + * last full setup per channel and fill a delta launch's missing fields + * from it. + */ +struct svc_ce_saved { + u64 h_object; + u64 offset_in; + u32 pitch_in, pitch_out, line_length, line_count; + u32 src_phys_mode, dst_phys_mode; + u32 have_in, have_pitch_in, have_pitch_out; + u32 have_src_mode, have_dst_mode; +}; +#define SVC_CE_SAVED_N 8 +static struct svc_ce_saved svc_ce_saved[SVC_CE_SAVED_N]; +static u32 svc_ce_saved_next; +static u32 svc_cedelta_log; + +static struct svc_ce_saved *svc_ce_saved_slot(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_CE_SAVED_N; i++) + if (svc_ce_saved[i].h_object == h) + return &svc_ce_saved[i]; + return NULL; +} + +static void svc_ce_saved_store(const struct svc_push_state *st, u64 h) +{ + struct svc_ce_saved *s = svc_ce_saved_slot(h); + + if (!s) { + s = &svc_ce_saved[svc_ce_saved_next++ % SVC_CE_SAVED_N]; + memset(s, 0, sizeof(*s)); + s->h_object = h; + } + s->offset_in = st->offset_in; + s->pitch_in = st->pitch_in; + s->pitch_out = st->pitch_out; + s->line_length = st->line_length; + s->line_count = st->line_count; + s->src_phys_mode = st->src_phys_mode; + s->dst_phys_mode = st->dst_phys_mode; + s->have_in = st->have_in; + s->have_pitch_in = st->have_pitch_in; + s->have_pitch_out = st->have_pitch_out; + s->have_src_mode = st->have_src_mode; + s->have_dst_mode = st->have_dst_mode; +} + /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ @@ -12914,6 +12990,291 @@ static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, return true; } +/* ------------------------------------------------------------------ */ +/* FERMI_TWOD (0x902d) executor */ +/* ------------------------------------------------------------------ */ +/* + * SESSION 153-CONTINUED-26: the X DDX paints its surfaces (the screen + * 0x120400000 1280x720 X8R8G8B8 and the cursor 0x120370000 256x256 + * A8R8G8B8) exclusively through subchannel-3 FERMI_TWOD methods: the + * SET_DST_* group, then RENDER_SOLID_PRIM (mode RECTS, color format, + * solid color, RENDER_SOLID_PRIM_POINT_X_Y packed pairs). Without this + * executor every 2D surface paint was a no-op and the surfaces the CE + * readback reads stayed unwritten (the screen byte pattern is all + * 0x00000000 - "the X server's drawing never executes"). + * + * Method map: cl902d.h (FERMI_TWOD_A); encodings: NVA06F_DMA_* + * (1 INC / 3 NON_INC / 4 IMMD_DATA, data in header bits 28:16 / 5 + * ONE_INC). Live-decoded per-frame dialog (guest RAM dump at + * 0x8d30c5f8): NON_INC 0x100 cnt17 (MME/global-render-enable window), + * INC 0x280 clip {0,0,0x7fff,0x7fff}, IMMD 0x204 layout, INC 0x214 + * pitch, INC 0x218 {w,h,upper,lower}, IMMD 0x200 fmt, IMMD 0x2ac + * operation, INC 0x580 {mode,fmt,color}, NON_INC 0x5e0 {packed x/y + * point pairs}, IMMD 0x10c NOTIFY, IMMD 0x100 NOP. + * + * The DST VA is (upper<<32)|lower resolved through the SAME operand + * machinery every other engine op uses (svc_fill), so the 2D painting + * and the DDX's own CE readback of the same VA always agree by + * construction. + */ +struct svc_twod_state { + u64 h_object; + u32 dst_fmt, dst_layout, dst_bsize, dst_pitch, dst_w, dst_h; + u64 dst_off; + u32 have_dst; + u32 operation, rop; + u32 prim_mode, prim_fmt, prim_color; + u32 pt_x, pt_y, have_pt; + u64 fills, fill_bytes, misses; + u64 last_fill_hash, last_fill_va; + u32 fill_skips; + u32 dst_log, fill_log, unk_log; +}; + +#define SVC_TWOD_STATES 4 +static struct svc_twod_state svc_twod_tab[SVC_TWOD_STATES]; + +unsigned gsp_sema_twod = 1; +module_param_named(sema_twod, gsp_sema_twod, uint, 0644); +MODULE_PARM_DESC(sema_twod, + "Execute FERMI_TWOD (0x902d) fills on subchannel 3 (default 1)"); + +static struct svc_twod_state *svc_twod_get(struct svc_channel *ch) +{ + u32 i, free_i = SVC_TWOD_STATES; + + for (i = 0; i < SVC_TWOD_STATES; i++) { + if (svc_twod_tab[i].h_object == ch->h_object) + return &svc_twod_tab[i]; + if (!svc_twod_tab[i].h_object && free_i == SVC_TWOD_STATES) + free_i = i; + } + if (free_i == SVC_TWOD_STATES) + free_i = 0; /* bounded table: steal slot 0 */ + memset(&svc_twod_tab[free_i], 0, sizeof(svc_twod_tab[0])); + svc_twod_tab[free_i].h_object = ch->h_object; + return &svc_twod_tab[free_i]; +} + +static u32 svc_twod_bpp(u32 fmt) +{ + switch (fmt) { + case 0xe8: case 0xf8: case 0xe9: case 0xfb: case 0xfc: case 0xee: + return 2; + case 0xf3: case 0xf4: case 0xf7: case 0x1c: + return 1; + default: + return 4; + } +} + +/* One RECTS primitive: fill the (xa,ya)-(xb,yb) axis-aligned rectangle. */ +static void svc_twod_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_twod_state *tw, u32 xa, u32 ya, u32 xb, + u32 yb, u64 push_va) +{ + u32 bpp = svc_twod_bpp(tw->dst_fmt); + u32 pitch, x0, x1, y0, y1, w, h; + u64 done = 0; + int rc; + + if (!tw->have_dst) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x push=0x%llx SKIP (no DST set)\n", + ch->h_object, push_va); + } + return; + } + /* + * Pitch: the PITCH layout carries it explicitly (the cursor dialog + * pushes 0x400 for a 256-wide 4-bpp surface). The BLOCKLINEAR + * screen dialog sets only w/h; the DDX's own CE readback of the + * same surface rides pitch = w*bpp (5120 = 1280*4), so the linear + * approximation uses w*bpp there - painting and readback agree. + */ + pitch = (tw->dst_layout == 1 && tw->dst_pitch) ? tw->dst_pitch : + tw->dst_w * bpp; + if (!bpp || !pitch || !tw->dst_w || !tw->dst_h) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x SKIP (bpp=%u pitch=%u w=%u h=%u)\n", + ch->h_object, bpp, pitch, tw->dst_w, + tw->dst_h); + } + return; + } + x0 = xa < xb ? xa : xb; + x1 = xa < xb ? xb : xa; + y0 = ya < yb ? ya : yb; + y1 = ya < yb ? yb : ya; + /* Surface-bound clamp (the DDX keeps CLIP at 0x7fff). */ + if (tw->dst_w >= 4 && tw->dst_w <= 0x8000) { + if (x0 >= tw->dst_w) + return; + if (x1 > tw->dst_w) + x1 = tw->dst_w; + } + if (tw->dst_h >= 4 && tw->dst_h <= 0x8000) { + if (y0 >= tw->dst_h) + return; + if (y1 > tw->dst_h) + y1 = tw->dst_h; + } + w = x1 - x0; + h = y1 - y0; + if (!w || !h) + return; + /* + * Replay dedupe: the DDX re-submits the same dialog push buffer + * for every repaint (the pump re-decodes ring entries), and the + * first pass of a reused slot is often a replay (allow=0). The + * fill is idempotent, so re-running is safe - but re-running on + * every replay pass would write megabytes per second for zero + * change. Execute once per (push VA, content hash); a changed + * push (new points/colors/surface) hashes differently and runs. + */ + if (tw->fills && push_va == tw->last_fill_va && + tw->last_fill_hash == svc_pass_hash) { + tw->fill_skips++; + return; + } + tw->last_fill_va = push_va; + tw->last_fill_hash = svc_pass_hash; + /* + * Row-wise fill through the shared operand machinery: the same + * resolution chain (shadow -> pin -> landing -> walk -> scan) the + * CE pitched fill uses, so the operand guards (LOWFILL / + * FILL-PT / FILL-HOLD) apply to 2D painting as well. + */ + rc = svc_fill(vgpu, tw->dst_off + (u64)y0 * pitch + (u64)x0 * bpp, + tw->prim_color, bpp, w, 0, CE_APERTURE_SYS, &done, + h, pitch); + tw->fills++; + if (!rc) + tw->fill_bytes += (u64)w * bpp * h; + else + tw->misses++; + if (tw->fill_log < 256) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x va=0x%llx lay=%u fmt=0x%x pitch=%u rect=%u,%u-%u,%u color=0x%x bpp=%u rows=%u done=%llu rc=%d push=0x%llx\n", + ch->h_object, tw->dst_off, tw->dst_layout, + tw->dst_fmt, pitch, x0, y0, x1, y1, tw->prim_color, + bpp, h, done, rc, push_va); + } +} + +static void svc_twod_method(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 m, u32 v, bool exec, + u64 push_va) +{ + struct svc_twod_state *tw = svc_twod_get(ch); + + switch (m) { + case 0x200: + tw->dst_fmt = v; + break; + case 0x204: + tw->dst_layout = v; + break; + case 0x208: + tw->dst_bsize = v; + break; + case 0x214: + tw->dst_pitch = v; + break; + case 0x218: + tw->dst_w = v; + break; + case 0x21c: + tw->dst_h = v; + break; + case 0x220: + tw->dst_off = (tw->dst_off & 0xffffffffULL) | + ((u64)(v & 0x1ffffff) << 32); + break; + case 0x224: + tw->dst_off = (tw->dst_off & ~0xffffffffULL) | v; + tw->have_dst = 1; + if (tw->dst_log < 64) { + tw->dst_log++; + pr_info("sema-twod-dst: h=0x%x va=0x%llx pitch=%u w=%u h=%u fmt=0x%x lay=%u bsz=0x%x\n", + ch->h_object, tw->dst_off, tw->dst_pitch, + tw->dst_w, tw->dst_h, tw->dst_fmt, + tw->dst_layout, tw->dst_bsize); + } + break; + case 0x2ac: + tw->operation = v; + break; + case 0x2a0: + tw->rop = v; + break; + case 0x580: + tw->prim_mode = v; + break; + case 0x584: + tw->prim_fmt = v; + break; + case 0x588: + tw->prim_color = v; + break; + case 0x5e0: { + u32 x = v & 0xffff, y = (v >> 16) & 0xffff; + + if (tw->prim_mode == 4 /* RECTS: point pairs */) { + if (!tw->have_pt) { + tw->pt_x = x; + tw->pt_y = y; + tw->have_pt = 1; + } else { + tw->have_pt = 0; + if (exec && gsp_sema_twod) + svc_twod_fill(vgpu, ch, tw, + tw->pt_x, tw->pt_y, + x, y, push_va); + } + } else if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x (point mode=%u)\n", + ch->h_object, m, v, tw->prim_mode); + } + break; + } + default: + if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x\n", + ch->h_object, m, v); + } + break; + } +} + +/* + * Dispatch a whole method run's data words. `inc` selects the NVA06F + * DMA semantics: INC_METHOD/ONE_INC (op 1/5) advance the method address + * per word (a contiguous register run); NON_INC (op 3) does NOT - every + * data word goes to the SAME method, which is how the DDX streams the + * RECTS point list into RENDER_SOLID_PRIM_POINT_X_Y (0x5e0; the 2-point + * rect and the 4-point double-band fills are one NON_INC run each, and + * its 17-word 0x100 run is a NO_OPERATION padding block). + */ +static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 addr, bool inc, u32 count, const u8 *data, + u32 avail, bool exec, u64 push_va) +{ + u32 k; + + for (k = 0; k < count && (u64)(k + 1) * 4 <= avail; k++) { + u32 v; + + memcpy(&v, data + 4 * k, 4); + svc_twod_method(vgpu, ch, inc ? addr + 4 * k : addr, v, exec, + push_va); + } +} + /* * SESSION 148 perf (r862/r864): the RC watchdog's class2d notify cycle * keep-alive that needs no push decode and no captured VA. Under the @@ -13624,6 +13985,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * NVC597_BEGIN (0x1618, an IMMD method - its immediate is * the topology in the count field) on the attached * subchannel of the attached channel opens the section. + * SESSION 153-CONTINUED-26: the glxgears-era client draw + * pushes carry no 0x1618 marker - their first run is the + * C597 GL render-target setup (INC 0x840, {1, rtVA, w, h, + * fmt, blksz}) followed by the 0x1b0 draw list - so the RT + * setup opens the section too. Both shapes are gated on + * the sticky attachment, the channel class, and a closed + * section. * The window starts at gl_win_start (the first C597 state * method after the last foreign-subchannel run); the state * methods before the marker are re-scanned so their operand @@ -13636,8 +14004,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * (the section close at the END method / push tail), which * svc_gl_flush slices into the verbatim 3D replay. */ - if (gsp_sema_gl_xlate && !ch->engine_type && op == 4 && - addr == 0x1618 && !svc_gl.seen && + if (gsp_sema_gl_xlate && !ch->engine_type && + ((op == 4 && addr == 0x1618) || + (op == 1 && addr == 0x840 && count >= 4)) && + !svc_gl.seen && svc_gl_att_has(ch->h_object, subch)) { u64 keep_rel = svc_gl.rel_va; u32 keep_pay = svc_gl.rel_payload; @@ -13649,6 +14019,36 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl.start_off = wstart; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* + * SESSION 153-CONTINUED-26: when the RT-setup run is + * the opener, its address word IS the draw's render + * target - put it into the candidate page set, or the + * forwarded replay's shadow relocation cannot rewrite + * the RT VA and the physical launch faults/times out + * (r990: `GR forward failed rc=-110` on every client + * draw while the X section's 7-pointer capture ran). + * Must run AFTER the section reset above. + */ + if (op == 1 && addr == 0x840 && off + 12 <= len) { + /* + * SESSION 153-CONTINUED-27: the run is + * [upper, lower, w, h, fmt, blksz] - the VA + * is ((upper & 0xff) << 32) | lower exactly + * like every other pair recognizer in + * svc_gl_field. Reading only the low word + * added 0x20400000 for a surface whose true + * VA is 0x120400000 (the X screen), so the + * backend's shadow relocation MISSed it and + * the physical replay wrote the raw VA with + * no PTE chain (r992 FAULT_PDE @ +0xa000). + */ + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_gl_add_ptr(((u64)(v0 & 0xff) << 32) | + (u64)(v1 & 0xfffffffc)); + } if (wstart < off) svc_gl_scan_window(wstart, off, buf, len, subch); @@ -13677,6 +14077,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl_shader_type[j] = (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD (0x902d) IMMD_DATA + * methods on subchannel 3 (SET_DST_FORMAT 0x200, layout, + * SET_OPERATION 0x2ac, NOTIFY 0x10c, ...). The value rides + * in the header's bits 28:16. State only; no execution. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 4 && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_method(vgpu, ch, addr, + (hdr >> 16) & 0xfff, + allow_ops || trusted, push_va); + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -13867,6 +14280,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD NON_INC/ONE_INC runs + * on subchannel 3. The dialogs' whole body is NON_INC (the + * 17-word context window and the point pairs); decode the + * data words as cl902d methods before the generic + * whole-run skip below consumes them. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && (op == 3 || op == 5) && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_run(vgpu, ch, addr, op == 5 ? true : false, + count, buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -13899,6 +14326,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memcpy(&v4, buf + off + 20, 4); /* + * SESSION 153-CONTINUED-26: FERMI_TWOD INC runs on + * subchannel 3 (the RENDER_SOLID_PRIM mode/format/color + * triplet at 0x580, the DST group, clip). The three + * notify shapes (0x100 cnt<=1, 0x104, 0x108, 0x10c) stay + * with the watchdog handler below; everything else in the + * cl902d space is decoded here and must not fall into the + * CE method chain. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 1 && + addr >= 0x0100 && addr <= 0x1400 && + !(addr == 0x0100 && count <= 1) && + addr != 0x0104 && addr != 0x0108 && addr != 0x010c) { + svc_twod_run(vgpu, ch, addr, true, count, + buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + off += 4 + count * 4; + continue; + } + + /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of @@ -14601,12 +15049,118 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.pitch_out = v0 & 0xffffff; st.have_pitch_out = 1; break; + /* + * SESSION 153 CONT.25: SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750), pushed as a count-2 (X,Y) + * or count-4 (X,Y,X,Y) INC run (nv_xaa_kepler-style + * SetSrcOrigin: x already scaled by compSizeFactor == Bpp + * for this class). No other class uses these method + * offsets (cl902d/cla06f clear). + */ + case CE_M_SRC_ORIGIN_X: + case CE_M_SRC_ORIGIN_Y: + case CE_M_DST_ORIGIN_X: + case CE_M_DST_ORIGIN_Y: + { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + st.src_origin_x = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + st.src_origin_y = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + st.dst_origin_x = vv[k]; + st.have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + st.dst_origin_y = vv[k]; + st.have_dst_origin = 1; + break; + } + } + } + break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); u32 n_ops_before = n_ops; bool dma_served = false; + /* + * SESSION 153 CONT.25: remember a full setup (this is + * the push that carried LINE_LENGTH + OFFSET_IN) as + * the channel's persistent CE state, then fill a + * transfer-shaped DELTA launch (the driver's per-strip + * follow-up: no LINE_LENGTH/OFFSET_IN of its own, + * OFFSET_OUT and the semaphore release present - the + * witnessed 52-byte pushes on h=0x1000013) from it so + * it builds its op instead of being silently dropped. + */ + if (allow_ops && st.line_length && st.have_in) + svc_ce_saved_store(&st, ch->h_object); + if (transfer != 0 && allow_ops && st.have_out && + st.have_sem && + (!st.line_length || !st.have_in)) { + struct svc_ce_saved *s = + svc_ce_saved_slot(ch->h_object); + + if (s && s->line_length && s->have_in) { + if (!st.line_length) { + st.line_length = + s->line_length; + st.line_count = + s->line_count; + } + if (!st.have_in) { + st.offset_in = + s->offset_in; + st.have_in = 1; + } + if (!st.have_pitch_in && + s->have_pitch_in) { + st.pitch_in = s->pitch_in; + st.have_pitch_in = 1; + } + if (!st.have_pitch_out && + s->have_pitch_out) { + st.pitch_out = s->pitch_out; + st.have_pitch_out = 1; + } + if (!st.have_src_mode && + s->have_src_mode) { + st.src_phys_mode = + s->src_phys_mode; + st.have_src_mode = 1; + } + if (!st.have_dst_mode && + s->have_dst_mode) { + st.dst_phys_mode = + s->dst_phys_mode; + st.have_dst_mode = 1; + } + if (gsp_sema_dbg && + svc_cedelta_log < 64) { + svc_cedelta_log++; + pr_info("sema-cedelta: h=0x%x push=0x%llx dma=0x%08x filled line=%u lc=%u in=0x%llx pi=%u po=%u sx=%u sy=%u (engine state inherited)\n", + ch->h_object, push_va, + dma, st.line_length, + st.line_count, + st.offset_in, + st.pitch_in, + st.pitch_out, + st.src_origin_x, + st.src_origin_y); + } + } + } + if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; @@ -14708,6 +15262,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->type = 0; m->dst = st.offset_out; + /* SESSION 153 CONT.25: blocklinear + * destination strip position. */ + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } /* * r367: the driver's own memset push * (channel_utils.c) proves the @@ -14856,6 +15423,38 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; /* + * SESSION 153 CONT.25: a BLOCKLINEAR + * operand positions each strip with + * SRC/DST_ORIGIN (x in bytes, y in + * rows) instead of folding it into + * OFFSET_IN/OUT. The linear shadow + * holds the pixels linearly, so the + * effective operand is + * base + origin_y*pitch + origin_x. + */ + if (st.have_src_origin) { + u32 sp = (st.have_pitch_in && + st.pitch_in) ? + st.pitch_in : + st.line_length; + + m->src += + (u64)st.src_origin_y * + sp + + st.src_origin_x; + } + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } + /* * r402: a 2D copy (cudaMemcpy2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes with @@ -14911,7 +15510,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (gsp_sema_dbg && (st.have_pitch_out || st.have_pitch_in) && - svc.op2d_raw_log < 12) { + svc.op2d_raw_log < 32) { u32 dl = len < 220 ? len : 220; svc.op2d_raw_log++; @@ -14926,8 +15525,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.pitch_in, m->dst, m->src, (u32)m->bytes, dma); - pr_info("sema-op2d-raw: raw %*phN\n", - dl, buf); + /* %*phN caps at 64 bytes; + * print the push whole in + * 64-byte halves. */ + pr_info("sema-op2d-raw: raw[0] %*phN\n", + dl > 64 ? 64 : dl, + buf); + if (dl > 64) + pr_info("sema-op2d-raw: raw[1] %*phN\n", + dl - 64, + buf + 64); } n_ops++; ops_seq++; @@ -15377,7 +15984,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * in one op"). Bounded. */ if (lm->type == 1) - pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%llu rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u so=%u/%u do=%u/%u\n", ch->h_object, lm->dst, lm->src, lm->bytes, lm->rows, lm->dpitch, @@ -15387,7 +15994,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_pitch_out, st.pitch_out, st.have_pitch_in, - st.pitch_in); + st.pitch_in, + st.have_src_origin, + st.src_origin_y, + st.have_dst_origin, + st.dst_origin_y); } } @@ -24214,7 +24825,7 @@ static void svc_fwd_hold_direct_note(void) * A bounded fallback run (sweep, when no forward is owed) keeps the list * from stalling when the launch was refused. */ -#define SVC_GR_COPY_PEND 8 +#define SVC_GR_COPY_PEND 64 struct svc_gr_copy_pend { bool valid; u32 h_object; @@ -24234,6 +24845,55 @@ static u32 svc_gr_copy_pend_next; static u64 svc_gr_copy_run_total; static u64 svc_gr_copy_run_log; +/* + * SESSION 153 CONT.25: recently-executed copy identities. The sweep + * re-decodes the ring continuation every pass, so an entry it just ran + * would be re-added (the valid-dedup above only covers live entries) and + * re-run forever - with a bounded per-pass run count that pins the runner + * on the same first slots. The content hash changes whenever the driver + * rewrites the push (new frame: the semaphore payload advances), so a + * (push_va, hash, src, dst, bytes, rows) match means "this exact strip + * already ran this frame". + */ +#define SVC_GR_COPY_DONE_N 512 +struct svc_gr_copy_done_rec { + u64 push_va, hash, src, dst; + u32 bytes, rows; +}; +static struct svc_gr_copy_done_rec svc_gr_copy_done[SVC_GR_COPY_DONE_N]; +static u32 svc_gr_copy_done_next; + +static bool svc_gr_copy_done_lookup(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_DONE_N; i++) + if (svc_gr_copy_done[i].push_va == push_va && + svc_gr_copy_done[i].hash == hash && + svc_gr_copy_done[i].src == src && + svc_gr_copy_done[i].dst == dst && + svc_gr_copy_done[i].bytes == bytes && + svc_gr_copy_done[i].rows == rows) + return true; + return false; +} + +static void svc_gr_copy_done_store(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + struct svc_gr_copy_done_rec *d = + &svc_gr_copy_done[svc_gr_copy_done_next++ % + SVC_GR_COPY_DONE_N]; + + d->push_va = push_va; + d->hash = hash; + d->src = src; + d->dst = dst; + d->bytes = bytes; + d->rows = rows; +} + static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, u64 src, u64 dst, u32 bytes, u32 rows, u32 dpitch, u32 spitch, u32 dma, u32 remap) @@ -24242,6 +24902,8 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, if (!bytes || !rows || !src || !dst) return; + if (svc_gr_copy_done_lookup(push_va, hash, src, dst, bytes, rows)) + return; /* this exact strip already ran for this content */ for (i = 0; i < SVC_GR_COPY_PEND; i++) { struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; @@ -24291,17 +24953,33 @@ static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, * when no forward is owed. The executor is the same svc_copy() the CE/GCE * op path uses (handles vidmem shadow sources and sysmem destinations). */ +/* + * SESSION 153 CONT.25: bounded per-call run count for the pending + * copy-out list. The X DDX readback keeps at most two strips in flight + * (the two scratch halves), so running the whole queue in one sweep pass + * overruns the server's consumption and every frame reads the newest + * strips only. Pace at a few per pass; 0 = unlimited (historical). + */ +static uint gsp_sema_gr_copy_run_max; +module_param_named(sema_gr_copy_run_max, gsp_sema_gr_copy_run_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_copy_run_max, + "Max pending GRCE copy-outs executed per sweep pass (0 = unlimited)"); + static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) { - u32 i; + u32 i, ran = 0; for (i = 0; i < SVC_GR_COPY_PEND; i++) { + if (gsp_sema_gr_copy_run_max && + ran >= gsp_sema_gr_copy_run_max) + break; struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; u64 done = 0; int rc; if (!p->valid) continue; + ran++; rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, p->rows, p->dpitch, p->spitch, p->h_object, 0, @@ -24309,6 +24987,8 @@ static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) svc_gr_copy_run_total++; if (rc == 0 || rc == SVC_COPY_FORWARDED) { p->valid = false; + svc_gr_copy_done_store(p->push_va, p->hash, p->src, + p->dst, p->bytes, p->rows); if (svc_gr_copy_run_log < 64) { svc_gr_copy_run_log++; pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", @@ -29892,6 +30572,9 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch u32 have_in = 0, have_out = 0; u32 pitch_in = 0, pitch_out = 0; u32 line_len = 0, line_count = 0; + u32 src_origin_x = 0, src_origin_y = 0; + u32 dst_origin_x = 0, dst_origin_y = 0; + u32 have_src_origin = 0, have_dst_origin = 0; u32 last_dma = 0, last_remap = 0; u32 cp_subch = ~0u; bool used_tail = false; @@ -30205,6 +30888,38 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch ((vv[k] & 0xffffu) << 16); } + } else if (addr >= CE_M_SRC_ORIGIN_X && + addr <= CE_M_DST_ORIGIN_Y && n) { + /* + * SESSION 153 CONT.25: the X DDX readback + * pair positions each 6-row strip with + * SRC_ORIGIN (x bytes / y rows) instead of + * folding it into OFFSET_IN - the same + * methods the main walk now decodes. The + * strip y steps 6 per push (A: 120, B: 126, + * A: 132, ...) and the copy must read + * OFFSET_IN + origin_y*PITCH_IN. + */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + src_origin_x = vv[k]; + have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + src_origin_y = vv[k]; + have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + dst_origin_x = vv[k]; + have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + dst_origin_y = vv[k]; + have_dst_origin = 1; + break; + } + } } else if (addr == 0x300 && count) { u32 dma = w0; @@ -30212,28 +30927,104 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a * release-only / semaphore op - no data * movement, nothing to record). + * + * SESSION 153 CONT.25: remember a full setup + * (LINE_LENGTH + OFFSET_IN present) as the + * channel's persistent CE state and fill a + * DELTA launch (the X DDX readback pair's + * second push: no setup of its own, only + * SRC_ORIGIN + SEM_A/PAYLOAD + OFFSET_OUT + + * LAUNCH_DMA) from it, exactly like the main + * walk - without this every odd strip of the + * frame is dropped. */ last_dma = dma; + if (have_in && off_in && line_len) { + struct svc_push_state tst; + + memset(&tst, 0, sizeof(tst)); + tst.offset_in = off_in; + tst.pitch_in = pitch_in; + tst.pitch_out = pitch_out; + tst.line_length = line_len; + tst.line_count = line_count; + svc_ce_saved_store(&tst, + ch->h_object); + } + if ((dma & 0x3) && have_out && off_out && + (!have_in || !line_len)) { + struct svc_ce_saved *sv = + svc_ce_saved_slot( + ch->h_object); + + if (sv && sv->line_length && + sv->have_in) { + if (!line_len) { + line_len = + sv->line_length; + line_count = + sv->line_count; + } + if (!have_in) { + off_in = + sv->offset_in; + have_in = 1; + } + if (!pitch_in) + pitch_in = + sv->pitch_in; + if (!pitch_out) + pitch_out = + sv->pitch_out; + } + } if ((dma & 0x3) && have_in && have_out && line_len && line_count && off_in && off_out) { + /* + * SESSION 153 CONT.25: fold the + * per-strip SRC/DST_ORIGIN into the + * operands (bytes on X, rows on Y; + * the linear shadow makes + * base + y*pitch + x the truth). + * line_len is BYTES for C5B5 + * (LINE_LENGTH_IN semantics), not + * dwords: the old *4 inflated every + * strip 4x. + */ + u64 src_f = off_in, dst_f = off_out; + + if (have_src_origin) + src_f += + (u64)src_origin_y * + (pitch_in ? + pitch_in : + line_len) + + src_origin_x; + if (have_dst_origin) + dst_f += + (u64)dst_origin_y * + (pitch_out ? + pitch_out : + line_len) + + dst_origin_x; if (inline_only) svc_ahead_copy_in(vgpu, ch->h_object, push_va, svc_push_hash(buf, len), - off_in, off_out, - line_len * 4, + src_f, dst_f, + line_len, line_count, dma); else if (forward) { svc_gr_copy_pend_add(ch->h_object, push_va, svc_push_hash(buf, len), - off_in, off_out, - line_len * 4, + src_f, dst_f, + line_len, line_count, pitch_out ? pitch_out : - line_len * 4, + line_len, pitch_in, dma, last_remap); cp_subch = subch; @@ -31551,8 +32342,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * backend is off), run any pending GRCE copies so * they cannot sit forever (the completion edge is * the primary path). + * + * SESSION 153 CONT.25: also run when a forward IS + * owed - the X DDX readback pair's copies have no + * forwarded launch behind them, and a stuck + * unoffered candidate (valid && !offered) blocked + * this fallback forever: 1443 copy pends, 0 runs, + * the server's per-copy completion wait timing out + * on every strip. The completion edge still runs + * them first when it fires; the runner retires + * each entry once, so running early is bounded. */ - if (gsp_sema_gr_xlate && !svc_gr_forward_pending()) + if (gsp_sema_gr_xlate) svc_gr_copy_pend_run(vgpu); } |