diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 106 |
1 files changed, 106 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8fc663f..e96b929 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -30688,6 +30688,17 @@ static bool svc_gr_page_inline_covers(u64 page) ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return true; } + /* + * SESSION 153-CONTINUED-91: the request's records may have been + * re-rooted onto this launch's bind targets (svc_gr_inline_retarget), + * so a page the request names can be covered by an inline write even + * when the candidate's original record page differs. + */ + for (i = 0; i < svc_fwd_req.gr_n_inline; i++) { + if ((svc_fwd_req.gr_inline[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } return false; } @@ -32627,6 +32638,92 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, n_cand, svc_fwd_req.gr_n_data, reloc_base); } +static u32 svc_gr_inline_retarget_log; + +/* + * SESSION 153-CONTINUED-91: delivery follows the launch. + * + * The captured program-family inline records (the real-upload payloads and + * the authored stage images) are keyed to the program family of the section + * that captured them, but that family is sticky guest state: the client + * cycles push slots whose content names a different program family per slot + * (observed live: records at 0x4350000/0x4300000 while the launched window + * binds 0x42d0000/0x41c0000/0x4400000). The record pages are noted into the + * launch page set, so the delivery lands on pages no bind names, the + * engine's own bind VAs keep whatever stale content the guest memory holds, + * and the DS latches `Shader Program Header N Error` (r1297: zero + * INLINE-APPLY, fetch page all zeros). + * + * Re-root every record onto the launch's own bind targets by in-page + * offset: the window's own binds (prog_bind_va) first, then the launch's + * program family at the record's own offset (which is also where the + * synthesized j=5/j=2 stage binds sit, family+0x0 / family+0x200). A record + * that continues the previous record's page (the page-crossing tail of a + * split real-upload) follows its head's new target so the split delivery + * stays contiguous. A record already on a build-bind page is left alone; + * the pass is idempotent and a no-op whenever the families already agree. + */ +static void svc_gr_inline_retarget(u64 prog_va, const u64 *binds, u32 n_binds) +{ + u64 lfam = prog_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 prev_page = 0, prev_new = 0; + u32 i, moved = 0; + + if (!lfam || !svc_gr_cand.n_inline) + return; + for (i = 0; i < svc_gr_cand.n_inline && i < SVC_GR_INLINE_MAX; i++) { + struct svc_gr_inline *inl = &svc_gr_cand.inline_recs[i]; + u64 page = inl->dst_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = inl->dst_va & (SEMA_PAGE_SIZE - 1); + u64 target = 0; + u32 b; + + if (!page) + continue; + /* Page-crossing tail: follow the head's new location. */ + if (prev_page && page == prev_page + SEMA_PAGE_SIZE && !off) { + u64 nv = (prev_new & ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE; + + if (nv != inl->dst_va) { + inl->dst_va = nv; + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + continue; + } + /* A build bind whose in-page offset matches this record. */ + for (b = 0; b < n_binds && b < SVC_GR_CAND_PROG_BINDS; b++) { + if (binds[b] && + (binds[b] & (SEMA_PAGE_SIZE - 1)) == off) { + target = binds[b]; + break; + } + } + /* Else the launch's own family at the record's offset. */ + if (!target) + target = lfam + off; + if ((target & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) { + inl->dst_va = (target & ~(u64)(SEMA_PAGE_SIZE - 1)) + + (inl->dst_va - page); + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + } + if (moved && svc_gr_inline_retarget_log < 16u) { + svc_gr_inline_retarget_log++; + pr_info("sema-gr: INLINE-RETARGET prog=0x%llx lfam=0x%llx binds=%u moved=%u/%u\n", + prog_va, lfam, n_binds, moved, + svc_gr_cand.n_inline); + for (i = 0; i < svc_gr_cand.n_inline && i < 8u; i++) + pr_info("sema-gr: inl[%u] dst=0x%llx len=%u\n", i, + svc_gr_cand.inline_recs[i].dst_va, + svc_gr_cand.inline_recs[i].len); + } +} + static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -32736,6 +32833,15 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_param_size = svc_gr_cand.param_size; memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, sizeof(svc_fwd_req.gr_qmd)); + /* + * SESSION 153-CONTINUED-91: re-root the captured program-family inline + * records onto this launch's own bind targets before they are copied + * into the request (see svc_gr_inline_retarget). The candidate's + * records are retargeted in place so the page-note loop below and + * svc_gr_scan_inline() both follow the delivered VAs. + */ + svc_gr_inline_retarget(svc_gr_cand.prog_va, svc_gr_cand.prog_bind_va, + svc_gr_cand.n_prog_binds); svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; |