diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 126 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 5 |
2 files changed, 111 insertions, 20 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 86205d1..b03ac30 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -2160,6 +2160,20 @@ static struct gsp_phys_boot_stage { u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; + /* + * m114: pre-commit bounce window. 112 pages planted into the main + * small-PT at slots 0x110..0x17f during stage 3 (the proven win0 + * pattern: the SM walker resolves the commit-time tree only). The + * launch copies the kernel's data pages in, rewrites the pointers to + * these VAs (reloc base 0x110000), and copies the results back after + * the release; page 100 (VA 0x174000) carries the mail message. + */ + struct gsp_phys_boot_chunk rmcall_gr_bounce; + u32 rmcall_gr_bounce_used; +#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 112u +#define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u +#define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL +#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 100u /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -9405,6 +9419,50 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* + * m114: allocate the bounce window and plant its PTEs into the main + * small-PT at slots 0x110..0x17f - pre-commit direct writes, the + * same pattern as the window-0 pool the SM provably reads. The mail + * message goes into bounce page GSP_PHYS_RMCALL_GR_MAIL_SLOT. + */ + if (!boot_stage.rmcall_gr_bounce.pages) { + struct gsp_phys_boot_chunk *ptb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + u32 i; + + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_bounce, + GSP_PHYS_RMCALL_GR_BOUNCE_PAGES)) { + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_bounce); + for (i = 0; i < GSP_PHYS_RMCALL_GR_BOUNCE_PAGES; i++) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_bounce, i)), + (u8 *)gsp_phys_chunk_page_va(ptb, 0) + + (size_t)(GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 + i) * 8); + dma_wmb(); + pr_info("vgsp-boot: rm-call: bounce window planted: %u pages at VA 0x%llx..0x%llx (main PT slots 0x%x..0x%x)\n", + GSP_PHYS_RMCALL_GR_BOUNCE_PAGES, + GSP_PHYS_RMCALL_GR_BOUNCE_VA0, + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (GSP_PHYS_RMCALL_GR_BOUNCE_PAGES - 1) * + GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0, + GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_BOUNCE_PAGES - 1); + { + static const char mail[] = + "the little gpu man says you've got mail"; + memcpy(gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, + GSP_PHYS_RMCALL_GR_MAIL_SLOT), + mail, sizeof(mail)); + } + } else { + pr_err("vgsp-boot: rm-call: bounce window allocation failed\n"); + } + } + + /* * P6a guest windows: start from explicit zero (invalid PTE) for both * slots, then, when the guest GPA is already pinned (it must be pinned * before stage 3 for this), pre-build both slots with the translated @@ -16672,15 +16730,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, boot_stage.rmcall_gr_data_pt [GSP_PHYS_RMCALL_GR_DATA_WINS - 1].pages); { - u64 mva = 0x1f0000ULL; + u64 mva = GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + ((u64)GSP_PHYS_RMCALL_GR_MAIL_SLOT + << 12); int mret = -EINVAL; - if (boot_stage.rmcall_gr_mail.pages) - mret = gsp_phys_rmcall_gr_data_map( - bar0, mva, - (u64)gsp_phys_chunk_page_dma( - &boot_stage.rmcall_gr_mail, 0), - boot_stage.rmcall_gr_submit_epoch); + if (boot_stage.rmcall_gr_bounce.pages) + mret = 0; if (!mret) { w[0] = (u32)mva; w[1] = (u32)(mva >> 32); @@ -17097,22 +17153,28 @@ gr_page_unmapped: le32_to_cpu(iv[172]), le32_to_cpu(iv[173])); } } - if (req->n_data) { - u32 dmapped = 0, dfailed = 0, di; + if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + u32 dcopied = 0, dfailed = 0, di; for (di = 0; di < req->n_data; di++) { - if (gsp_phys_rmcall_gr_data_map(bar0, - req->data[di].reloc_va ? - req->data[di].reloc_va : - req->data[di].va, - req->data[di].hpa, - boot_stage.rmcall_gr_submit_epoch)) + u32 bi = (u32)((req->data[di].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> + 12); + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + !req->data[di].hpa) { dfailed++; - else - dmapped++; + continue; + } + memcpy(gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, bi), + (u8 *)__va(req->data[di].hpa), + GSP_PHYS_PAGE_SIZE); + dcopied++; } - pr_info("vgsp-gr: guest data identity-map: %u pages in the engine tree, %u failed (pt pages=%u)\n", - dmapped, dfailed, boot_stage.rmcall_gr_data_pt_n); + dma_wmb(); + pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped\n", + dcopied, dfailed); if (gsp_phys_rmcall_gr_recommit_value()) { struct gsp_phys_rmcall_pdb_params rp; struct { @@ -17517,6 +17579,32 @@ gr_page_unmapped: } gsp_phys_gr_ok++; + /* + * m114: the kernel ran against the pre-commit bounce window; copy the + * results back to the guest's real pages (identity HPAs in the + * request). All mapped pages are written back; inputs are unchanged + * by the kernels, so copying them back is a no-op. + */ + if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + u32 wbi, wrote = 0; + + for (wbi = 0; wbi < req->n_data; wbi++) { + u32 bi = (u32)((req->data[wbi].reloc_va - + GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> + 12); + + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + !req->data[wbi].hpa) + continue; + memcpy((u8 *)__va(req->data[wbi].hpa), + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_bounce, bi), + GSP_PHYS_PAGE_SIZE); + wrote++; + } + pr_info("vgsp-gr: bounce copy-out: %u pages written back\n", + wrote); + } poll = w1; if (bar0) pr_info("vgsp-gr: SKED readback (ok): HWW_ESR=0x%08x INFO2=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x\n", diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index caaaae3..24b3adf 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -9562,6 +9562,9 @@ static bool svc_gr_page_inline_covers(u64 page) * the demo buffers are small on the native path). */ #define SVC_GR_DATA_PAGES_PER_PTR 64u +/* m114: total pages the pre-commit bounce window can carry (slot 100 is + * the mail message). */ +#define SVC_GR_DATA_BOUNCE_MAX 100u static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { @@ -9633,7 +9636,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) continue; /* already mapped */ if (svc_resolve_va(vgpu, va, &pa, &ap)) break; - if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_MAX) + if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) return; rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; |