From abb6b30b0d1020f30171f3e361a1591baed3676b Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Wed, 7 Oct 2026 14:42:02 -0400 Subject: vgsp: m102 - relocate guest data into the server-managed VA region and rewrite bank pointers The server refuses UPDATE_PDE_2 (status 0x1f) for guest CUDA VAs: they are client-managed, while the golden VAS reserves the server-managed region [0x100000000, 0x11fffffff) (gsp_phys_booter.c:8455). Map each resolved data page there at an affine-relocated VA (identity HPA, no copies), rewrite every 8-byte pointer value inside the shadowed const-bank pages (banks 0..7) that falls inside a mapped guest window to the relocated VA, and raise the Tier-1 window pool to 32 windows x 512 PTEs so the relocated pages fit. The kernel now reaches its real data through the server-installed windows. --- kernel/gsp_phys_booter.c | 51 +++++++++++++++++++++++++++++++++--- kernel/gsp_phys_booter.h | 9 +++++++ kernel/gsp_sema_service.c | 66 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 115 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 04788d2..53932d4 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1104,8 +1104,8 @@ bool gsp_phys_boot_requested(void) * inline writes. Both are deliberately outside rmcall_chunks so the COPY0 * stage-2 registration walk stays byte-identical. */ -#define GSP_PHYS_RMCALL_GR_PT_PAGES 8u -#define GSP_PHYS_RMCALL_GR_PT_ENTRIES 32u +#define GSP_PHYS_RMCALL_GR_PT_PAGES 32u +#define GSP_PHYS_RMCALL_GR_PT_ENTRIES 512u #define GSP_PHYS_RMCALL_GR_SHADOW_PAGES 24u /* @@ -16150,6 +16150,44 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * carries GENERIC_MEMORY (0x06) in KIND[63:56] like the Turing UVM/RM * mappings (PITCH does not exist on Turing). */ +/* + * m102: rewrite the data pointers inside a shadowed const-bank page: any + * 8-byte value that falls inside a mapped guest data window is replaced by + * the affine-relocated server-managed VA (the engine cannot walk the + * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). + */ +static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank) +{ + u32 rew = 0, k; + + if (!req->n_data_wins) + return; + for (k = 0; k + 1 < n_words; k += 2) { + u64 v = (u64)w[k] | ((u64)w[k + 1] << 32); + u64 wb; + u32 x; + + if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 7)) + continue; + wb = v & ~0x1fffffULL; + for (x = 0; x < req->n_data_wins; x++) { + if (req->data_wins[x].va_base != wb) + continue; + v = req->data_wins[x].reloc_base | + (v & 0x1fffffULL); + w[k] = (u32)v; + w[k + 1] = (u32)(v >> 32); + rew++; + break; + } + } + if (rew) + pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", + bank, rew); +} + /* * m101: map one guest data page through the server walker (Tier-1 * DMA_UPDATE_PDE_2), exactly like the launch request pages. On the @@ -16500,6 +16538,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, GSP_PHYS_PAGE_SIZE); copied++; } + if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && + !(pg->qmd_word & 1)) + gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + GSP_PHYS_PAGE_SIZE / 4, req, + (pg->qmd_word - 32) / 2); /* * m98: dump the program prologue (qmd_word 48, from * the QMD's in-page entry offset) and the param/const-0 @@ -16892,7 +16935,9 @@ gr_page_unmapped: for (di = 0; di < req->n_data; di++) { if (gsp_phys_rmcall_gr_data_map(bar0, - req->data[di].va, + req->data[di].reloc_va ? + req->data[di].reloc_va : + req->data[di].va, req->data[di].hpa, boot_stage.rmcall_gr_submit_epoch)) dfailed++; diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 29cf07f..896a436 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -366,6 +366,13 @@ struct gsp_phys_gr_inline { struct gsp_phys_gr_data_page { u64 va; /* guest VA, page aligned */ u64 hpa; /* host PA, page aligned */ + u64 reloc_va; /* m102: server-managed VA this hpa is mapped at */ +}; + +/* m102: guest 2 MB window -> relocated (server-managed) window base. */ +struct gsp_phys_gr_data_win { + u64 va_base; /* guest VA window base (2 MB aligned) */ + u64 reloc_base; /* relocated VA window base (2 MB aligned) */ }; struct gsp_phys_gr_submit { @@ -390,6 +397,8 @@ struct gsp_phys_gr_submit { u32 n_inl; const struct gsp_phys_gr_data_page *data; /* m99: identity pages */ u32 n_data; + const struct gsp_phys_gr_data_win *data_wins; /* m102: window map */ + u32 n_data_wins; u64 rel_hpa; /* guest-visible QMD release0 page, or 0 */ u32 rel_off; u32 rel_payload; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6dd0c7e..dfc9922 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3251,9 +3251,24 @@ struct svc_gr_page { struct svc_gr_data { u64 va; /* guest VA, page aligned */ u64 pa; /* guest PA, page aligned (0 = unresolved) */ + u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ }; +/* + * m102: guest window -> relocated window base. The server only installs + * page-directory entries in its managed region (the golden VAS reserves + * [0x100000000, 0x11fffffff), 256 x 2 MB windows); guest CUDA VAs are + * client-managed and refused with 0x1f. Data pages are mapped in the + * managed region at an affine-relocated VA and the pointers in the bank + * pages are rewritten to match. + */ +#define SVC_GR_DATA_WINS_MAX 64 +struct svc_gr_data_win { + u64 va_base; /* guest window base (2 MB aligned) */ + u64 reloc_base; /* relocated window base */ +}; + struct svc_gr_cand { bool valid; bool offered; /* consumed by a request or a fallback */ @@ -8277,6 +8292,8 @@ struct svc_fwd_req { struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX]; u32 gr_n_data; struct svc_gr_data gr_data[SVC_GR_DATA_MAX]; + u32 gr_n_data_wins; + struct svc_gr_data_win gr_data_wins[SVC_GR_DATA_WINS_MAX]; }; static struct svc_fwd_req svc_fwd_req; @@ -9087,6 +9104,7 @@ static void svc_fwd_gr_work(void) */ static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX]; static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX]; + static struct gsp_phys_gr_data_win data_wins[SVC_GR_DATA_WINS_MAX]; struct gsp_phys_gr_submit gs; struct gsp_phys_backend_copy_result cp = { 0 }; u64 rel_hpa = 0; @@ -9174,9 +9192,17 @@ static void svc_fwd_gr_work(void) break; data[gs.n_data].va = dg->va; data[gs.n_data].hpa = dhpa; + data[gs.n_data].reloc_va = dg->reloc_va; gs.n_data++; } gs.data = data; + gs.n_data_wins = svc_fwd_req.gr_n_data_wins; + for (i = 0; i < svc_fwd_req.gr_n_data_wins; i++) { + data_wins[i].va_base = svc_fwd_req.gr_data_wins[i].va_base; + data_wins[i].reloc_base = + svc_fwd_req.gr_data_wins[i].reloc_base; + } + gs.data_wins = data_wins; gs.pages = pages; gs.n_pages = svc_fwd_req.gr_n_pages; gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline; @@ -9529,20 +9555,24 @@ static bool svc_gr_page_inline_covers(u64 page) * Resolution only happens on the service thread; the work item turns the * guest PAs into HPAs and the backend plants the identity PTEs. */ +#define SVC_GR_DATA_RELOC_BASE 0x100000000ULL + static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; - u32 n_win = 0; + u32 n_win = 0, n_cand; u32 off = 0, i, w; u8 kind; svc_fwd_req.gr_n_data = 0; + svc_fwd_req.gr_n_data_wins = 0; /* * m100: scan every const-bank page the launch carries (QMD words * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from * c[0x4][0x0], not from the c[0x0] param page). Each page's first * 2 KB is scanned for plausible device VAs. */ + n_cand = 0; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; @@ -9568,24 +9598,43 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) (v & 3)) continue; wv = v & ~0x1fffffULL; - for (w = 0; w < n_win; w++) + for (w = 0; w < n_cand; w++) if (wins[w] == wv) break; - if (w == n_win && n_win < ARRAY_SIZE(wins)) - wins[n_win++] = wv; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) + wins[n_cand++] = wv; } } - for (w = 0; w < n_win; w++) { + for (w = 0; w < n_cand; w++) { int wi; for (wi = -1; wi <= 16; wi++) { u64 wv = wins[w] + (u64)wi * 0x200000ULL; - u32 got = 0, p; + u64 rbase; + u32 got = 0, p, x; if (wi < 0 && wins[w] < 0x200000ULL) continue; if (wv >= (1ULL << 47)) break; + for (x = 0; x < svc_fwd_req.gr_n_data_wins; x++) + if (svc_fwd_req.gr_data_wins[x].va_base == wv) + break; + if (x == svc_fwd_req.gr_n_data_wins) { + if (svc_fwd_req.gr_n_data_wins >= + SVC_GR_DATA_WINS_MAX) + return; + svc_fwd_req.gr_data_wins[x].va_base = wv; + rbase = SVC_GR_DATA_RELOC_BASE + + ((u64)svc_fwd_req.gr_n_data_wins << + 21); + svc_fwd_req.gr_data_wins[x].reloc_base = + rbase; + svc_fwd_req.gr_n_data_wins++; + } else { + rbase = + svc_fwd_req.gr_data_wins[x].reloc_base; + } for (p = 0; p < 512; p++) { u64 va = wv + (u64)p * SEMA_PAGE_SIZE; u64 pa = 0; @@ -9600,6 +9649,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) [svc_fwd_req.gr_n_data++]; rec->va = va; rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + rec->reloc_va = rbase + (va & 0x1fffffULL); rec->ap = (u8)svc_ap_of(ap); got++; } @@ -9609,8 +9659,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) return; } } - pr_info("vgsp-sema: svc-gr data fill: %u windows, %u pages resolved\n", - n_win, svc_fwd_req.gr_n_data); + pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u windows, %u pages resolved\n", + n_cand, svc_fwd_req.gr_n_data_wins, svc_fwd_req.gr_n_data); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, -- cgit v1.2.3