diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 166 |
1 files changed, 166 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 19fb67e..6db26a4 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1009,6 +1009,18 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \ (GSP_PHYS_RMCALL_VA_BASE + 8 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 8u +/* + * m99: engine-tree branches for identity-mapping the forwarded kernel's + * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual -> + * small PT hierarchy (root[48:47] -> PD2[46:38] -> PD1[37:29] -> PD0[28:21] + * -> PT[20:12]). PD1 pages are cached per PD2 index, PD0 pages per + * (PD2,PD1) index, PT pages per (PD2,PD1,PD0) index; the tables are fresh + * (never fetched) when planted, so the first access at kernel runtime sees + * them even though the GMMU has no PTE-line invalidate. + */ +#define GSP_PHYS_RMCALL_GR_DATA_PD1_MAX 4u +#define GSP_PHYS_RMCALL_GR_DATA_PD0_MAX 8u +#define GSP_PHYS_RMCALL_GR_DATA_PT_MAX 64u /* * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module @@ -2138,6 +2150,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_qmdwin; u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ + /* m99: guest data identity-map table branches (see the PD*_MAX block). */ + struct gsp_phys_boot_chunk + rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; + struct gsp_phys_boot_chunk + rmcall_gr_data_pd0[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX]; + struct gsp_phys_boot_chunk + rmcall_gr_data_pt[GSP_PHYS_RMCALL_GR_DATA_PT_MAX]; + u32 rmcall_gr_data_pd1_idx[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; + u32 rmcall_gr_data_pd0_idx[GSP_PHYS_RMCALL_GR_DATA_PD0_MAX]; + u32 rmcall_gr_data_pt_idx[GSP_PHYS_RMCALL_GR_DATA_PT_MAX]; + u32 rmcall_gr_data_pd1_n, rmcall_gr_data_pd0_n, rmcall_gr_data_pt_n; + u64 rmcall_gr_data_pages; struct { u32 pde_index; u8 pt_slot; @@ -16119,6 +16143,135 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) } /* + * m99: guest data identity mapping. Plant (VA -> HPA) into the engine page + * tree our walker uses. Only root[0] VAs (< 2^47) are reachable; the tables + * along the path are allocated lazily from module chunks and are fresh, so + * planting before the launch is visible to the engine's first walk. The PTE + * carries GENERIC_MEMORY (0x06) in KIND[63:56] like the Turing UVM/RM + * mappings (PITCH does not exist on Turing). + */ +static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd1_get(u32 i2) +{ + struct gsp_phys_boot_chunk *pd2 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2]; + u32 k; + + for (k = 0; k < boot_stage.rmcall_gr_data_pd1_n; k++) + if (boot_stage.rmcall_gr_data_pd1_idx[k] == i2) + return &boot_stage.rmcall_gr_data_pd1[k]; + if (boot_stage.rmcall_gr_data_pd1_n >= GSP_PHYS_RMCALL_GR_DATA_PD1_MAX) + return NULL; + k = boot_stage.rmcall_gr_data_pd1_n; + if (gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_pd1[k], 1)) + return NULL; + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd1[k]); + boot_stage.rmcall_gr_data_pd1_idx[k] = i2; + boot_stage.rmcall_gr_data_pd1_n++; + put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_pd1[k], 0)), + (u8 *)gsp_phys_chunk_page_va(pd2, 0) + + (size_t)i2 * 8); + dma_wmb(); + pr_info("vgsp-gr: data-map PD2[%u] -> pd1 dma=0x%llx\n", i2, + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd1[k], 0)); + return &boot_stage.rmcall_gr_data_pd1[k]; +} + +static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pd0_get(u32 i2, + u32 i1) +{ + struct gsp_phys_boot_chunk *pd1 = gsp_phys_rmcall_gr_data_pd1_get(i2); + u32 key = (i2 << 9) | i1; + u32 k; + + if (!pd1) + return NULL; + for (k = 0; k < boot_stage.rmcall_gr_data_pd0_n; k++) + if (boot_stage.rmcall_gr_data_pd0_idx[k] == key) + return &boot_stage.rmcall_gr_data_pd0[k]; + if (boot_stage.rmcall_gr_data_pd0_n >= GSP_PHYS_RMCALL_GR_DATA_PD0_MAX) + return NULL; + k = boot_stage.rmcall_gr_data_pd0_n; + if (gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_pd0[k], 1)) + return NULL; + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pd0[k]); + boot_stage.rmcall_gr_data_pd0_idx[k] = key; + boot_stage.rmcall_gr_data_pd0_n++; + put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_pd0[k], 0)), + (u8 *)gsp_phys_chunk_page_va(pd1, 0) + + (size_t)i1 * 8); + dma_wmb(); + pr_info("vgsp-gr: data-map PD1[%u] -> pd0 dma=0x%llx (i2=%u)\n", i1, + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pd0[k], 0), + i2); + return &boot_stage.rmcall_gr_data_pd0[k]; +} + +static struct gsp_phys_boot_chunk *gsp_phys_rmcall_gr_data_pt_get(u32 i2, + u32 i1, + u32 i0) +{ + struct gsp_phys_boot_chunk *pd0 = + gsp_phys_rmcall_gr_data_pd0_get(i2, i1); + u32 key = (i2 << 18) | (i1 << 9) | i0; + u32 k; + + if (!pd0) + return NULL; + for (k = 0; k < boot_stage.rmcall_gr_data_pt_n; k++) + if (boot_stage.rmcall_gr_data_pt_idx[k] == key) + return &boot_stage.rmcall_gr_data_pt[k]; + if (boot_stage.rmcall_gr_data_pt_n >= GSP_PHYS_RMCALL_GR_DATA_PT_MAX) + return NULL; + k = boot_stage.rmcall_gr_data_pt_n; + if (gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_pt[k], 1)) + return NULL; + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_pt[k]); + boot_stage.rmcall_gr_data_pt_idx[k] = key; + boot_stage.rmcall_gr_data_pt_n++; + /* 16 B dual PDE: big half unused, small half -> 4 KB PT. */ + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)i0 * 16); + put_unaligned_le64(gsp_phys_rmcall_pde((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_pt[k], 0)), + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)i0 * 16 + 8); + dma_wmb(); + pr_info("vgsp-gr: data-map PD0[%u] -> pt dma=0x%llx (i2=%u i1=%u)\n", i0, + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_gr_data_pt[k], 0), + i2, i1); + return &boot_stage.rmcall_gr_data_pt[k]; +} + +static int gsp_phys_rmcall_gr_data_map(u64 va, u64 hpa) +{ + struct gsp_phys_boot_chunk *pt; + u32 r = (u32)((va >> 47) & 0x1u); + u32 i2 = (u32)((va >> 38) & 0x1ffu); + u32 i1 = (u32)((va >> 29) & 0x1ffu); + u32 i0 = (u32)((va >> 21) & 0x1ffu); + u32 ip = (u32)((va >> 12) & 0x1ffu); + u64 pte; + + if (r != 0) + return -ERANGE; /* root[0] only; the demo VAs are < 2^47 */ + pt = gsp_phys_rmcall_gr_data_pt_get(i2, i1, i0); + if (!pt) + return -ENOSPC; + pte = gsp_phys_rmcall_pte(hpa) | (6ULL << 56); + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)ip * 8); + dma_wmb(); + boot_stage.rmcall_gr_data_pages++; + return 0; +} + +/* * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The * caller has resolved the launch's pages; every mapping is fresh or proven * unchanged, the QMD release0 and the channel release are authored either by @@ -16831,6 +16984,19 @@ gr_page_unmapped: le32_to_cpu(iv[172]), le32_to_cpu(iv[173])); } } + if (req->n_data) { + u32 dmapped = 0, dfailed = 0, di; + + for (di = 0; di < req->n_data; di++) { + if (gsp_phys_rmcall_gr_data_map(req->data[di].va, + req->data[di].hpa)) + dfailed++; + else + dmapped++; + } + pr_info("vgsp-gr: guest data identity-map: %u pages in the engine tree, %u failed (pt pages=%u)\n", + dmapped, dfailed, boot_stage.rmcall_gr_data_pt_n); + } if (gsp_phys_rmcall_gr_gpc_mmu_init_value()) gsp_phys_rmcall_gr_gpc_mmu_init(bar0); pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", |