diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1068 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 6 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 8 |
3 files changed, 1063 insertions, 19 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0c2e75a..764e261 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -330,6 +330,14 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707u #define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO 0x20801109u #define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX 0x2080012bu +/* + * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106), the golden + * client's VAS construct control. Mode 25 sends it only in the gated + * zero-level no-op-range form (see + * GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); the captured three level PAs + * are server-RM-internal and have no safe client source. + */ +#define GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES 0x90f10106u #define GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX 0x2080012cu /* * NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063): @@ -382,6 +390,64 @@ bool gsp_phys_boot_requested(void) * 32 MB but the live server MAIN is 0x133000; PATCH lives at +0x140000 to * stay clear of MAIN's tail). */ #define GSP_PHYS_RMCALL_VA_GR_PATCH (GSP_PHYS_RMCALL_VA_GR_CTX + 0x140000ULL) +/* + * Golden9e (mode 25): the promoted FECS/PRIV/UNR buffers and the VA-only + * 3D globals are placed inside the same 2 MB window as MAIN/PATCH so every + * promoted VA is backed by the ctx PDB page tables (the real golden client + * mapped all of them in its VAS; the earlier mode-25 replay wrote unmapped + * VAs, which can fault the engine context load). + */ +#define GSP_PHYS_RMCALL_VA_GOLDEN9_FECS (GSP_PHYS_RMCALL_VA_GR_CTX + 0x150000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV (GSP_PHYS_RMCALL_VA_GR_CTX + 0x160000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR (GSP_PHYS_RMCALL_VA_GR_CTX + 0x1e0000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB (GSP_PHYS_RMCALL_VA_GR_CTX + 0x260000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL (GSP_PHYS_RMCALL_VA_GR_CTX + 0x270000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB (GSP_PHYS_RMCALL_VA_GR_CTX + 0x2b0000ULL) +#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL (GSP_PHYS_RMCALL_VA_GR_CTX + 0xab0000ULL) +/* + * Golden9k: every promoted context buffer is allocated with the size + * GET_ENGINE_CONTEXT_PROPERTIES reports for its engine (the client's own + * source for those sizes is the GSP static info; the same control returns + * them on this server). Live TU104 sizes (golden9k cycle, 2026-10-05): + * FECS_EVENT 0x10000, PRIV/UNR 0x80000 each, BUNDLE_CB 0x3000, PAGEPOOL + * 0x20000, ATTRIBUTE_CB 0x7d7400, RTV_CB_GLOBAL 0x80000. The golden9j + * hardcoded 0x1000/1-page shapes cannot hold these (the engine's ctx-init + * copy of the register access maps is 512 KB per map; the attribute CB is + * ~8 MB), which is the leading explanation for the invariant + * INJECTED_BUNDLE_ERROR + FECS ctxsw TDR. The VAs below are spaced by the + * per-buffer caps; a reported size above its cap aborts before any promote. + */ +#define GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX 0x10000u +#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX 0x80000u +#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX 0x10000u +#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX 0x40000u +#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX 0x800000u +#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX 0x80000u +#define GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID 0x17u +#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID 0x18u +#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID 0x11u +#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID 0x12u +#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID 0x13u +#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID 0x14u +/* + * Golden9e server FBMEM allocations for the small promoted buffers (PATCH is + * sized by the PATCH engine geometry query; FECS/PRIV/UNR are one 4 KB page + * each). The real golden wire carried all of them in FBMEM. Golden9j adds the + * four 3D globals (BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL): the real + * client allocated them, promoted their VAs (VA-only entries) and mapped them + * in its VAS, so the ctx-init bundle can read/write them; the ladder promoted + * the VAs with no backing at all until now. + */ +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH 0xcaf00064u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS 0xcaf00068u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV 0xcaf0006cu +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR 0xcaf00070u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE 0xcaf00074u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL 0xcaf00078u +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR 0xcaf0007cu +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV 0xcaf00080u +#define GSP_PHYS_RMCALL_GR_GLOBAL_SLOT 0x10000u +#define GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES 16u /* GR context-promote bisection modes (gsp_phys_rmcall_gr_promote). */ #define GSP_PHYS_RMCALL_GR_PROMOTE_OFF 0u @@ -514,8 +580,51 @@ bool gsp_phys_boot_requested(void) * entryCount shrinks to the number of real entries. */ #define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT 24u +/* + * Mode 25 (2026-10-05, opt-in): the real golden-client construct captured + * from the 570.144 host client (REPORT-GOLDEN-CAPTURE-DECODED), replayed on + * the golden handles in gsp_phys_rmcall_golden_replay: the captured VAS + * COPY_SERVER_RESERVED_PDES control (0x90f10106, gated by the fwclient mask + * bit 8 because the level PAs are server-internal), the captured channel + * shape (flags 0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries + * 0x20, engineType 1, host-buffer memdescs) and the single atomic 9-entry + * GPU_PROMOTE_CTX on the golden channel (MAIN id0 combined, PATCH id2 + * combined, ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined; + * sysmem physAttr 0x6 is the documented deviation). The golden VAS/channel/3D + * object stay alive and the ladder's own V6 FWCLIENT promote then follows on + * the golden-initialized GR-instance state. Requires gsp_phys_rmcall_golden=1 + * plus gsp_phys_rmcall_engine=1; a failed golden step aborts the ladder with + * no retry and no follow-up promote. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 25u +/* + * Existing fwclient mask bit 8: mode-25 gate for the captured VAS + * COPY_SERVER_RESERVED_PDES control. The captured control copies three + * server-RM-internal page-table levels the ladder cannot source safely, so + * mode 25 sends only the zero-level no-op-range form and only when this bit + * is set; the default mask 0x3f leaves it off (skipped, logged). + */ +#define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES (1u << 8) +/* + * Golden9i: bind the golden channel to the ladder's SHARED_MANAGEMENT VAS + * (gsp_phys_rmcall_vas_shared=1) instead of the PTETABLE_HEAP_MANAGED golden + * VAS, and pre-install the page-table window on it with DMA_UPDATE_PDE_2 + * before the golden promote. Bit 9 selects this path; bits 8+9 together are + * invalid (the golden VAS controls are skipped when bit 9 is set). + */ +#define GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS (1u << 9) +/* + * Golden9k: allocate and promote the golden context buffers at the sizes + * GET_ENGINE_CONTEXT_PROPERTIES reports (FECS_EVENT/PRIV_ACCESS_MAP/ + * UNRESTRICTED_PRIV_ACCESS_MAP/BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL) + * instead of the golden9j/j one-page placeholder sizes, and back every + * promoted VA with PTEs covering the full size across as many 2 MB windows + * as needed. Bit 10 selects this path; the golden9i/j shapes stay available + * for comparison. + */ +#define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS (1u << 10) #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) + (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) /* * Work-submit token notifier slot. The default index is @@ -757,6 +866,18 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL 0xcaf00054u #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ 0xcaf00058u #define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER 0xcaf0005cu +/* + * Mode-25 golden9d: one FBMEM block sliced into the golden channel's + * instance/USERD/RAMFC/mthd memdescs (the real capture used client-local + * FBMEM for all four; host sysmem made the engine's instance/RAMFC fetch + * hang during the ctxsw wait). + */ +#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM 0xcaf00060u +#define GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE 0x4000u +#define GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE 0x0000u +#define GSP_PHYS_RMCALL_GOLDEN_FB_USERD 0x1000u +#define GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC 0x2000u +#define GSP_PHYS_RMCALL_GOLDEN_FB_MTHD 0x3000u /* * Golden-image replay sizes (kernel_graphics.c:2058-2060, 2273-2276): the @@ -793,6 +914,18 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31) /* + * Golden-image channel flag word captured from the real 570.144 host client + * (0x00200320): PRIVILEGED_CHANNEL (bit 5), USERD_INDEX_VALUE=3 (bits 10:8, + * the client's chId % 8) and USERD_INDEX_PAGE_FIXED (bit 21). SKIP_SCRUBBER + * (bit 23) and SKIP_CTXBUFFER_ALLOC (bit 31) are deliberately clear: the real + * golden channel is client-managed through the registry, not through the + * channel flags. + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9 \ + (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (3u << 8) | \ + (1u << 21)) + +/* * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc * RPC never replied, every later RPC timed out, and the fail-safe held the @@ -844,6 +977,15 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, + /* + * Golden9g: the three client page-table levels the real golden VAS + * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES + * (0x90f10106): PD2 (0x2f, size 0x20), PD1 (0x26) and PD0 (0x1d), + * chained 0 -> 1 -> 2 with the proven GP10X sysmem PDEs. + */ + GSP_PHYS_RMCALL_BUF_GOLDEN_PT0, + GSP_PHYS_RMCALL_BUF_GOLDEN_PT1, + GSP_PHYS_RMCALL_BUF_GOLDEN_PT2, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -1426,6 +1568,8 @@ static struct gsp_phys_boot_stage { bool rmcall_ctx_valid; /* mode 11: the context buffer is a server-owned FBMEM allocation */ bool rmcall_ctx_fbmem; + /* mode 25 golden9e: PATCH/FECS/PRIV/UNR are server-owned FBMEM too */ + bool rmcall_golden_fb_aux; u64 rmcall_ctx_pa; /* context buffer DMA base */ u64 rmcall_ctx_va; /* promoted GPU VA window base */ u32 rmcall_ctx_size; /* promoted total size */ @@ -1487,6 +1631,19 @@ static struct gsp_phys_boot_stage { u64 rmcall_fecs_pa; u64 rmcall_priv_pa; u64 rmcall_unrestricted_pa; + /* golden9j: FBMEM backing for the four promoted 3D globals */ + u64 rmcall_bundle_pa; + u64 rmcall_pagepool_pa; + u64 rmcall_attr_pa; + u64 rmcall_rtv_pa; + u32 rmcall_bundle_size; + u32 rmcall_pagepool_size; + u32 rmcall_attr_size; + u32 rmcall_rtv_size; + /* golden9k: sizes queried from GET_ENGINE_CONTEXT_PROPERTIES */ + u32 rmcall_fecs_size; + u32 rmcall_priv_size; + u32 rmcall_unr_size; /* * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3 * gsp_phys_rmcall_gr_promote value, the number of accepted promote @@ -2118,6 +2275,14 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_ctx_prepared = false; boot_stage.rmcall_ctx_valid = false; boot_stage.rmcall_ctx_fbmem = false; + boot_stage.rmcall_golden_fb_aux = false; + boot_stage.rmcall_fecs_size = 0; + boot_stage.rmcall_priv_size = 0; + boot_stage.rmcall_unr_size = 0; + boot_stage.rmcall_bundle_size = 0; + boot_stage.rmcall_pagepool_size = 0; + boot_stage.rmcall_attr_size = 0; + boot_stage.rmcall_rtv_size = 0; boot_stage.rmcall_ctx_pa = 0; boot_stage.rmcall_ctx_va = 0; boot_stage.rmcall_ctx_size = 0; @@ -6173,6 +6338,31 @@ struct gsp_phys_rmcall_promote_params { }; /* + * NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS (184 bytes): hSubDevice, + * subDeviceId, pageSize, virtAddrLo, virtAddrHi, numLevelsToCopy and six level + * entries {physAddr, size, aperture, pageShift}. The ladder has no safe source + * for the server-RM level PAs the real client copied, so the gated mode-25 + * path sends only the zero-level no-op-range form. + */ +struct gsp_phys_rmcall_vas_copy_pdes_level { + __le64 phys_addr; + __le64 size; + __le32 aperture; + __le32 page_shift; +}; + +struct gsp_phys_rmcall_vas_copy_pdes_params { + __le32 h_sub_device; + __le32 sub_device_id; + __le64 page_size; + __le64 virt_addr_lo; + __le64 virt_addr_hi; + __le32 num_levels_to_copy; + __le32 pad; + struct gsp_phys_rmcall_vas_copy_pdes_level levels[6]; +}; + +/* * NV2080_CTRL_GPU_EVICT_CTX_PARAMS (20 bytes, * ctrl/ctrl2080/ctrl2080gpu.h:1000-1006): engineType/hClient/ChID/ * hChanClient/hObject. Evicts the channel's virtual engine context; on the @@ -6538,6 +6728,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, u32 *out_rpc_result, u32 *out_status, const char *what); static void gsp_phys_rmcall_pdb_build(void); +static u64 gsp_phys_rmcall_pde(u64 pa); /* * Shared-management VAS: install the ladder's page-table window through the @@ -7076,6 +7267,18 @@ gsp_phys_rmcall_mem_desc(u32 index) return d; } +static struct gsp_phys_rmcall_mem_desc +gsp_phys_rmcall_mem_desc_fb(u64 base, u64 size) +{ + struct gsp_phys_rmcall_mem_desc d; + + d.base = cpu_to_le64(base); + d.size = cpu_to_le64(size); + d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM); + d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); + return d; +} + /* * Channel-alloc variants tried in table order after the TSG exists. V5 * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated @@ -7241,6 +7444,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote"; case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT: return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6"; + case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9: + return "real golden-client construct on the golden handles (captured channel 0x00200320 + single atomic 9-entry PROMOTE_CTX) then the V6 FWCLIENT follow-on promote"; default: return "off (PROMOTE_CTX never sent)"; } @@ -7248,7 +7453,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) /* * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a - * load-time module parameter (0..13, see gsp_phys_gpu.c); stage 3 snapshots it + * load-time module parameter (0..GSP_PHYS_RMCALL_GR_PROMOTE_MAX, see + * gsp_phys_gpu.c); stage 3 snapshots it * next to the engine so every later step of this staging uses one shape. The * module parameter's set callback already clamps out-of-range values with a * warning; the check here is a second line of defense against a future @@ -7424,7 +7630,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)) { + GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) { vars = &gsp_phys_rmcall_channel_variant_skipctx; attempts = 1; } @@ -7668,6 +7876,142 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle, step, handle, h_class, status, rpc, ret); } +/* + * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8. + * golden9g: the real client's VAS construction (gvaspaceReserveSplitVaSpace -> + * _gvaspaceReserveVaForClientRm) allocates the three client page-table levels + * for the server range [4GB, 4.5GB] and hands their PAs to the server with + * this control; the server mirrors them with mmuWalkModifyLevelInstance. The + * ladder now fabricates the identical chain in host sysmem (pageShifts + * 0x2f/0x26/0x1d, sizes 0x20/0x1000/0x1000, aperture SYS_COH) with the proven + * GP10X sysmem PDEs, so the golden VAS has the same server-side walker state + * as the real client's. Set gsp_phys_rmcall_fwclient_mask bit 8 (0x100) to + * send it; without the bit the control is skipped and the deviation logged. + */ +static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0) +{ + struct gsp_phys_rmcall_vas_copy_pdes_params *p; + u64 pa0, pa1, pa2; + u32 rpc_result = 0, status = 0; + int ret; + + if (!(gsp_phys_rmcall_fwclient_mask_value() & + GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) { + pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped (set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the fabricated three-level chain)\n", + GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); + return 0; + } + + /* Build the client page-table chain: PT0(PD2[0]) -> PT1(PD1[0]) -> PT2(PD0). */ + gsp_phys_chunk_zero( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0]); + gsp_phys_chunk_zero( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1]); + gsp_phys_chunk_zero( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2]); + pa0 = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0); + pa1 = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0); + pa2 = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2], 0); + put_unaligned_le64(gsp_phys_rmcall_pde(pa1), + (u8 *)gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0)); + put_unaligned_le64(gsp_phys_rmcall_pde(pa2), + (u8 *)gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0)); + + p = kvzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + p->page_size = cpu_to_le64(1ULL << 29); /* GMMU_PD1_VADDR_BIT_LO */ + p->virt_addr_lo = cpu_to_le64(0x100000000ULL); /* SPLIT_VAS_SERVER_RM_MANAGED_VA_START */ + p->virt_addr_hi = cpu_to_le64(0x11fffffffULL); + p->num_levels_to_copy = cpu_to_le32(3); + p->levels[0].phys_addr = cpu_to_le64(pa0); + p->levels[0].size = cpu_to_le64(0x20); + p->levels[0].aperture = cpu_to_le32(4); /* GMMU_APERTURE_SYS_COH */ + p->levels[0].page_shift = cpu_to_le32(0x2f); + p->levels[1].phys_addr = cpu_to_le64(pa1); + p->levels[1].size = cpu_to_le64(0x1000); + p->levels[1].aperture = cpu_to_le32(4); + p->levels[1].page_shift = cpu_to_le32(0x26); + p->levels[2].phys_addr = cpu_to_le64(pa2); + p->levels[2].size = cpu_to_le64(0x1000); + p->levels[2].aperture = cpu_to_le32(4); + p->levels[2].page_shift = cpu_to_le32(0x1d); + /* The captured RPC targets the VAS itself (hObject=0xbaba0042). */ + ret = gsp_phys_rmcall_control(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS, + GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES, + p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, + "VASPACE_COPY_SERVER_RESERVED_PDES (3 fabricated levels)"); + pr_info("vgsp-boot: rm-call: GOLDEN9G VAS COPY_SERVER_RESERVED_PDES: numLevels=3 pageSize=0x20000000 range=[0x%llx,0x%llx] levels={0x%llx/0x20/0x2f, 0x%llx/0x1000/0x26, 0x%llx/0x1000/0x1d} status=0x%x rpc_result=0x%x ret=%d\n", + le64_to_cpu(p->virt_addr_lo), le64_to_cpu(p->virt_addr_hi), + pa0, pa1, pa2, status, rpc_result, ret); + kvfree(p); + return ret; +} + +/* + * golden9g: install the golden VAS context window (VA_GR_CTX >> 21) in the + * server's own MMU walker via NV0080_CTRL_CMD_DMA_UPDATE_PDE_2, the supported + * client map path for a non-external VAS. The PT pages hold the golden9e + * VIDMEM leaf PTEs (built by gsp_phys_rmcall_pdb_build) for the FBMEM + * MAIN/PATCH/FECS/PRIV/UNR buffers, so the GR engine can fetch the promoted + * context VAs through the golden VAS exactly as it did through the ladder + * VAS's raw PDB. Gated by fwclient mask bit 8 together with the copy control. + */ +static int gsp_phys_rmcall_golden9_vas_windows(void __iomem *bar0) +{ + struct gsp_phys_rmcall_update_pde2_params p; + u32 i; + int ret; + + if (!(gsp_phys_rmcall_fwclient_mask_value() & + GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) + return 0; + if (!boot_stage.rmcall_ctx_pt_pages) { + pr_err("vgsp-boot: rm-call: GOLDEN9G ctx window install: no ctx PT pages built; aborting the golden replay with no retry\n"); + return -EINVAL; + } + for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) { + u32 ctx_index = (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + i); + u64 ctx_pt_pa = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_ctx_pt[i], 0); + u32 rpc_result = 0, status = 0; + + memset(&p, 0, sizeof(p)); + p.pde_index = cpu_to_le32(ctx_index); + p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); + p.pt[0].phys_addr = cpu_to_le64(ctx_pt_pa); + p.pt[0].aperture = + cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); + p.pt[1].aperture = + cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); + p.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS); + p.sub_device_id = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2, + &p, sizeof(p), sizeof(p), &p, sizeof(p), + &rpc_result, &status, + "DMA_UPDATE_PDE_2 (golden9g golden VAS ctx window)"); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window %u rejected: status=0x%x rpc_result=0x%x ret=%d; aborting the golden replay with no retry\n", + ctx_index, status, rpc_result, ret); + return ret; + } + pr_info("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window accepted: pdeIndex=%u ptPa=0x%llx hVas=0x%x status=0x%x\n", + ctx_index, ctx_pt_pa, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS, status); + } + return 0; +} + +static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0); + static int gsp_phys_rmcall_golden_mem(void __iomem *bar0, const struct gsp_phys_rmcall_mem_obj_shape *shape, const char *step) @@ -7693,6 +8037,7 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) struct gsp_phys_rmcall_channel_params ch; struct gsp_phys_rmcall_mem_obj_shape shape; u32 rpc = 0, status = 0; + u64 golden_fb = 0; bool pb_ok = true; int ret; @@ -7718,6 +8063,52 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) return ret; } + /* + * Mode 25 / golden9i: bind the golden channel to the ladder's + * SHARED_MANAGEMENT VAS and pre-install the ctx window in its server + * walker (DMA_UPDATE_PDE_2), the supported map path for a non-external + * VAS. Otherwise golden9g sends the captured VAS construct control to + * the PTETABLE_HEAP_MANAGED golden VAS and skips the walker install + * (refused 0x1f on a client-managed VAS). + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + gsp_phys_rmcall_engine_is_gr()) { + if (gsp_phys_rmcall_fwclient_mask_value() & + GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) { + if (!boot_stage.rmcall_vas || + gsp_phys_rmcall_vas_shared_value() == 0) { + pr_err("vgsp-boot: rm-call: GOLDEN9I needs the ladder SHARED_MANAGEMENT VAS (rmcall_vas=0x%x shared=%u); aborting the golden replay with no retry\n", + boot_stage.rmcall_vas, + gsp_phys_rmcall_vas_shared_value()); + return -EINVAL; + } + ret = gsp_phys_rmcall_map_pt_window(bar0); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9I shared-VAS ctx window install failed (%d); aborting the golden replay with no retry\n", + ret); + return ret; + } + } else { + ret = gsp_phys_rmcall_golden9_vas_control(bar0); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n", + ret); + return ret; + } + /* + * The walker install is attempted but non-fatal (0x1f on + * the client-managed golden VAS): the channel-alloc datum + * is still collected and the ladder aborts later if the + * promote cannot proceed. + */ + ret = gsp_phys_rmcall_golden9_vas_windows(bar0); + if (ret) + pr_warn("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); continuing without the walker mapping (the golden VAS controls are non-fatal here)\n", + ret); + } + } + /* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */ memset(&shape, 0, sizeof(shape)); shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS; @@ -7774,6 +8165,66 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) ch.h_userd_memory[0] = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD); ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + /* + * Mode 25: the captured golden channel shape. The real client's + * instance/USERD/RAMFC/mthd blocks were client-local memmgr + * allocations (FBMEM in the capture); the ladder substitutes the + * registered host sysmem chunks and documents the deviation. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + gsp_phys_rmcall_engine_is_gr()) { + ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9); + ch.internal_flags = cpu_to_le32( + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL); + /* + * Golden9f/9i VAS binding: the captured shape runs on the + * PTETABLE_HEAP_MANAGED golden VAS; golden9i (mask bit 9) + * instead binds to the ladder's SHARED_MANAGEMENT VAS whose + * server walker takes the ctx-window DMA_UPDATE_PDE_2 install + * (the client-managed golden VAS refuses it 0x1f). The + * IS_EXTERNALLY_OWNED ladder default is the golden9c shape and + * is not used by either golden9 mode. + */ + if (gsp_phys_rmcall_fwclient_mask_value() & + GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) + ch.h_va_space = + cpu_to_le32(boot_stage.rmcall_vas); + else + ch.h_va_space = + cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS); + ch.h_object_buffer = cpu_to_le32(0); + /* + * The captured golden wire carried client-local FBMEM + * instance/USERD/RAMFC/mthd memdescs; the earlier host-sysmem + * substitution made the engine's instance/RAMFC fetch hang in + * the commit's ctxsw wait (FECS TDRs + fatal assert). Allocate + * one FBMEM block and slice it like the client memmgr would. + * hUserdMemory[0] stays 0 (the real wire's handle is + * client-local and unavailable here). + */ + ch.h_userd_memory[0] = cpu_to_le32(0); + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM, + GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, &golden_fb, + "GOLDEN channel memdescs"); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9 channel FBMEM memdesc block (0x%x bytes) allocation failed: %d; the golden replay aborts with no retry\n", + GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, ret); + return ret; + } + ch.instance_mem = gsp_phys_rmcall_mem_desc_fb( + golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE, 0x1000); + ch.userd_mem = gsp_phys_rmcall_mem_desc_fb( + golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_USERD, 0x200); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb( + golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb( + golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000); + pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n", + GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9, + GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb); + } ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, @@ -7793,6 +8244,23 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) return ret; } + /* + * Mode 25: the single atomic 9-entry GPU_PROMOTE_CTX on the golden + * channel, in the captured order/forms. It runs after the channel + * alloc and before the 3D object, matching the wire order of the real + * client (its GR-object constructor promotes before it allocates the + * object). A failure aborts the whole golden replay with no retry and + * no follow-up promote; the golden VAS/channel/3D objects stay alive + * for the ladder's own V6 FWCLIENT follow-on promote. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + gsp_phys_rmcall_engine_is_gr()) { + ret = gsp_phys_rmcall_golden9_promote(bar0); + if (ret) + return ret; + } + /* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */ ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, GSP_PHYS_RMCALL_CLASS_GR_3D, @@ -7854,6 +8322,36 @@ static u64 gsp_phys_rmcall_pte_vid(u64 fb_offset) } /* + * Golden9k: map [va, va+size) with VIDMEM PTEs into the ctx small-PT pages. + * One rmcall_ctx_pt page covers one 2 MB window (windows start at + * GSP_PHYS_RMCALL_VA_GR_CTX >> 21); a range may span windows, and the PT + * pages for every window it touches must already exist (gr_ctx_prepare plus + * the golden9k extension in golden9_fb_aux_prepare). Every slot written is + * distinct from MAIN's and from the other aux buffers (the caller places + * them on 1 MB slots). + */ +static void gsp_phys_rmcall_golden9_map_range(u64 va, u64 pa, u32 size) +{ + u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21); + u32 pages = DIV_ROUND_UP(size, GSP_PHYS_PAGE_SIZE); + u32 i; + + for (i = 0; i < pages; i++) { + u64 v = va + (u64)i * GSP_PHYS_PAGE_SIZE; + u32 win = (u32)(v >> 21) - first_win; + u32 slot = (u32)((v >> 12) & 0x1ffu); + u8 *ptv; + + if (win >= boot_stage.rmcall_ctx_pt_pages) + break; + ptv = gsp_phys_chunk_page_va(&boot_stage.rmcall_ctx_pt[win], 0); + put_unaligned_le64(gsp_phys_rmcall_pte_vid(pa + + (u64)i * GSP_PHYS_PAGE_SIZE), + ptv + (size_t)slot * 8); + } +} + +/* * Build the channel's page-directory hierarchy around the 2 MB VA window * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's * src/dst/semaphore sysmem pages: @@ -8052,6 +8550,73 @@ static void gsp_phys_rmcall_pdb_build(void) &boot_stage.rmcall_ctx_pt[pt_page], 0) + (size_t)pt_slot * 8); } + /* + * Golden9e/j/k: map the FBMEM PATCH/FECS/PRIV/UNR and the four + * 3D global buffers at their promoted VAs with VIDMEM PTEs. + * Every buffer sits on its own 1 MB slot, so the ranges never + * overlap and can span 2 MB windows (the helper picks the PT + * page per page). + */ + if (boot_stage.rmcall_golden_fb_aux) { + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GR_PATCH, + boot_stage.rmcall_patch_pa, + boot_stage.rmcall_patch_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, + boot_stage.rmcall_fecs_pa, + boot_stage.rmcall_fecs_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, + boot_stage.rmcall_priv_pa, + boot_stage.rmcall_priv_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_UNR, + boot_stage.rmcall_unrestricted_pa, + boot_stage.rmcall_unr_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, + boot_stage.rmcall_bundle_pa, + boot_stage.rmcall_bundle_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, + boot_stage.rmcall_pagepool_pa, + boot_stage.rmcall_pagepool_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, + boot_stage.rmcall_attr_pa, + boot_stage.rmcall_attr_size); + gsp_phys_rmcall_golden9_map_range( + GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, + boot_stage.rmcall_rtv_pa, + boot_stage.rmcall_rtv_size); + pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx size=0x%x (pa=0x%llx) fecs va=0x%llx size=0x%x (pa=0x%llx) priv va=0x%llx size=0x%x (pa=0x%llx) unr va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs)\n", + GSP_PHYS_RMCALL_VA_GR_PATCH, + boot_stage.rmcall_patch_size, + boot_stage.rmcall_patch_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, + boot_stage.rmcall_fecs_size, + boot_stage.rmcall_fecs_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, + boot_stage.rmcall_priv_size, + boot_stage.rmcall_priv_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_UNR, + boot_stage.rmcall_unr_size, + boot_stage.rmcall_unrestricted_pa); + pr_info("vgsp-boot: rm-call: GOLDEN9J globals mapped: bundle va=0x%llx size=0x%x (pa=0x%llx) pagepool va=0x%llx size=0x%x (pa=0x%llx) attr va=0x%llx size=0x%x (pa=0x%llx) rtv va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs, full-size coverage)\n", + GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, + boot_stage.rmcall_bundle_size, + boot_stage.rmcall_bundle_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, + boot_stage.rmcall_pagepool_size, + boot_stage.rmcall_pagepool_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, + boot_stage.rmcall_attr_size, + boot_stage.rmcall_attr_pa, + GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, + boot_stage.rmcall_rtv_size, + boot_stage.rmcall_rtv_pa); + } pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n", GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size, boot_stage.rmcall_ctx_pages, @@ -8110,6 +8675,302 @@ static void gsp_phys_rmcall_pdb_build(void) } /* + * Golden9k: query the size/alignment GET_ENGINE_CONTEXT_PROPERTIES reports + * for one engine context buffer (the same control the MAIN/PATCH geometry + * uses; the server derives the values from the static info the stock client + * caches at GSP boot). Returns 0 with size/align set, the RPC error on + * failure, -EINVAL on a zero size (buffer unsupported on this engine). + */ +static int gsp_phys_rmcall_engine_ctx_size(void __iomem *bar0, u32 engine_id, + const char *name, u32 *size, + u32 *align) +{ + struct gsp_phys_rmcall_engine_ctx_props props; + u32 rpc_result = 0, status = 0; + int ret; + + memset(&props, 0, sizeof(props)); + props.engine_id = cpu_to_le32(engine_id); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES, + &props, sizeof(props), 24 + sizeof(props), + &props, sizeof(props), &rpc_result, &status, + "GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9k)"); + *size = le32_to_cpu(props.size); + *align = le32_to_cpu(props.alignment); + pr_info("vgsp-boot: rm-call: GOLDEN9K %s geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n", + name, engine_id, *size, *align, status, rpc_result); + if (ret) + return ret; + if (!*size) { + pr_err("vgsp-boot: rm-call: GOLDEN9K %s reports size 0 (unsupported); aborting with no promote and no retry\n", + name); + return -EINVAL; + } + return 0; +} + +/* + * Mode-25 golden9e: allocate the promoted PATCH/FECS/PRIV/UNR context + * buffers in server FBMEM (the real golden wire carried FBMEM for every + * context buffer; the earlier host-sysmem substitution is a documented + * deviation that leaves the engine's context load reading host memory). + * PATCH is sized by the PATCH engine geometry query (engineId 0x10); the + * three global buffers are one 4 KB page each. Runs before + * gsp_phys_rmcall_pdb_build() so the PDB can map every promoted VA in the + * same 2 MB window as MAIN. Idempotent; a failure aborts the ladder. + */ +static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0) +{ + struct gsp_phys_rmcall_engine_ctx_props props; + u32 raw, rpc_result = 0, status = 0; + u32 i; + u64 fb = 0; + int ret; + + if (boot_stage.rmcall_golden_fb_aux) + return 0; + + memset(&props, 0, sizeof(props)); + props.engine_id = cpu_to_le32(GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES, + &props, sizeof(props), 24 + sizeof(props), + &props, sizeof(props), &rpc_result, &status, + "GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9e PATCH)"); + raw = le32_to_cpu(props.size); + pr_info("vgsp-boot: rm-call: GOLDEN9E PATCH geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n", + GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID, raw, + le32_to_cpu(props.alignment), status, rpc_result); + if (ret) + return ret; + if (!raw || raw > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) { + pr_err("vgsp-boot: rm-call: GOLDEN9E PATCH size 0x%x is outside the sanity bound; aborting the ladder with no retry\n", + raw); + return -E2BIG; + } + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH, raw, &fb, + "GOLDEN9E PATCH"); + if (ret) + return ret; + boot_stage.rmcall_patch_pa = fb; + boot_stage.rmcall_patch_size = raw; + boot_stage.rmcall_patch_prepared = true; + + /* + * Golden9k (fwclient-mask bit 10): size every promoted buffer from the + * engine-context-properties query. The bInitialize=1 entries (FECS_EVENT/ + * PRIV/UNR) are written by the server's init copy, and the VA-only + * globals are written by the ctxsw bundle through the page tables; the + * golden9j one-page shapes cannot hold either (the Turing private and + * unrestricted register access maps are 512 KB each). The stock client's + * sizes come from the GSP static info; this control reports the same + * values. A size above its per-buffer VA cap aborts before any promote. + */ + if (gsp_phys_rmcall_fwclient_mask_value() & + GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS) { + u32 align; + + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID, + "FECS_EVENT", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_fecs_size = raw; + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID, + "PRIV_ACCESS_MAP", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_priv_size = raw; + boot_stage.rmcall_unr_size = raw; + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID, + "BUNDLE_CB", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_bundle_size = raw; + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID, + "PAGEPOOL", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_pagepool_size = raw; + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID, + "ATTRIBUTE_CB", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_attr_size = raw; + ret = gsp_phys_rmcall_engine_ctx_size(bar0, + GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID, + "RTV_CB_GLOBAL", &raw, &align); + if (ret) + return ret; + boot_stage.rmcall_rtv_size = raw; + if (boot_stage.rmcall_fecs_size > + GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX || + boot_stage.rmcall_priv_size > + GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX || + boot_stage.rmcall_bundle_size > + GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX || + boot_stage.rmcall_pagepool_size > + GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX || + boot_stage.rmcall_attr_size > + GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX || + boot_stage.rmcall_rtv_size > + GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX) { + pr_err("vgsp-boot: rm-call: GOLDEN9K a promoted buffer exceeds its per-buffer VA cap (fecs=0x%x/0x%x priv=0x%x/0x%x bundle=0x%x/0x%x pagepool=0x%x/0x%x attr=0x%x/0x%x rtv=0x%x/0x%x); aborting with no promote and no retry\n", + boot_stage.rmcall_fecs_size, + GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX, + boot_stage.rmcall_priv_size, + GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX, + boot_stage.rmcall_bundle_size, + GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX, + boot_stage.rmcall_pagepool_size, + GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX, + boot_stage.rmcall_attr_size, + GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX, + boot_stage.rmcall_rtv_size, + GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX); + return -E2BIG; + } + pr_info("vgsp-boot: rm-call: GOLDEN9K sized buffers: fecs=0x%x priv=0x%x unr=0x%x bundle=0x%x pagepool=0x%x attr=0x%x rtv=0x%x (all queried)\n", + boot_stage.rmcall_fecs_size, + boot_stage.rmcall_priv_size, + boot_stage.rmcall_unr_size, + boot_stage.rmcall_bundle_size, + boot_stage.rmcall_pagepool_size, + boot_stage.rmcall_attr_size, + boot_stage.rmcall_rtv_size); + } else { + boot_stage.rmcall_fecs_size = 0x1000; + boot_stage.rmcall_priv_size = 0x1000; + boot_stage.rmcall_unr_size = 0x1000; + boot_stage.rmcall_bundle_size = 0x1000; + boot_stage.rmcall_pagepool_size = 0x1000; + boot_stage.rmcall_attr_size = 0x1000; + boot_stage.rmcall_rtv_size = 0x1000; + } + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS, + boot_stage.rmcall_fecs_size, &fb, + "GOLDEN9E FECS_EVENT"); + if (ret) + return ret; + boot_stage.rmcall_fecs_pa = fb; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV, + boot_stage.rmcall_priv_size, &fb, + "GOLDEN9E PRIV_ACCESS_MAP"); + if (ret) + return ret; + boot_stage.rmcall_priv_pa = fb; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR, + boot_stage.rmcall_unr_size, &fb, + "GOLDEN9E UNR_PRIV_ACCESS_MAP"); + if (ret) + return ret; + boot_stage.rmcall_unrestricted_pa = fb; + + /* + * Golden9j: the four 3D globals. The real client allocated each one + * with the size its own geometry query returned and mapped it in the + * channel VAS; the promote carries the VAs (bInitialize=0, no PA), so + * the ctx-init bundle reads and writes them through the page tables. + * Golden9k sizes them from the same query as the rest; golden9j kept + * them at one page each. + */ + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE, + boot_stage.rmcall_bundle_size, &fb, + "GOLDEN9J BUNDLE_CB"); + if (ret) + return ret; + boot_stage.rmcall_bundle_pa = fb; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL, + boot_stage.rmcall_pagepool_size, &fb, + "GOLDEN9J PAGEPOOL"); + if (ret) + return ret; + boot_stage.rmcall_pagepool_pa = fb; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR, + boot_stage.rmcall_attr_size, &fb, + "GOLDEN9J ATTRIBUTE_CB"); + if (ret) + return ret; + boot_stage.rmcall_attr_pa = fb; + + ret = gsp_phys_rmcall_alloc_fb_mem(bar0, + GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV, + boot_stage.rmcall_rtv_size, &fb, + "GOLDEN9J RTV_CB_GLOBAL"); + if (ret) + return ret; + boot_stage.rmcall_rtv_pa = fb; + + /* + * Golden9k: the sized layout spans as many 2 MB windows as the highest + * promoted VA needs; allocate the extra small-PT pages now, before the + * PDB build and the DMA_UPDATE_PDE_2 window installs consume + * rmcall_ctx_pt_pages. MAIN's pages were already sized/allocated. + */ + { + u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21); + u32 last_win = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL + + boot_stage.rmcall_rtv_size - 1) >> 21); + u32 need = last_win - first_win + 1; + + if (need > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) { + pr_err("vgsp-boot: rm-call: GOLDEN9K sized layout needs %u PT pages > max %u; aborting with no promote and no retry\n", + need, GSP_PHYS_RMCALL_GR_CTX_PT_MAX); + return -E2BIG; + } + for (i = boot_stage.rmcall_ctx_pt_pages; i < need; i++) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_ctx_pt[i], + 1u); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9K ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n", + i, need, ret); + return ret; + } + gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]); + } + if (need > boot_stage.rmcall_ctx_pt_pages) { + pr_info("vgsp-boot: rm-call: GOLDEN9K ctx PT pages extended %u -> %u for the sized layout (last VA 0x%llx + 0x%x)\n", + boot_stage.rmcall_ctx_pt_pages, need, + GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, + boot_stage.rmcall_rtv_size); + boot_stage.rmcall_ctx_pt_pages = need; + } + } + + boot_stage.rmcall_golden_fb_aux = true; + pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx size=0x%x priv=0x%llx size=0x%x unr=0x%llx size=0x%x (server FBMEM, all mapped in the ctx PDB windows)\n", + boot_stage.rmcall_patch_pa, boot_stage.rmcall_patch_size, + boot_stage.rmcall_fecs_pa, boot_stage.rmcall_fecs_size, + boot_stage.rmcall_priv_pa, boot_stage.rmcall_priv_size, + boot_stage.rmcall_unrestricted_pa, + boot_stage.rmcall_unr_size); + pr_info("vgsp-boot: rm-call: GOLDEN9J global buffers ready: bundle=0x%llx size=0x%x pagepool=0x%llx size=0x%x attr=0x%llx size=0x%x rtv=0x%llx size=0x%x (server FBMEM, PTEs cover the full size)\n", + boot_stage.rmcall_bundle_pa, boot_stage.rmcall_bundle_size, + boot_stage.rmcall_pagepool_pa, boot_stage.rmcall_pagepool_size, + boot_stage.rmcall_attr_pa, boot_stage.rmcall_attr_size, + boot_stage.rmcall_rtv_pa, boot_stage.rmcall_rtv_size); + return 0; +} + +/* * H1 GR context bind, step 1-2: query the engine-context geometry, then * allocate the zeroed, physically contiguous sysmem context buffer the * promote installs. GR0 only (the caller gates); any failure aborts the @@ -8240,7 +9101,9 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) { + GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) { u64 fb_base = 0; ret = gsp_phys_rmcall_alloc_fb_mem(bar0, @@ -8292,6 +9155,19 @@ static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) boot_stage.rmcall_ctx_fbmem ? "server FBMEM, VIDMEM aperture PTEs" : "host sysmem, contiguous, zeroed"); + /* + * Mode-25 golden9e: prepare the FBMEM PATCH/FECS/PRIV/UNR buffers now, + * before the PDB build maps their promoted VAs. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) { + ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0); + if (ret) { + pr_err("vgsp-boot: rm-call: GOLDEN9E FBMEM aux prepare failed (%d); aborting the ladder with no retry\n", + ret); + return ret; + } + } return 0; err_free: @@ -8543,6 +9419,132 @@ static void gsp_phys_rmcall_gr_fwclient_small_prepare(void) } /* + * Mode-25 golden 9-entry promote (0x2080012b): ONE atomic control on the + * golden channel with the captured entry order/forms - MAIN id0 combined, + * PATCH id2 combined, ids 3/4/5/6 VA-only globals (BUNDLE_CB, PAGEPOOL, + * ATTRIBUTE_CB, RTV_CB_GLOBAL) and ids 9/10/11 (FECS_EVENT, PRIV_ACCESS_MAP, + * UNRESTRICTED_PRIV_ACCESS_MAP) combined. The ladder's prepared sysmem + * context buffers back the combined entries (physAttr 0x6, the documented + * deviation from the capture's VIDMEM 0x4); the VA-only globals keep + * booking-only VAs above the context window. All nine buffers must be + * present; a short set or a rejected control aborts the whole replay with no + * retry and never a partial accept followed by another promote. + */ +static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0) +{ + struct gsp_phys_rmcall_promote_params *p; + u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR; + u32 fattr; + u32 rpc_result = 0, status = 0; + unsigned int i; + int ret; + + if (!boot_stage.rmcall_ctx_prepared || + !boot_stage.rmcall_ctx_pa || !boot_stage.rmcall_ctx_size) { + pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: no prepared MAIN context buffer (prepared=%u pa=0x%llx size=0x%x)\n", + boot_stage.rmcall_ctx_prepared, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size); + return -EINVAL; + } + /* + * Golden9e: PATCH/FECS/PRIV/UNR are server FBMEM allocations prepared + * with the MAIN (gr_ctx_prepare, before the PDB build). Idempotent. + */ + ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0); + if (ret) + return ret; + if (!boot_stage.rmcall_patch_pa || !boot_stage.rmcall_fecs_pa || + !boot_stage.rmcall_priv_pa || + !boot_stage.rmcall_unrestricted_pa) { + pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: patch=0x%llx fecs=0x%llx priv=0x%llx unr=0x%llx (all nine buffers must be present); no promote is sent and the ladder aborts with no retry\n", + boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa, + boot_stage.rmcall_priv_pa, + boot_stage.rmcall_unrestricted_pa); + return -ENOMEM; + } + fattr = boot_stage.rmcall_golden_fb_aux ? + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr; + + p = kvzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + + p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + p->h_client = cpu_to_le32(0); + p->ch_id = cpu_to_le32(0); + p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL); + p->entry_count = cpu_to_le32(9); + + gsp_phys_rmcall_promote_entry_fill(p, 0, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_fbmem ? + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr, 1, 0); + gsp_phys_rmcall_promote_entry_fill(p, 1, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH, + boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH, + boot_stage.rmcall_patch_size, fattr, 1, 0); + gsp_phys_rmcall_promote_entry_fill(p, 2, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB, + 0, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, 0, 0, 0, 0); + gsp_phys_rmcall_promote_entry_fill(p, 3, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL, + 0, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, 0, 0, 0, 0); + gsp_phys_rmcall_promote_entry_fill(p, 4, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB, + 0, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, 0, 0, 0, 0); + gsp_phys_rmcall_promote_entry_fill(p, 5, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL, + 0, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, 0, 0, 0, 0); + gsp_phys_rmcall_promote_entry_fill(p, 6, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT, + boot_stage.rmcall_fecs_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, + boot_stage.rmcall_fecs_size, fattr, 1, 0); + gsp_phys_rmcall_promote_entry_fill(p, 7, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP, + boot_stage.rmcall_priv_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, + boot_stage.rmcall_priv_size, fattr, 1, 0); + gsp_phys_rmcall_promote_entry_fill(p, 8, + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP, + boot_stage.rmcall_unrestricted_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_UNR, + boot_stage.rmcall_unr_size, fattr, 1, 0); + + for (i = 0; i < 9; i++) + pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n", + i, le16_to_cpu(p->entry[i].buffer_id), + le64_to_cpu(p->entry[i].gpu_phys_addr), + le64_to_cpu(p->entry[i].gpu_virt_addr), + le64_to_cpu(p->entry[i].size), + le32_to_cpu(p->entry[i].phys_attr), + p->entry[i].b_initialize, p->entry[i].b_nonmapped, + (i == 2 || i == 3 || i == 4 || i == 5) ? + " - VA-only booking, no PA" : ""); + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, + "GPU_PROMOTE_CTX (golden9, 9 entries, atomic)"); + pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9: entryCount=9 status=0x%x rpc_result=0x%x (hObject=golden channel 0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; gr_promote_param=%u)\n", + status, rpc_result, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, + boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_size, + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9); + if (ret) { + pr_err("vgsp-boot: rm-call: PROMOTE_CTX golden9 (9 entries) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the golden context is not committed and the ladder aborts with no retry and no follow-up promote\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + kvfree(p); + return 0; +} + +/* * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5 * entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the @@ -8676,6 +9678,11 @@ static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0) * entry in the combined kgrobjPromoteContext form where the ladder has * a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers * are omitted, all logged per entry. + * - mode 25: the same follow-on FWCLIENT promote on the ladder's V6 + * channel, after gsp_phys_rmcall_golden_replay replayed the real golden + * construct (captured VAS control, captured channel shape and the single + * atomic 9-entry PROMOTE_CTX on the golden channel) so the server-side + * GR-instance state exists before this control runs. * Mode 0 never reaches this function (stage 3 stops before it); the early * return is a belt-and-braces no-op. Success means the server installed the * context memdesc, added the VA to the engine-context vaList, committed the @@ -8708,6 +9715,17 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF) return -EOPNOTSUPP; /* + * Mode 25's golden construct lives in the golden replay + * (gsp_phys_rmcall_golden_replay); without both gates the replay never + * ran, so the follow-on FWCLIENT promote would have no golden-initialized + * GR-instance state to consume. Abort instead of sending it blind. + */ + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) { + pr_err("vgsp-boot: rm-call: mode 25 requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n"); + return -EINVAL; + } + /* * Modes 8/9/10 must run on the V6 channel: without the * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping * route that asserted GSP-RM in modes 1/2/4 (or rejects the @@ -8726,7 +9744,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) && boot_stage.rmcall_channel_variant != gsp_phys_rmcall_channel_variant_skipctx.id) { pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", @@ -9406,7 +10425,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) return ret; } boot_stage.rmcall_gr_promote_step = 1; - } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) { + } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) { /* * Mode 24: the reference FWCLIENT non-3D set as ONE control. * Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT @@ -9423,7 +10443,9 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) * stock shape is init-only too; both deviations are logged * per entry. Auxiliary buffers that cannot be sourced are * omitted and logged; entryCount is the number of real - * entries. + * entries. Mode 25 shares this follow-on promote on the + * ladder's V6 channel after its golden construct; the + * existing mask selects its entries too (bits 0-7). */ u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR; u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL; @@ -9508,7 +10530,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, - boot_stage.rmcall_ctx_size, attr, 1, 0); + boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_fbmem ? + GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : + attr, 1, 0); gsp_phys_rmcall_gr_promote_log_entry(0, &p->entry[0], "combined (PDB-mapped ctx VA)"); @@ -10236,15 +11261,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) * replay the stock client's post-load golden channel allocations after * the ladder's VAS/TSG exist and before the ladder's own channel/GR * object, so the server-side GR-instance state the GR engine-context - * commit needs is initialized first. The first failure aborts the whole - * ladder with no retry; with the parameter 0 this block is skipped - * entirely (no RPC, no log). + * commit needs is initialized first. With mode 25 the replay carries + * the single atomic 9-entry promote and is no longer best-effort: a + * failure aborts the whole ladder with no retry and no follow-up + * promote. For every other mode the replay keeps its historical + * best-effort continuation. */ if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) { ret = gsp_phys_rmcall_golden_replay(bar0); - if (ret) + if (ret) { + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 && + gsp_phys_rmcall_engine_is_gr()) { + pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n", + ret); + return ret; + } pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n", ret); + } } /* @@ -14500,6 +15535,11 @@ static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40 "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40"); static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48, "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)"); +static_assert(sizeof(struct gsp_phys_rmcall_vas_copy_pdes_params) == 184, + "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS must be 184 bytes"); +static_assert(offsetof(struct gsp_phys_rmcall_vas_copy_pdes_params, + num_levels_to_copy) == 32, + "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.numLevelsToCopy must be at offset 32"); static_assert(sizeof(struct gsp_phys_rmcall_evict_ctx_params) == 20, "NV2080_CTRL_GPU_EVICT_CTX_PARAMS must be 20 bytes"); static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 80, @@ -14543,8 +15583,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30, - "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33, + "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 5e6b264..21313a2 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -511,8 +511,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val, return ret; if (gsp_phys_rmcall_gr_promote_param < 0 || - gsp_phys_rmcall_gr_promote_param > 24) { - pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..24; clamping to 0 (no PROMOTE_CTX)\n", + gsp_phys_rmcall_gr_promote_param > 25) { + pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..25; clamping to 0 (no PROMOTE_CTX)\n", gsp_phys_rmcall_gr_promote_param); gsp_phys_rmcall_gr_promote_param = 0; } @@ -531,7 +531,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote, &gsp_phys_rmcall_gr_promote_ops, &gsp_phys_rmcall_gr_promote_param, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote, - "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); values outside 0..24 clamp to 0 with a warning"); + "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; values outside 0..25 clamp to 0 with a warning"); int gsp_phys_rmcall_gr_promote_value(void) { diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 9e809e3..a38067a 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -250,8 +250,12 @@ int gsp_phys_rmcall_engine_value(void); * PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the * V6 KMD-managed channel: combined PA+VA entries where the ladder has * a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted - * and logged. - * The parameter's set callback clamps any value outside 0..24 to 0 with a + * and logged; + * 25 = the real golden-client construct on the golden replay handles + * (captured channel shape + single atomic 9-entry PROMOTE_CTX on the + * golden channel) followed by the V6 FWCLIENT promote; requires + * gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1. + * The parameter's set callback clamps any value outside 0..25 to 0 with a * warning and warns on every nonzero enable (a failed promote can assert * GSP-RM and wedge the device; recovery needs a reboot with human approval). * The booter snapshots the value when the rm-call ladder's stage 3 starts |