From 09f241601423b1ae98e97846ee451768ff8a0a9e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 10:58:17 -0400 Subject: gsp-phys: GR engine context allocate/map/promote before the GR sem push (default off; live-unverified at promote) --- kernel/gsp_phys_booter.c | 812 ++++++++++++++++++++++++++++++++++++++++++++++- kernel/gsp_phys_booter.h | 14 +- kernel/gsp_phys_gpu.c | 56 ++++ kernel/gsp_phys_gpu.h | 23 ++ 4 files changed, 897 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 2199cae..9232961 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -254,6 +254,78 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_GR_PARAM_VERSION 0x2u #define GSP_PHYS_RMCALL_GR_PARAM_SIZE 0x10u +/* + * H1 GR context bind (gsp_phys_rmcall_engine=1 only; the CTXNOTVALID fix). + * The ladder's VAS carries the externally-owned flag, so the server does not + * commit the GR engine context on its own: the GR push then faults with PBDMA + * intr1=0x80000000 (CTXNOTVALID) because RAMFC ENG_CTX_VALID stays 0. The + * client must issue a GPU_PROMOTE_CTX carrying one MAIN entry with both the + * client buffer's physical address/size (bInitialize=1) and the VA mapped in + * the client's own VAS (bNonmapped=0); the server installs the context + * memdesc, adds the VA to the engine-context vaList, commits the context + * (ENG_CTX_VALID=1) and runs the GR golden-image bind. + * + * - NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES (device, engineId 0 + * = GR MAIN) returns the context save-region size/alignment + * (ctrl/ctrl0080/ctrl0080fifo.h:118-154); + * - NV2080_CTRL_CMD_FIFO_GET_INFO (subdevice) index 3 = + * MAX_SUBCONTEXT_PER_GROUP returns the per-group subcontext count + * (ctrl/ctrl2080/ctrl2080fifo.h:221-277; the reference UVM path calls + * exactly this query, nv_gpu_ops.c:2438-2455); + * - NV2080_CTRL_CMD_GPU_PROMOTE_CTX 0x2080012b is the 560-byte + * NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS + * (ctrl/ctrl2080/ctrl2080gpu.h:867-972; wire twin + * NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS_v1A_20, g_sdk-structures.h:86-113); + * - NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_INFO 0x20801219 is the best-effort + * readback (ctrl/ctrl2080/ctrl2080gr.h:1033-1112), called like the + * reference kernel client (nv_gpu_ops.c:10060-10163). + * + * physAttr 0x6 = APERTURE_NCOH_SYS (2) | GPU_CACHEABLE_NO (1 << 2), the same + * value the RM's own external-VAS promote builds + * (kernel_falcon.c:226-254; kernel_graphics_context.c:1814-1850). The context + * VA window sits above the P6b guest scratch pool in fresh PD0 entries 17+, + * so no line the engine may have fetched is rewritten and the scratch pool is + * untouched. + * + * OBSERVED WEDGE (p6h1b, 2026-10-02): that value-1 single entry made GSP-RM + * assert immediately on submission: NOCAT rec seq=47 recType=5 + * errorCode=0x52a8b04 source="ASSERT" faultingEngine="" (a host/server-RM + * assert in the promote path, not an engine fault: no 0x1006 and no NOCAT + * faultingEngine="GR"), then no fn76 reply within the 2000 ms bound - the + * server consumed the command (its readPtr advanced) but wrote no reply. + * Teardown could not clear WPR2 (fuse-locked PLM), leaving the host in a + * locked fail-safe that needs a reboot with human approval. The promote is + * therefore opt-in and bisectable via gsp_phys_rmcall_gr_promote: + * 0 = off (never send the control; stop after the safe half); + * 1 = the p6h1b single entry (bInitialize=1/bNonmapped=0, size raw+subctx); + * 2 = reference external-VAS shape: PA-only nonmapped entry (bInitialize=1, + * bNonmapped=1, no VA) first, then a VA-only entry (bInitialize=0, + * bNonmapped=0, same PA/VA/size); + * 3 = single entry like 1 but with size = the exact raw save size. + * Every nonzero value warns at the promote site (once per staging). + */ +#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707u +#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO 0x20801109u +#define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX 0x2080012bu +#define GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO 0x20801219u +#define GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID 0u +#define GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT 3u +#define GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR 0x6u +#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN 0u +#define GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE (32u * 1024u * 1024u) +/* One 4 KB PT page covers one 2 MB PD0 slot; 32 MB max -> 16 pages max. */ +#define GSP_PHYS_RMCALL_GR_CTX_PT_MAX \ + (GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE / (2u * 1024u * 1024u)) +#define GSP_PHYS_RMCALL_VA_GR_CTX 0x02200000ULL + +/* GR context-promote bisection modes (gsp_phys_rmcall_gr_promote). */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_OFF 0u +#define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE 1u +#define GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE 2u +#define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW 3u +#define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ + (GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW) + /* * Work-submit token notifier slot. The default index is * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1 @@ -1056,6 +1128,35 @@ static struct gsp_phys_boot_stage { bool rmcall_engine_change_logged; /* mid-session change noted once */ bool rmcall_gr_ce_refused_logged; /* CE-on-GR refusal noted once */ u32 rmcall_gr_object; /* TURING_COMPUTE_A handle (0=none) */ + /* + * H1 GR context bind (stage 3, GR0 only; all zero on COPY0): + * rmcall_ctx_prepared means the geometry queries ran and the zeroed + * context buffer plus its PT pages are allocated (gsp_phys_rmcall_ + * pdb_build maps the VA window); rmcall_ctx_valid means the server + * accepted NV2080_CTRL_CMD_GPU_PROMOTE_CTX for the MAIN buffer. The + * chunks are freed by gsp_phys_rmcall_fini() on release. + */ + bool rmcall_ctx_prepared; + bool rmcall_ctx_valid; + u64 rmcall_ctx_pa; /* context buffer DMA base */ + u64 rmcall_ctx_va; /* promoted GPU VA window base */ + u32 rmcall_ctx_size; /* promoted total size */ + u32 rmcall_ctx_pages; /* 4 KB pages mapped */ + u32 rmcall_ctx_pt_pages; /* small-PT pages used */ + /* + * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3 + * gsp_phys_rmcall_gr_promote value, the number of accepted promote + * controls (0 = none, 1 = PA/init accepted, 2 = VA accepted; mode 2 + * reaches 2, modes 1/3 stop at 1) and the entry size actually sent + * (mode 3 sends the exact raw save size). A parameter change after + * the snapshot is logged once and ignored. + */ + u32 rmcall_gr_promote; + bool rmcall_gr_promote_valid; + bool rmcall_gr_promote_change_logged; + bool rmcall_gr_promote_warned; /* promote-site wedge warning logged */ + u32 rmcall_gr_promote_step; + u32 rmcall_gr_promote_size; u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ @@ -1106,6 +1207,14 @@ static struct gsp_phys_boot_stage { struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT]; + /* + * H1 GR context bind host buffers (GR0 only): one contiguous buffer and + * up to GSP_PHYS_RMCALL_GR_CTX_PT_MAX fresh small-PT pages. They are + * deliberately outside rmcall_chunks so the COPY0 ladder's stage-2 + * registration walk and count are byte-identical. + */ + struct gsp_phys_boot_chunk rmcall_ctx; + struct gsp_phys_boot_chunk rmcall_ctx_pt[GSP_PHYS_RMCALL_GR_CTX_PT_MAX]; struct gsp_phys_boot_chunk fmc; struct gsp_phys_radix3 radix3; struct gsp_phys_boot_chunk signature; @@ -1637,6 +1746,19 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_engine_change_logged = false; boot_stage.rmcall_gr_ce_refused_logged = false; boot_stage.rmcall_gr_object = 0; + boot_stage.rmcall_ctx_prepared = false; + boot_stage.rmcall_ctx_valid = false; + boot_stage.rmcall_ctx_pa = 0; + boot_stage.rmcall_ctx_va = 0; + boot_stage.rmcall_ctx_size = 0; + boot_stage.rmcall_ctx_pages = 0; + boot_stage.rmcall_ctx_pt_pages = 0; + boot_stage.rmcall_gr_promote = 0; + boot_stage.rmcall_gr_promote_valid = false; + boot_stage.rmcall_gr_promote_change_logged = false; + boot_stage.rmcall_gr_promote_warned = false; + boot_stage.rmcall_gr_promote_step = 0; + boot_stage.rmcall_gr_promote_size = 0; boot_stage.rmcall_token = 0; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; @@ -1668,6 +1790,10 @@ static void gsp_phys_rmcall_reset(void) memset(&boot_stage.rmcall_chunks[i], 0, sizeof(boot_stage.rmcall_chunks[i])); } + memset(&boot_stage.rmcall_ctx, 0, sizeof(boot_stage.rmcall_ctx)); + for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) + memset(&boot_stage.rmcall_ctx_pt[i], 0, + sizeof(boot_stage.rmcall_ctx_pt[i])); } int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, @@ -5480,6 +5606,95 @@ struct gsp_phys_rmcall_gr_alloc_params { __le32 caps; }; +/* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS (12 bytes): + * engineId in, alignment/size out (ctrl/ctrl0080/ctrl0080fifo.h:150-154). */ +struct gsp_phys_rmcall_engine_ctx_props { + __le32 engine_id; + __le32 alignment; + __le32 size; +}; + +/* + * NV2080_CTRL_FIFO_GET_INFO_PARAMS (2056 bytes): fifoInfoTblSize, the inline + * 256-entry NVXXXX_CTRL_XXX_INFO table ({index,data} each) and engineType + * (ctrl/ctrl2080/ctrl2080fifo.h:269-277; entry struct ctrlxxxx.h:71-74). The + * server echoes the whole struct back, so the request carries the full size + * the reference client passes (nv_gpu_ops.c:2438-2455). + */ +struct gsp_phys_rmcall_fifo_get_info_params { + __le32 fifo_info_tbl_size; + struct { + __le32 index; + __le32 data; + } fifo_info_tbl[256]; + __le32 engine_type; +}; + +/* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY (32 bytes), + * ctrl/ctrl2080/ctrl2080gpu.h:893-901. */ +struct gsp_phys_rmcall_promote_entry { + __le64 gpu_phys_addr; + __le64 gpu_virt_addr; + __le64 size; + __le32 phys_attr; + __le16 buffer_id; + u8 b_initialize; + u8 b_nonmapped; +}; + +/* + * NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS (560 bytes = 48-byte header + 16 x + * 32-byte entries): engineType/hClient/ChID/hChanClient/hObject/hVirtMemory, + * virtAddress/size, entryCount and the 8-byte aligned entry array + * (ctrl2080gpu.h:959-971; wire twin g_sdk-structures.h:99-111). entry[] sits + * at 48 because entryCount is followed by 4 bytes of padding. + */ +struct gsp_phys_rmcall_promote_params { + __le32 engine_type; + __le32 h_client; + __le32 ch_id; + __le32 h_chan_client; + __le32 h_object; + __le32 h_virt_memory; + __le64 virt_address; + __le64 size; + __le32 entry_count; + __le32 pad; + struct gsp_phys_rmcall_promote_entry entry[16]; +}; + +/* NV2080_CTRL_GR_CTX_BUFFER_INFO (88 bytes), + * ctrl/ctrl2080/ctrl2080gr.h:1066-1081. */ +struct gsp_phys_rmcall_ctx_buffer_info { + __le64 alignment; + __le64 size; + __le64 buffer_handle; + __le64 page_count; + __le64 phys_addr; + __le32 buffer_type; + __le32 aperture; + __le32 kind; + __le32 page_size; + __le32 b_is_contigous; + __le32 b_global_buffer; + __le32 b_local_buffer; + __le32 b_device_descendant; + u8 uuid[16]; +}; + +/* + * NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS (5648 bytes): hUserClient, + * hChannel, bufferCount and the inline 64-entry info array + * (ctrl2080gr.h:1107-1112). Best-effort readback only. + */ +struct gsp_phys_rmcall_ctx_buffer_info_params { + __le32 h_user_client; + __le32 h_channel; + __le32 buffer_count; + __le32 pad; + struct gsp_phys_rmcall_ctx_buffer_info ctx_buffer_info[64]; +}; + /* Bounded little-endian read of a reply payload word: a read past the * received payload is never performed (off+4 <= payload_len). */ static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec, @@ -6145,6 +6360,64 @@ static void gsp_phys_rmcall_engine_check_change(void) v, gsp_phys_rmcall_engine_name()); } +static const char *gsp_phys_rmcall_gr_promote_name(void) +{ + switch (boot_stage.rmcall_gr_promote) { + case GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE: + return "single-entry bInitialize=1 bNonmapped=0 size=raw+subctx (p6h1b)"; + case GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE: + return "two-phase PA-only nonmapped then VA-only"; + case GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW: + return "single-entry bInitialize=1 bNonmapped=0 size=raw"; + default: + return "off (PROMOTE_CTX never sent)"; + } +} + +/* + * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a + * load-time module parameter (0..3, see gsp_phys_gpu.c); stage 3 snapshots it + * next to the engine so every later step of this staging uses one shape. The + * module parameter's set callback already clamps out-of-range values with a + * warning; the check here is a second line of defense against a future + * writer. On a COPY0 ladder nothing is logged (the promote path never runs), + * so COPY0 output stays byte-identical. + */ +static void gsp_phys_rmcall_gr_promote_snapshot(void) +{ + int v; + + if (boot_stage.rmcall_gr_promote_valid) + return; + v = gsp_phys_rmcall_gr_promote_value(); + if (v < 0 || v > (int)GSP_PHYS_RMCALL_GR_PROMOTE_MAX) { + pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_gr_promote=%d is outside 0..3; clamping to 0 (no PROMOTE_CTX)\n", + v); + v = 0; + } + boot_stage.rmcall_gr_promote = (u32)v; + boot_stage.rmcall_gr_promote_valid = true; + if (!gsp_phys_rmcall_engine_is_gr()) + return; + pr_info("gsp-phys-boot: rm-call: stage-3 gr_promote_param=%d (%s); PROMOTE_CTX is opt-in, every attempt warns (a failed promote can wedge GSP-RM)\n", + v, gsp_phys_rmcall_gr_promote_name()); +} + +static void gsp_phys_rmcall_gr_promote_check_change(void) +{ + int v = gsp_phys_rmcall_gr_promote_value(); + u32 live = (v >= 0 && v <= (int)GSP_PHYS_RMCALL_GR_PROMOTE_MAX) ? + (u32)v : GSP_PHYS_RMCALL_GR_PROMOTE_OFF; + + if (!boot_stage.rmcall_gr_promote_valid || + live == boot_stage.rmcall_gr_promote || + boot_stage.rmcall_gr_promote_change_logged) + return; + boot_stage.rmcall_gr_promote_change_logged = true; + pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_gr_promote changed to %d after the stage-3 snapshot; the ladder keeps gr_promote_param=%u for the rest of this staging (mid-session changes are ignored; reload the module for another variant)\n", + v, boot_stage.rmcall_gr_promote); +} + /* * H1 CE-refusal gate: true when the current staging's ladder channel speaks * GR, i.e. when a CE method block must never be submitted on it. Logs once @@ -6503,6 +6776,51 @@ static void gsp_phys_rmcall_pdb_build(void) (u8 *)gsp_phys_chunk_page_va(pd0, 0) + (size_t)(1 + i) * 16 + 8); } + /* + * H1 GR context window: when the GR ladder prepared the promoted MAIN + * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this + * builder), append its page tables as fresh PD0 dual entries 17.. (the + * first slots above the 16-slot guest scratch pool) and map every 4 KB + * page at GSP_PHYS_RMCALL_VA_GR_CTX. Only freshly allocated PT pages + * and previously zero PD0 entries are written: the ladder and scratch + * mappings stay byte-identical, no engine-fetched line is rewritten, and + * the COPY0 ladder never enters this block. + */ + if (boot_stage.rmcall_ctx_prepared) { + for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) { + u32 pd0_idx = (u32)(((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + + i) & 0xffu); + u64 ctx_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_ctx_pt[i], 0)); + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(ctx_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + for (i = 0; i < boot_stage.rmcall_ctx_pages; i++) { + u32 pt_page = i / (GSP_PHYS_PAGE_SIZE / 8u); + u32 pt_slot = i % (GSP_PHYS_PAGE_SIZE / 8u); + u64 ctx_pte = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_ctx, i)); + + put_unaligned_le64(ctx_pte, + (u8 *)gsp_phys_chunk_page_va( + &boot_stage.rmcall_ctx_pt[pt_page], + 0) + (size_t)pt_slot * 8); + } + pr_info("gsp-phys-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (fresh slots above the scratch pool; no fetched line rewritten)\n", + GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_pages, + boot_stage.rmcall_ctx_pt_pages, + (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu), + (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu) + + boot_stage.rmcall_ctx_pt_pages - 1); + } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; @@ -6550,6 +6868,372 @@ static void gsp_phys_rmcall_pdb_build(void) GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } +/* + * H1 GR context bind, step 1-2: query the engine-context geometry, then + * allocate the zeroed, physically contiguous sysmem context buffer the + * promote installs. GR0 only (the caller gates); any failure aborts the + * ladder with no retry and no fallback (a split buffer cannot be promoted + * through the single PA+size entry, so a per-page allocation is a failure + * here, not a fallback). Must run before gsp_phys_rmcall_pdb_build() so the + * VA window is mapped before the channel alloc commits the PDB base; the + * queries need only the stage-1 device/subdevice handles. + * + * The promoted MAIN size is ALIGN_UP(saveSize, 4 KB) + maxSubctx * 4 KB: + * the save region queried from the device FIFO, rounded to a page, plus one + * 4 KB context header page per subcontext in the TSG's group. The result is + * sanity-bounded (<= 32 MB) and the PT page count follows from the 2 MB + * coverage of one small-PT page. + */ +static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0) +{ + struct gsp_phys_rmcall_engine_ctx_props props; + struct gsp_phys_rmcall_fifo_get_info_params *fifo; + struct gsp_phys_boot_chunk *ctx = &boot_stage.rmcall_ctx; + u64 total; + u32 raw, alignment, max_subctx; + u32 pages, pt_pages, rpc_result = 0, status = 0; + u32 entry_size; + u32 i; + int ret; + + if (!boot_stage.pdev) + return -EINVAL; + /* + * A repeated stage-3 run (debugfs write after an earlier failure) must + * not allocate over the live chunks: reuse the prepared geometry. + */ + if (boot_stage.rmcall_ctx_prepared) { + pr_info("gsp-phys-boot: rm-call: GR ctx already prepared (pa=0x%llx va=0x%llx size=0x%x); reusing\n", + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_size); + return 0; + } + + memset(&props, 0, sizeof(props)); + props.engine_id = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, + GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES, + &props, sizeof(props), 24 + sizeof(props), + &props, sizeof(props), &rpc_result, &status, + "GR_GET_ENGINE_CONTEXT_PROPERTIES"); + raw = le32_to_cpu(props.size); + alignment = le32_to_cpu(props.alignment); + pr_info("gsp-phys-boot: rm-call: GR ctx geometry: engineId=%u rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x (hDevice=0x%x)\n", + GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID, raw, alignment, status, + rpc_result, boot_stage.rmcall_device); + if (ret) + return ret; + if (!raw || alignment < GSP_PHYS_PAGE_SIZE) { + pr_err("gsp-phys-boot: rm-call: GR ctx geometry invalid (rawSize=0x%x alignment=0x%x); aborting the ladder with no retry\n", + raw, alignment); + return -EIO; + } + + fifo = kvzalloc(sizeof(*fifo), GFP_KERNEL); + if (!fifo) + return -ENOMEM; + fifo->fifo_info_tbl_size = cpu_to_le32(1); + fifo->fifo_info_tbl[0].index = cpu_to_le32( + GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT); + fifo->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO, + fifo, sizeof(*fifo), 24 + sizeof(*fifo), + fifo, sizeof(*fifo), &rpc_result, &status, + "GR_FIFO_GET_INFO_MAX_SUBCONTEXT"); + max_subctx = le32_to_cpu(fifo->fifo_info_tbl[0].data); + pr_info("gsp-phys-boot: rm-call: GR ctx maxSubctx=%u (index=%u engineType=0x%x status=0x%x rpc_result=0x%x hSubdevice=0x%x)\n", + max_subctx, GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT, + GSP_PHYS_RMCALL_ENGINE_GR0, status, rpc_result, + boot_stage.rmcall_subdevice); + kvfree(fifo); + if (ret) + return ret; + + total = ALIGN((u64)raw, GSP_PHYS_PAGE_SIZE) + + (u64)max_subctx * GSP_PHYS_PAGE_SIZE; + if (!total || total > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) { + pr_err("gsp-phys-boot: rm-call: GR ctx MAIN size 0x%llx (raw=0x%x maxSubctx=%u) is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n", + total, raw, max_subctx, GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE); + return -E2BIG; + } + pages = (u32)(total / GSP_PHYS_PAGE_SIZE); + pt_pages = DIV_ROUND_UP(pages, GSP_PHYS_PAGE_SIZE / 8u); + if (pt_pages > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) { + pr_err("gsp-phys-boot: rm-call: GR ctx MAIN needs %u PT pages, above the %u-page window (size=0x%x); aborting the ladder with no retry\n", + pt_pages, GSP_PHYS_RMCALL_GR_CTX_PT_MAX, (u32)total); + return -E2BIG; + } + /* + * Mode 3 promotes exactly raw; every other mode promotes the padded + * total. The backing allocation always covers total, so a mode-3 + * entry always names a subset of the allocated bytes. + */ + entry_size = (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW) ? + raw : (u32)total; + boot_stage.rmcall_gr_promote_size = entry_size; + pr_info("gsp-phys-boot: rm-call: GR ctx MAIN: raw=0x%x align=0x%x maxSubctx=%u total=0x%x pages=%u ptPages=%u gr_promote_param=%u entrySize=0x%x\n", + raw, alignment, max_subctx, (u32)total, pages, pt_pages, + boot_stage.rmcall_gr_promote, entry_size); + + ret = gsp_phys_chunk_alloc(boot_stage.pdev, ctx, pages); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GR ctx buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n", + pages, (u32)total, ret); + return ret; + } + if (ctx->page_dma) { + pr_err("gsp-phys-boot: rm-call: GR ctx buffer is not physically contiguous (per-page fallback); PROMOTE_CTX takes one PA+size, so the GR context cannot be promoted; aborting the ladder with no retry\n"); + gsp_phys_chunk_free(boot_stage.pdev, ctx); + return -ENOMEM; + } + gsp_phys_chunk_zero(ctx); + + for (i = 0; i < pt_pages; i++) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_ctx_pt[i], 1u); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GR ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n", + i, pt_pages, ret); + goto err_free; + } + gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]); + } + + boot_stage.rmcall_ctx_pa = (u64)ctx->dma; + boot_stage.rmcall_ctx_va = GSP_PHYS_RMCALL_VA_GR_CTX; + boot_stage.rmcall_ctx_size = (u32)total; + boot_stage.rmcall_ctx_pages = pages; + boot_stage.rmcall_ctx_pt_pages = pt_pages; + boot_stage.rmcall_ctx_prepared = true; + pr_info("gsp-phys-boot: rm-call: GR ctx buffer prepared: total=0x%x pages=%u ptPages=%u pa=0x%llx va=0x%llx (contiguous, zeroed; PDB mapping in gsp_phys_rmcall_pdb_build)\n", + boot_stage.rmcall_ctx_size, boot_stage.rmcall_ctx_pages, + boot_stage.rmcall_ctx_pt_pages, boot_stage.rmcall_ctx_pa, + boot_stage.rmcall_ctx_va); + return 0; + +err_free: + for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) + gsp_phys_chunk_free(boot_stage.pdev, + &boot_stage.rmcall_ctx_pt[i]); + gsp_phys_chunk_free(boot_stage.pdev, ctx); + return ret; +} + +/* + * Fill the single MAIN promote entry with the prepared PA/VA and the + * snapshot's entry size. initialize/nonmapped carry the bisection shape: + * mode 1/3 use {1,0} (PA+VA in one control), mode 2 step 1 uses {1,1} (PA + * only, va=0) and step 2 uses {0,0} (VA only, same PA/size). + */ +static void gsp_phys_rmcall_gr_promote_entry( + struct gsp_phys_rmcall_promote_params *p, u64 va, + u8 initialize, u8 nonmapped) +{ + p->entry[0].gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_ctx_pa); + p->entry[0].gpu_virt_addr = cpu_to_le64(va); + p->entry[0].size = cpu_to_le64(boot_stage.rmcall_gr_promote_size); + p->entry[0].phys_attr = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR); + p->entry[0].buffer_id = cpu_to_le16( + GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN); + p->entry[0].b_initialize = initialize; + p->entry[0].b_nonmapped = nonmapped; +} + +/* + * H1 GR context bind, step 3-4: send NV2080_CTRL_CMD_GPU_PROMOTE_CTX in the + * shape selected by gsp_phys_rmcall_gr_promote (snapshotted in stage 3): + * - mode 1 (p6h1b): one MAIN entry with PA, external-VAS VA and size + * ALIGN_UP(raw, 4 KB) + maxSubctx * 4 KB; + * - mode 2: reference external-VAS shape, a PA-only nonmapped control + * (bInitialize=1/bNonmapped=1, no VA) followed - only on success - by a + * VA-only control (bInitialize=0/bNonmapped=0, same PA/VA/size); + * - mode 3: like 1 but the entry size is the exact raw save size. + * Mode 0 never reaches this function (stage 3 stops before it); the early + * return is a belt-and-braces no-op. Success means the server installed the + * context memdesc, added the VA to the engine-context vaList, committed the + * context (ENG_CTX_VALID=1) and ran the GR golden-image bind; only then may + * stage 4 BIND/submit run. + * + * OBSERVED WEDGE: mode 1 asserted GSP-RM on submission (NOCAT recType=5 + * errorCode=0x52a8b04 source="ASSERT", then no fn76 reply within the 2000 ms + * bound; no 0x1006, no engine fault) and WPR2 could not be cleared afterwards + * (fuse-locked PLM -> reboot with human approval). The first promote control + * of any staging is therefore warned once below; on failure the function + * aborts with no retry, reports exactly which step was last accepted and + * leaves the context buffer allocated until staging free/rmmod. The control + * targets the subdevice, like the reference client promote + * (kernel_graphics_object.c:134-139; kernel_falcon.c:271-272). + */ +static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) +{ + struct gsp_phys_rmcall_promote_params *p; + u32 rpc_result = 0, status = 0; + u32 mode = boot_stage.rmcall_gr_promote; + u32 size = boot_stage.rmcall_gr_promote_size; + int ret; + + if (!boot_stage.rmcall_ctx_prepared || + !boot_stage.rmcall_channel || !boot_stage.rmcall_gr_object) + return -EINVAL; + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF) + return -EOPNOTSUPP; + if (!boot_stage.rmcall_gr_promote_warned) { + boot_stage.rmcall_gr_promote_warned = true; + pr_warn("gsp-phys-boot: rm-call: WARNING: gr_promote_param=%u sends GPU_PROMOTE_CTX: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n", + mode); + } + + p = kvzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + + p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0); + p->h_client = cpu_to_le32(0); + p->ch_id = cpu_to_le32(0); + p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_object = cpu_to_le32(boot_stage.rmcall_channel); + p->h_virt_memory = cpu_to_le32(0); + p->virt_address = cpu_to_le64(0); + p->size = cpu_to_le64(0); + p->entry_count = cpu_to_le32(1); + + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE) { + /* Step 1/2: PA-only nonmapped entry (no VA). */ + gsp_phys_rmcall_gr_promote_entry(p, 0, 1, 1); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (PA-only)"); + pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX PA-only (step 1/2): pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, size, status, rpc_result, + boot_stage.rmcall_channel, boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX PA-only (step 1/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/2); no further controls are sent and the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + + /* Step 2/2: VA-only entry (same PA/VA/size, initialize off). */ + gsp_phys_rmcall_gr_promote_entry(p, boot_stage.rmcall_ctx_va, + 0, 0); + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, + "GPU_PROMOTE_CTX (VA-only)"); + pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX VA-only (step 2/2): pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + size, status, rpc_result, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX VA-only (step 2/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: PA-only nonmapped accepted (1/2); no further controls are sent and the ladder aborts before BIND with no retry\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 2; + } else { + /* + * Modes 1 and 3: one MAIN entry carrying both the PA + * (bInitialize=1) and the external-VAS VA (bNonmapped=0); + * mode 3 carries the exact raw size. + */ + gsp_phys_rmcall_gr_promote_entry(p, boot_stage.rmcall_ctx_va, + 1, 0); + ret = gsp_phys_rmcall_control(bar0, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, + p, sizeof(*p), 24, NULL, 0, + &rpc_result, &status, "GPU_PROMOTE_CTX MAIN"); + pr_info("gsp-phys-boot: rm-call: PROMOTE_CTX MAIN: pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n", + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + size, status, rpc_result, boot_stage.rmcall_channel, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_ENGINE_GR0, mode); + if (ret) { + pr_err("gsp-phys-boot: rm-call: PROMOTE_CTX MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry (read the failed-reply dump above: 0x23 invalid client, 0x36 invalid parent, 0x56 not supported)\n", + status, rpc_result, + gsp_phys_rmcall_status_str(status), ret); + kvfree(p); + return ret; + } + boot_stage.rmcall_gr_promote_step = 1; + } + + kvfree(p); + boot_stage.rmcall_ctx_valid = true; + return 0; +} + +/* + * H1 GR context bind, step 5: best-effort readback of the committed context + * buffer geometry. Never aborts: a server that does not implement + * NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_INFO (or returns a short reply) only + * loses the verification line. The request/response shape mirrors the + * reference kernel client (nv_gpu_ops.c:10060-10163): the control targets the + * subdevice with hUserClient/hChannel in the params. + */ +static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0) +{ + struct gsp_phys_rmcall_ctx_buffer_info_params *p; + u32 rpc_result = 0, status = 0, count, shown, i; + int ret; + + p = kvzalloc(sizeof(*p), GFP_KERNEL); + if (!p) { + pr_warn("gsp-phys-boot: rm-call: GR ctx info skipped: no memory for the %zu-byte best-effort query\n", + sizeof(*p)); + return; + } + p->h_user_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_channel = cpu_to_le32(boot_stage.rmcall_channel); + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO, + p, sizeof(*p), 24 + sizeof(*p), + p, sizeof(*p), &rpc_result, &status, + "GR_GET_CTX_BUFFER_INFO"); + if (ret) { + pr_warn("gsp-phys-boot: rm-call: GR ctx info (best effort) failed: status=0x%x rpc_result=0x%x ret=%d; the promote result stands\n", + status, rpc_result, ret); + kvfree(p); + return; + } + + count = le32_to_cpu(p->buffer_count); + shown = min_t(u32, count, 4u); + pr_info("gsp-phys-boot: rm-call: GR_GET_CTX_BUFFER_INFO: bufferCount=%u (showing %u)\n", + count, shown); + for (i = 0; i < shown; i++) { + const struct gsp_phys_rmcall_ctx_buffer_info *ci = + &p->ctx_buffer_info[i]; + + pr_info("gsp-phys-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n", + i, le32_to_cpu(ci->buffer_type), + le64_to_cpu(ci->phys_addr), + le32_to_cpu(ci->aperture), le64_to_cpu(ci->size), + le64_to_cpu(ci->page_count), + le64_to_cpu(ci->alignment), + le32_to_cpu(ci->b_is_contigous)); + } + kvfree(p); +} + + /* * Stage 3: explicit VAS with its page directory, then channel group (COPY0, * physical) and the channel. The VAS and PDB must both succeed: a channel @@ -6579,6 +7263,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) struct gsp_phys_rmcall_vas_params vas; struct gsp_phys_rmcall_pdb_params pdb; struct gsp_phys_rmcall_tsg_params tsg; + char promote_suffix[80]; u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0; int ret; @@ -6587,6 +7272,8 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) * first engine-dependent request (TSG/channel alloc params below). */ gsp_phys_rmcall_engine_snapshot(); + /* H1: same load-time freeze for the promote bisection shape. */ + gsp_phys_rmcall_gr_promote_snapshot(); /* * The TSG below carries this externally-owned FERMI_VASPACE_A, whose @@ -6634,6 +7321,22 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) pdb_rpc); return ret; } + /* + * H1: on a GR ladder, query and allocate the engine-context MAIN + * buffer BEFORE the PDB hierarchy is built, so its VA window is mapped + * before the channel alloc commits the page-directory base. The queries + * need only the stage-1 device/subdevice handles; any failure aborts the + * ladder with no retry. On COPY0 this block is skipped entirely (no + * RPC, no allocation, no log). + */ + if (gsp_phys_rmcall_engine_is_gr()) { + ret = gsp_phys_rmcall_gr_ctx_prepare(bar0); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 3 aborted: GR context buffer prepare failed (%d); the GR context is not committed and the ladder aborts with no retry\n", + ret); + return ret; + } + } /* * The server only stored our root memdesc; the GPFIFO, pushbuffer and * CE src/dst/semaphore fetches translate through it, so fill the @@ -6683,9 +7386,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) /* * H1: bind the compute engine object to the just-allocated GR channel - * before stage 3 returns. A rejection aborts the ladder (the GR - * method block's SET_OBJECT would fail too) and is never retried. On a - * COPY0 ladder this step is skipped entirely: no RPC, no log. + * and commit the engine context before stage 3 returns. The TURING_ + * COMPUTE_A alloc rejection aborts the ladder (the GR method block's + * SET_OBJECT would fail too) and is never retried. The promote then + * installs the MAIN context buffer and commits ENG_CTX_VALID; a + * rejection aborts before BIND and is never retried. On a COPY0 ladder + * this whole block is skipped entirely: no RPC, no log. */ if (gsp_phys_rmcall_engine_is_gr()) { ret = gsp_phys_rmcall_alloc_gr_object(bar0); @@ -6694,15 +7400,47 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) ret); return ret; } + /* + * gr_promote_param=0 stops here: the whole pre-promote half + * (geometry queries, contiguous context buffer, PDB mapping, + * channel and compute object) ran and PROMOTE_CTX is never + * sent, so BIND/SCHEDULE/submit cannot run against an + * uncommitted engine context. The stage fails on purpose so + * the ladder records the stop and stage 4 cannot proceed. + */ + if (boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_OFF) { + pr_info("gsp-phys-boot: rm-call: gr_promote_param=0: GPU_PROMOTE_CTX not sent; GR ctx prepared (pa=0x%llx va=0x%llx size=0x%x) and PDB-mapped, but not committed; the ladder stops before BIND/SCHEDULE/submit (safe half complete; set gsp_phys_rmcall_gr_promote=1/2/3 and reload to attempt the promote)\n", + boot_stage.rmcall_ctx_pa, + boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_size); + return -EOPNOTSUPP; + } + ret = gsp_phys_rmcall_gr_ctx_promote(bar0); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 3 aborted: GPU_PROMOTE_CTX failed (%d); the GR context is not committed and BIND/SCHEDULE/submit are not attempted (no retry; UNLOAD + rmmod remains the fail-safe)\n", + ret); + return ret; + } + gsp_phys_rmcall_gr_ctx_info(bar0); } - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n", + if (gsp_phys_rmcall_engine_is_gr()) + scnprintf(promote_suffix, sizeof(promote_suffix), + " gr_promote_param=%u step=%u entrySize=0x%x", + boot_stage.rmcall_gr_promote, + boot_stage.rmcall_gr_promote_step, + boot_stage.rmcall_gr_promote_size); + else + promote_suffix[0] = '\0'; + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base, gsp_phys_rmcall_engine_name(), - gsp_phys_rmcall_engine_is_gr() ? " hGr=0xcaf0002c" : ""); + gsp_phys_rmcall_engine_is_gr() ? " hGr=0xcaf0002c" : "", + promote_suffix); return 0; } @@ -7975,6 +8713,8 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) /* H1: note (once) a mid-session engine-parameter change, keep snapshot. */ gsp_phys_rmcall_engine_check_change(); + /* H1: same note for the promote bisection shape. */ + gsp_phys_rmcall_gr_promote_check_change(); /* * The winning variant requested server-allocated memdescs: there is no @@ -8633,6 +9373,9 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx); + for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++) + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]); gsp_phys_rmcall_reset(); } @@ -8747,6 +9490,19 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2, boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); + /* COPY0 sessions print no line (byte-identical state output). */ + if (boot_stage.rmcall_ctx_prepared) + seq_printf(m, + "rm_call_ctx: prepared=%u valid=%u pa=0x%llx va=0x%llx size=0x%x pages=%u ptPages=%u gr_promote_param=%u step=%u entrySize=0x%x\n", + boot_stage.rmcall_ctx_prepared, + boot_stage.rmcall_ctx_valid, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va, + boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_pages, + boot_stage.rmcall_ctx_pt_pages, + boot_stage.rmcall_gr_promote, + boot_stage.rmcall_gr_promote_step, + boot_stage.rmcall_gr_promote_size); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -10033,6 +10789,39 @@ static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, size) == 8, "NV_GR_ALLOCATION_PARAMETERS.size must be at offset 8"); static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, caps) == 12, "NV_GR_ALLOCATION_PARAMETERS.caps must be at offset 12"); +static_assert(sizeof(struct gsp_phys_rmcall_engine_ctx_props) == 12, + "NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS must be 12 bytes"); +static_assert(offsetof(struct gsp_phys_rmcall_engine_ctx_props, size) == 8, + "engine context properties.size must be at offset 8"); +static_assert(sizeof(struct gsp_phys_rmcall_fifo_get_info_params) == 2056, + "NV2080_CTRL_FIFO_GET_INFO_PARAMS must be 2056 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_fifo_get_info_params, + engine_type) == 2052, + "NV2080_CTRL_FIFO_GET_INFO_PARAMS.engineType must be at offset 2052"); +static_assert(sizeof(struct gsp_phys_rmcall_promote_entry) == 32, + "NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY must be 32 bytes"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_entry, + gpu_virt_addr) == 8, + "promote entry.gpuVirtAddr must be at offset 8"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_entry, phys_attr) == 24, + "promote entry.physAttr must be at offset 24"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_entry, b_nonmapped) == 31, + "promote entry.bNonmapped must be at offset 31"); +static_assert(sizeof(struct gsp_phys_rmcall_promote_params) == 560, + "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS must be 560 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_params, virt_address) == 24, + "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.virtAddress must be at offset 24"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40, + "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40"); +static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48, + "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)"); +static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 88, + "NV2080_CTRL_GR_CTX_BUFFER_INFO must be 88 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info_params) == 5648, + "NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS must be 5648 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_ctx_buffer_info_params, + ctx_buffer_info) == 16, + "NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS.ctxBufferInfo must be at offset 16 (8-byte aligned)"); static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368, "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8, @@ -10105,6 +10894,19 @@ static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE == 512, "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)"); +static_assert(((GSP_PHYS_RMCALL_VA_GR_CTX >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_CTX >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_CTX >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xff) == + ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) + + GSP_PHYS_RMCALL_GUEST_PT_PAGES && + ((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xff) + + GSP_PHYS_RMCALL_GR_CTX_PT_MAX <= 256, + "the GR ctx window must use fresh PD0 entries right above the guest scratch pool under the same root/PD2/PD1 chain"); +static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + + (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the GR ctx window must stay below the VAS vaSize limit"); /* * GspSystemInfo wire ABI, pinned against the reference header compile diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 81f1803..3a19060 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -416,9 +416,17 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); * selects the GR0 engine for the same ladder: the TSG/channel carry * engineType 1, the channel walk is a single no-retry V5 attempt, a * TURING_COMPUTE_A (0xc5c0) engine object is allocated on the channel, - * BIND uses engineType 1 and stage 4 releases a GR semaphore instead of - * copying (logged as REAL GR SEM OK / REAL GR SEM FAILED); all CE - * forwarding paths refuse once per staging. Writing 9 re-runs one GR + * and the GR engine-context MAIN buffer is queried (device context + * properties + subdevice MAX_SUBCONTEXT_PER_GROUP), allocated as one + * contiguous zeroed sysmem buffer, mapped through fresh PDB slots above + * the guest scratch pool and promoted with NV2080_CTRL_CMD_GPU_PROMOTE_CTX + * carrying both its PA (bInitialize) and VAS VA (bNonmapped=0) so the + * server commits ENG_CTX_VALID before BIND uses engineType 1; stage 4 + * then releases a GR semaphore instead of copying (logged as REAL GR SEM + * OK / REAL GR SEM FAILED); all CE forwarding paths refuse once per + * staging. Every context query/allocation/promote failure aborts the + * ladder with no retry and the context buffer is freed by + * gsp_phys_boot_staging_free() on release. Writing 9 re-runs one GR * semaphore release on an already-built GR ladder without a full stage * cycle (no RPC; requires the GR object and work-submit token). * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 227246b..a52714c 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -341,6 +341,62 @@ int gsp_phys_rmcall_engine_value(void) } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_engine_value); +/* + * H1 GR context-promote bisection selector (default 0). The p6h1b single + * entry (value 1, size raw+subctx) made GSP-RM assert on submission (NOCAT + * recType=5 errorCode=0x52a8b04, no fn76 reply) and WPR2 could not be + * cleared afterwards, so every other promote shape is opt-in here: + * 0 = PROMOTE_CTX is never sent (the GR ladder still runs the geometry + * queries, the context buffer/PT allocation, the PDB mapping and the + * channel/compute alloc, then stops before BIND with a clear log); + * 1 = single MAIN entry, bInitialize=1/bNonmapped=0, size raw+subctx; + * 2 = two-phase external-VAS shape (PA-only nonmapped first, then VA-only); + * 3 = single MAIN entry like 1 but with size = the exact raw save size. + * The set callback clamps out-of-range values to 0 and warns on every nonzero + * enable: a failed GR context promote can wedge the device and needs a reboot + * with human approval, so run it only when reboot readiness is confirmed. + * The value is snapshotted when stage 3 starts; later sysfs changes are noted + * once and ignored, so a session can never mix promote shapes. + */ +static int gsp_phys_rmcall_gr_promote_param; + +static int gsp_phys_rmcall_gr_promote_set(const char *val, + const struct kernel_param *kp) +{ + int ret; + + ret = param_set_int(val, kp); + if (ret) + return ret; + if (gsp_phys_rmcall_gr_promote_param < 0 || + gsp_phys_rmcall_gr_promote_param > 3) { + pr_warn("gsp-phys: gsp_phys_rmcall_gr_promote=%d is outside 0..3; clamping to 0 (no PROMOTE_CTX)\n", + gsp_phys_rmcall_gr_promote_param); + gsp_phys_rmcall_gr_promote_param = 0; + } + if (gsp_phys_rmcall_gr_promote_param != 0) + pr_warn("gsp-phys: WARNING: gsp_phys_rmcall_gr_promote=%d enabled: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n", + gsp_phys_rmcall_gr_promote_param); + return 0; +} + +static const struct kernel_param_ops gsp_phys_rmcall_gr_promote_ops = { + .set = gsp_phys_rmcall_gr_promote_set, + .get = param_get_int, +}; + +module_param_cb(gsp_phys_rmcall_gr_promote, + &gsp_phys_rmcall_gr_promote_ops, + &gsp_phys_rmcall_gr_promote_param, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote, + "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only; 3: single entry size=raw); values outside 0..3 clamp to 0 with a warning"); + +int gsp_phys_rmcall_gr_promote_value(void) +{ + return gsp_phys_rmcall_gr_promote_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value); + /* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index f34d7a2..4e5e823 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -175,6 +175,29 @@ bool gsp_phys_fault_inject_enabled(void); */ int gsp_phys_rmcall_engine_value(void); +/* + * GR context-promote bisection selector (gsp_phys_rmcall_gr_promote; default + * 0). Returns the module-parameter value, one of: + * 0 = never send NV2080_CTRL_CMD_GPU_PROMOTE_CTX. The GR ladder still runs + * the engine-context geometry queries, allocates and PDB-maps the MAIN + * buffer, allocates the channel/compute object, then stops before BIND + * with a clear log (the whole safe half; reviewable without a promote); + * 1 = single MAIN entry, bInitialize=1/bNonmapped=0, size = + * ALIGN_UP(raw save size, 4 KB) + maxSubctx * 4 KB (the p6h1b shape + * that asserted GSP-RM with errorCode 0x52a8b04); + * 2 = reference external-VAS two-phase shape: a PA-only nonmapped entry + * (bInitialize=1/bNonmapped=1, no VA) first, then a VA-only entry + * (bInitialize=0/bNonmapped=0) with the same PA/VA/size; + * 3 = single MAIN entry like 1 but with size = the exact raw save size. + * The parameter's set callback clamps any value outside 0..3 to 0 with a + * warning and warns on every nonzero enable (a failed promote can assert + * GSP-RM and wedge the device; recovery needs a reboot with human approval). + * The booter snapshots the value when the rm-call ladder's stage 3 starts + * and ignores later changes, so this accessor is for the snapshot and + * diagnostics only. + */ +int gsp_phys_rmcall_gr_promote_value(void); + /* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): -- cgit v1.2.3