diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 509 |
1 files changed, 397 insertions, 112 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 920ccdb..3ba8714 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -199,10 +199,15 @@ bool gsp_phys_boot_requested(void) * NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C * (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43), * TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM - * 0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN - * 0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE - * 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), FERMI_VASPACE_A 0x90F1 - * (class/cl90f1.h:30), NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813 + * 0x81 (class/cl84a0.h:45), NV01_MEMORY_LIST_FBMEM 0x82 + * (class/cl84a0.h:47), NV01_MEMORY_LOCAL_USER 0x40 (class/cl0040.h:34), + * NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN 0xC36F0108 + * (ctrl/ctrlc36f.h:82), NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_ + * NOTIF_INDEX 0xC36F010A (ctrl/ctrlc36f.h:135), NVA06F_CTRL_CMD_ + * GPFIFO_SCHEDULE 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), + * NVA06F_CTRL_CMD_BIND 0xA06F0104 (ctrl/ctrla06f/ctrla06fgpfifo.h:83), + * FERMI_VASPACE_A 0x90F1 (class/cl90f1.h:30), + * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813 * (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9 * (class/cl2080_notification.h:292). */ @@ -214,16 +219,36 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CLASS_DEVICE 0x0080u #define GSP_PHYS_RMCALL_CLASS_SUBDEVICE 0x2080u #define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM 0x0081u +#define GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER 0x0040u #define GSP_PHYS_RMCALL_CLASS_TSG 0xa06cu #define GSP_PHYS_RMCALL_CLASS_CHANNEL 0xc46fu #define GSP_PHYS_RMCALL_CLASS_CE 0xc5b5u #define GSP_PHYS_RMCALL_CLASS_VASPACE 0x90f1u #define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN 0xc36f0108u +#define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u +#define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY 0x801813u #define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u /* + * Work-submit token notifier slot. The default index is + * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1 + * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex + * = 3 for subdevice 0 before asking for the token + * (nvidia-push-init.c:303-316), so the 4 KB FBMEM notifier below must span at + * least (3 + 1) * sizeof(NvNotification) = 64 B. + */ +#define GSP_PHYS_RMCALL_NOTIF_INDEX 3u + +/* NVOS32 type of the error-notifier allocation (nvos.h:939); attr bits + * 26:25 = VIDMEM location and 31:29 = UNCACHED coherency are both zero, the + * shape the RM's own channel-notifier allocation uses + * (mem_utils_gm107.c:344-352). */ +#define GSP_PHYS_RMCALL_OS32_TYPE_IMAGE 0u +#define GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED 0u + +/* * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3 * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is * the smallest deterministic size this ladder needs and keeps vaBase 0. The @@ -262,13 +287,14 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL 0x2u /* - * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 (kernel/vgpu/ - * rm_plugin_shared_code.h:66) and NV_MEMORY_UNCACHED=1 (kernel/os/ - * nv_memory_type.h:31), which is the default RM system-memory CPU cache - * attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE + * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 and ADDR_FBMEM=2 + * (kernel/vgpu/rm_plugin_shared_code.h:66-67) and NV_MEMORY_UNCACHED=1 + * (kernel/os/nv_memory_type.h:31), which is the default RM system-memory CPU + * cache attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE * aperture (clc5b5.h:56-60). */ #define GSP_PHYS_RMCALL_ADDR_SYSMEM 1u +#define GSP_PHYS_RMCALL_ADDR_FBMEM 2u #define GSP_PHYS_RMCALL_CACHE_UNCACHED 1u #define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM 2u @@ -291,6 +317,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 0xcaf00014u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u +#define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u @@ -327,6 +354,7 @@ enum gsp_phys_rmcall_buf { }; #define GSP_PHYS_RMCALL_COPY_SIZE 0x1000u +#define GSP_PHYS_RMCALL_NOTIFIER_SIZE 0x1000u #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES 8u #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF 0x8cu /* Nvc46fControl GPPut (clc46f.h:57) */ #define GSP_PHYS_RMCALL_USERD_GPGET_OFF 0x88u /* Nvc46fControl GPGet (clc46f.h:56) */ @@ -861,6 +889,9 @@ static struct gsp_phys_boot_stage { * the variant table for its number) */ bool rmcall_channel_server_mem; /* winning variant had server-allocated mem */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ + u64 rmcall_notifier_base; /* its returned frame-buffer address */ + bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1373,6 +1404,9 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_channel_variant = 0; boot_stage.rmcall_channel_server_mem = false; boot_stage.rmcall_token = 0; + boot_stage.rmcall_notifier_mem = 0; + boot_stage.rmcall_notifier_base = 0; + boot_stage.rmcall_notifier_valid = false; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -4208,6 +4242,41 @@ struct gsp_phys_rmcall_mem_desc { __le32 cache_attrib; }; +/* + * NV_MEMORY_ALLOCATION_PARAMS (nvos.h:1634-1668), 128 bytes. The u64 fields + * are 8-byte aligned; offset and limit are [IN/OUT], so a frame-buffer + * allocation returns the allocated FB offset (and its limit) in the echoed + * params. type/attr carry the NVOS32 enums (type IMAGE, location VIDMEM, + * coherency UNCACHED are all the zero/default values). + */ +struct gsp_phys_rmcall_memory_alloc_params { + __le32 owner; + __le32 type; + __le32 flags; + __le32 width; + __le32 height; + __le32 pitch; + __le32 attr; + __le32 attr2; + __le32 format; + __le32 compr_covg; + __le32 zcull_covg; + __le32 pad0; + __le64 range_lo; + __le64 range_hi; + __le64 size; + __le64 alignment; + __le64 offset; + __le64 limit; + __le64 address; + __le32 ctag_offset; + __le32 h_va_space; + __le32 internalflags; + __le32 tag; + __le32 numa_node; + __le32 pad1; +}; + /* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags, * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */ struct gsp_phys_rmcall_vas_params { @@ -4565,13 +4634,103 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) } /* + * Allocate the channel error-context notifier in FBMEM. On GSP the channel + * notify path rejects a non-FBMEM error-context memdesc with + * NV_ERR_INVALID_STATE (kernel_channel.c:1924-1925), and the server's channel + * constructor materializes that memdesc from NV_CHANNEL_ALLOC_PARAMS' + * errorNotifierMem base/size/addressSpace; the notifier slot it writes is + * notifyIndex (set to GSP_PHYS_RMCALL_NOTIF_INDEX by the stage-4 sequence). + * + * The memlist registration used for the sysmem ladder buffers cannot produce + * a safe FBMEM memdesc for a host client (classes 0x81/0x82 only reference + * frame-buffer pages a guest already owns; 0x82 would accept our host PFNs as + * literal FB offsets). The frame-buffer page is therefore allocated from the + * GSP-RM itself with GSP_RM_ALLOC (103) class NV01_MEMORY_LOCAL_USER (0x40), + * mirroring the RM's own channel-notifier allocation (type IMAGE, location + * VIDMEM, coherency UNCACHED; mem_utils_gm107.c:344-352); the server returns + * the allocated FB offset in NV_MEMORY_ALLOCATION_PARAMS.offset, which is what + * errorNotifierMem.base carries. + * + * Failure is non-fatal: the channel alloc then keeps a zeroed notifier + * memdesc (server SYSMEM/ADDR_UNKNOWN, the cycle-19 shape) and the stage-4 + * GET_WORK_SUBMIT_TOKEN recovery path still obtains the token from the + * control reply. The FB object is a server object like the VAS/TSG/channel + * and is not torn down by rm-call fini (UNLOAD + rmmod remains the fail-safe). + */ +static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) +{ + struct gsp_phys_rmcall_memory_alloc_params mem; + struct gsp_phys_rmcall_rm_alloc_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + sizeof(mem); + u32 status; + u64 base; + int ret; + + memset(&mem, 0, sizeof(mem)); + mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE); + mem.size = cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE); + mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED); + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_parent = cpu_to_le32(boot_stage.rmcall_device); + req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM); + req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER); + req->params_size = cpu_to_le32(sizeof(mem)); + memcpy(req->params, &mem, sizeof(mem)); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, + req, req_len, 20, &rec, true, NULL); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", + GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM, + GSP_PHYS_RMCALL_NOTIFIER_SIZE, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (status != 0 || + rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8) { + pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", + status, rec.payload_len, + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 + + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset)); + gsp_phys_rx_record_free(&rec); + + boot_stage.rmcall_notifier_mem = GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM; + boot_stage.rmcall_notifier_base = base; + boot_stage.rmcall_notifier_valid = true; + pr_info("gsp-phys-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n", + boot_stage.rmcall_notifier_mem, GSP_PHYS_RMCALL_NOTIFIER_SIZE, + boot_stage.rmcall_notifier_base, GSP_PHYS_RMCALL_NOTIF_INDEX); + return 0; +} + +/* * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at - * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). + * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional) + * receives the server control status even when it is nonzero; the echoed + * params are copied into out before that status is checked, so a caller can + * still recover server-written output (the work-submit token) from a failed + * reply. */ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, const void *params, u32 params_len, u32 reply_min, void *out, u32 out_len, - u32 *out_rpc_result, const char *what) + u32 *out_rpc_result, u32 *out_status, + const char *what) { struct gsp_phys_rmcall_rm_control_req *req; struct gsp_phys_rx_record rec; @@ -4579,6 +4738,8 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, u32 status; int ret; + if (out_status) + *out_status = 0; req = kvzalloc(req_len, GFP_KERNEL); if (!req) return -ENOMEM; @@ -4609,6 +4770,8 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, return -EIO; } status = gsp_phys_rmcall_reply_u32(&rec, 12); + if (out_status) + *out_status = status; if (out && out_len && rec.payload_len > 24) { u32 n = min_t(u32, out_len, rec.payload_len - 24); @@ -4806,6 +4969,22 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, ch->cid = cpu_to_le32(1); /* client hint; server assigns its own */ ch->sub_device_id = cpu_to_le32(0); ch->internal_flags = cpu_to_le32(v->internal_flags); + /* + * The error-context notifier must be an FBMEM memdesc on GSP + * (kernel_channel.c:1924-1925) and the server builds pErrContextMemDesc + * from these fields; carry the stage-3 FBMEM allocation when it exists. + * The other memdescs stay the client sysmem chunks. + */ + if (boot_stage.rmcall_notifier_valid) { + ch->error_notifier_mem.base = + cpu_to_le64(boot_stage.rmcall_notifier_base); + ch->error_notifier_mem.size = + cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE); + ch->error_notifier_mem.address_space = + cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM); + ch->error_notifier_mem.cache_attrib = + cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); + } if (v->server_mem) return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */ ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( @@ -4818,28 +4997,94 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, } /* + * Channel-alloc variant walk (same already-held gsp_phys_rm_lock and gates): + * the first rpc_result==0 wins and records its handle/variant/server-mem + * state. Failed variants log one compact line each and no payload dump; an + * all-failed walk ends in one summary line with every variant's + * rpc_result/status. Kept out of stage 3 so the inline stack stays small. + */ +static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) +{ + struct gsp_phys_rmcall_channel_params ch; + u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; + u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; + u32 i; + int ret; + + ret = -EIO; + for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) { + const struct gsp_phys_rmcall_channel_variant *v = + &gsp_phys_rmcall_channel_variants[i]; + u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : + boot_stage.rmcall_device; + + gsp_phys_rmcall_channel_params_init(&ch, v); + v_rpc[i] = 0; + v_status[i] = 0; + ret = gsp_phys_rmcall_alloc(bar0, + boot_stage.rmcall_client, parent, + v->handle, + GSP_PHYS_RMCALL_CLASS_CHANNEL, + &ch, sizeof(ch), false, NULL, v->name, + false, &v_rpc[i], &v_status[i]); + pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", + v->id, i + 1, + ARRAY_SIZE(gsp_phys_rmcall_channel_variants), + v->name, parent, v->handle, v->flags, + v->internal_flags, ret, v_rpc[i], v_status[i]); + if (!ret) { + boot_stage.rmcall_channel = v->handle; + boot_stage.rmcall_channel_variant = v->id; + boot_stage.rmcall_channel_server_mem = v->server_mem; + pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n", + v->id, v->name, v->handle); + return 0; + } + } + + { + char summary[256]; + u32 pos = 0; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) + pos += scnprintf(summary + pos, sizeof(summary) - pos, + "%sV%u %s rpc=0x%x status=0x%x", + i ? "; " : "", + gsp_phys_rmcall_channel_variants[i].id, + gsp_phys_rmcall_channel_variants[i].name, + v_rpc[i], v_status[i]); + pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", + ARRAY_SIZE(gsp_phys_rmcall_channel_variants), summary); + } + return ret; +} + +/* * Stage 3: explicit VAS with its page directory, then channel group (COPY0, * physical) and the channel. The VAS and PDB must both succeed: a channel * whose VAS has no page-directory base fails the context-share/instance-block * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel - * alloc then walks the variants below in order (same already-held - * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants - * log one compact line each and no payload dump, and an all-failed walk ends - * in one summary line with every variant's rpc_result/status. Then - * GET_WORK_SUBMIT_TOKEN. The token is requested before scheduling as - * specified; because the server's TU102 generator requires the channel to be - * on a runlist already (kernel_fifo_tu102.c:110-116, which GPFIFO_SCHEDULE - * establishes at kernel_channel.c:3244), a nonzero status is logged and - * retried after the schedule in stage 4 instead of aborting. + * alloc then walks the variants in gsp_phys_rmcall_alloc_channel in order + * (same already-held gsp_phys_rm_lock and gates); the first rpc_result==0 + * wins. Failed variants log one compact line each and no payload dump, and an + * all-failed walk ends in one summary line with every variant's + * rpc_result/status. + * + * Before the walk, one 4 KB FBMEM error-notifier page is allocated from the + * GSP-RM and its memdesc fields are carried in every channel variant (the + * server's GSP-only notify path requires the error-context notifier to live + * in ADDR_FBMEM, kernel_channel.c:1924-1925). The work-submit token is not + * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist + * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE -> + * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in + * stage 4. */ static int gsp_phys_rmcall_stage3(void __iomem *bar0) { struct gsp_phys_rmcall_vas_params vas; struct gsp_phys_rmcall_pdb_params pdb; struct gsp_phys_rmcall_tsg_params tsg; - struct gsp_phys_rmcall_channel_params ch; u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0; - u32 token = 0; int ret; /* @@ -4881,7 +5126,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY, &pdb, sizeof(pdb), 16, NULL, 0, - &pdb_rpc, "DMA_SET_PAGE_DIRECTORY"); + &pdb_rpc, NULL, "DMA_SET_PAGE_DIRECTORY"); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n", boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries), @@ -4910,79 +5155,27 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) return ret; boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG; - { - u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; - u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; - u32 i; - - ret = -EIO; - for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) { - const struct gsp_phys_rmcall_channel_variant *v = - &gsp_phys_rmcall_channel_variants[i]; - u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : - boot_stage.rmcall_device; - - gsp_phys_rmcall_channel_params_init(&ch, v); - v_rpc[i] = 0; - v_status[i] = 0; - ret = gsp_phys_rmcall_alloc(bar0, - boot_stage.rmcall_client, parent, - v->handle, - GSP_PHYS_RMCALL_CLASS_CHANNEL, - &ch, sizeof(ch), false, NULL, v->name, - false, &v_rpc[i], &v_status[i]); - pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", - v->id, i + 1, - ARRAY_SIZE(gsp_phys_rmcall_channel_variants), - v->name, parent, v->handle, v->flags, - v->internal_flags, ret, v_rpc[i], v_status[i]); - if (!ret) { - boot_stage.rmcall_channel = v->handle; - boot_stage.rmcall_channel_variant = v->id; - boot_stage.rmcall_channel_server_mem = v->server_mem; - pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n", - v->id, v->name, v->handle); - break; - } - } - if (ret) { - char summary[256]; - u32 pos = 0; - - for (i = 0; - i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); - i++) - pos += scnprintf(summary + pos, - sizeof(summary) - pos, - "%sV%u %s rpc=0x%x status=0x%x", - i ? "; " : "", - gsp_phys_rmcall_channel_variants[i].id, - gsp_phys_rmcall_channel_variants[i].name, - v_rpc[i], v_status[i]); - pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", - ARRAY_SIZE(gsp_phys_rmcall_channel_variants), - summary); - return ret; - } - } - - ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, - GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, - &token, sizeof(token), 28, &token, - sizeof(token), NULL, - "GET_WORK_SUBMIT_TOKEN"); - if (ret) { - pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n", + /* + * Allocate the channel's error-context notifier page in FBMEM before + * the channel alloc so every variant carries the memdesc. A failure is + * logged and tolerated: the channel then keeps the zeroed (server + * ADDR_UNKNOWN/SYSMEM) notifier and stage 4's token recovery covers the + * notify rejection. + */ + ret = gsp_phys_rmcall_alloc_notifier_mem(bar0); + if (ret) + pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", ret); - boot_stage.rmcall_token = 0; - } else { - boot_stage.rmcall_token = token; - } - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n", + ret = gsp_phys_rmcall_alloc_channel(bar0); + if (ret) + return ret; + + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, - boot_stage.rmcall_token); + boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", + boot_stage.rmcall_notifier_base); return 0; } @@ -5042,11 +5235,21 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma, } /* - * Stage 4: GPFIFO_SCHEDULE, (retry) GET_WORK_SUBMIT_TOKEN, then the first + * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> + * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN - then the first * real work item: write the CE method block to the pushbuffer, publish the * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem * semaphore word. Afterwards dst must equal src byte for byte. + * + * BIND is the precondition the TU102 token generator needs (runlist + * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets + * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for + * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a + * target slot. Every control failure logs its rpc_result/status and aborts + * before the doorbell; a 0x40 GET_WORK_SUBMIT_TOKEN whose echoed params carry + * a nonzero token still proceeds (the server writes the token before the + * notifier update, kernel_channel.c:3471-3477). */ static int gsp_phys_rmcall_stage4(void __iomem *bar0) { @@ -5056,10 +5259,13 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; /* NVA06F engineType */ + u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; /* NVC36F index */ u8 sched_params[2] = { 1, 0 }; /* NVA06F bEnable=1, bSkipSubmit=0 */ resource_size_t bar0_len; u32 *pb_va, *userd, *ring_va; u32 words, gpput, gpget, slot, e0, e1, poll; + u32 token, rpc_result, status, runlist_id, ch_id, doorbell; u64 pb_dma, sem_dma, src_dma, dst_dma; unsigned long deadline; int ret; @@ -5086,27 +5292,78 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) return -EIO; } + /* (a) BIND the channel to the COPY0 engine (nvidia-push-init.c:343-353). */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, + &bind_params, sizeof(bind_params), 16, + NULL, 0, &rpc_result, &status, + "BIND_CHANNEL"); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n", + bind_params, rpc_result, status); + return ret; + } + + /* (b) GPFIFO_SCHEDULE bEnable=1, bSkipSubmit=0 (existing step). */ + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, - NULL, 0, NULL, "GPFIFO_SCHEDULE"); - if (ret) + NULL, 0, &rpc_result, &status, + "GPFIFO_SCHEDULE"); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n", + rpc_result, status); return ret; + } - if (!boot_stage.rmcall_token) { - u32 token = 0; + /* (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316). */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, + ¬if_index, sizeof(notif_index), 16, + NULL, 0, &rpc_result, &status, + "SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) failed (rpc_result=0x%x status=0x%x)\n", + notif_index, rpc_result, status); + return ret; + } - ret = gsp_phys_rmcall_control( - bar0, boot_stage.rmcall_channel, - GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, - &token, sizeof(token), 28, &token, sizeof(token), - NULL, "GET_WORK_SUBMIT_TOKEN(after schedule)"); - if (ret) { - pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n"); - return ret; - } - boot_stage.rmcall_token = token; + /* + * (d) GET_WORK_SUBMIT_TOKEN. The helper copies the echoed params into + * token before checking status, so a nonzero token can be recovered + * from a failed (0x40) reply: the server writes the token into the + * caller's params before the notifier update (kernel_channel.c:3471- + * 3477). A zero token on failure still aborts. + */ + token = 0; + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, + sizeof(token), &rpc_result, &status, + "GET_WORK_SUBMIT_TOKEN"); + if (ret && token != 0) { + pr_warn("gsp-phys-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n", + rpc_result, status, token); + ret = 0; } + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n", + notif_index, rpc_result, status, token); + return ret; + } + if (!token) { + pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n"); + return -EIO; + } + boot_stage.rmcall_token = token; pb_va = gsp_phys_chunk_page_va(pb, 0); ring_va = gsp_phys_chunk_page_va(ring, 0); @@ -5142,12 +5399,26 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) dma_wmb(); put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + /* + * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | + * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated + * token already is that value; log its runlist/ChID decomposition and + * write the formula's result (identical unless the token carries stray + * high bits, which is logged). + */ + runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; + ch_id = boot_stage.rmcall_token & 0xfffu; + doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); + if (doorbell != boot_stage.rmcall_token) + pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + boot_stage.rmcall_token, doorbell); wmb(); - writel(boot_stage.rmcall_token, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); + writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); - pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x doorbell@0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", pb_dma, words, slot, gpget, gpput, e0, e1, - boot_stage.rmcall_token, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, + boot_stage.rmcall_token, runlist_id, ch_id, doorbell, + GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS); @@ -5320,7 +5591,9 @@ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) /* * Free every ladder chunk and clear all ladder state, so a re-prime in the * same module load starts from zero (no stale server handles/tokens/stage). - * Safe on a partially built ladder. + * Safe on a partially built ladder. The FBMEM error-notifier is a server + * object like the VAS/TSG/channel (no host chunk); only its state is cleared, + * teardown is UNLOAD + rmmod. */ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) { @@ -5419,7 +5692,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); seq_printf(m, - "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x\n", + "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x notifier=%s base=0x%llx\n", boot_stage.rmcall_ran, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret, boot_stage.rmcall_rpcs, boot_stage.rmcall_client, @@ -5427,7 +5700,9 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, - boot_stage.rmcall_token); + boot_stage.rmcall_token, + boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", + boot_stage.rmcall_notifier_base); seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), @@ -6271,6 +6546,12 @@ static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56, "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56"); static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24, "NV_MEMORY_DESC_PARAMS must be 24 bytes"); +static_assert(sizeof(struct gsp_phys_rmcall_memory_alloc_params) == 128, + "NV_MEMORY_ALLOCATION_PARAMS must be 128 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, size) == 64, + "NV_MEMORY_ALLOCATION_PARAMS.size must be at offset 64"); +static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) == 80, + "NV_MEMORY_ALLOCATION_PARAMS.offset [IN/OUT] must be at offset 80"); static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48, "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32, @@ -6301,6 +6582,10 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_gro "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244, "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, error_notifier_mem) == 248, + "NV_CHANNEL_ALLOC_PARAMS.errorNotifierMem (FBMEM on GSP) must be at offset 248"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ecc_error_notifier_mem) == 272, + "NV_CHANNEL_ALLOC_PARAMS.eccErrorNotifierMem must be at offset 272"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296, "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, |