summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c659
-rw-r--r--kernel/gsp_phys_gpu.c28
-rw-r--r--kernel/gsp_phys_gpu.h18
3 files changed, 666 insertions, 39 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index c2d4b95..e434910 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -237,6 +237,18 @@ bool gsp_phys_boot_requested(void)
* leading candidate for the 0x40 the V6 KMD promote returns.
*/
#define GSP_PHYS_RMCALL_CLASS_CTXSHARE 0x9067u
+/*
+ * Golden-image replay classes (kernel_graphics.c kgraphicsCreateGoldenImage
+ * Channel_IMPL:2042-2453): the GPFIFO pushbuffer's physical object is
+ * NV01_MEMORY_SYSTEM 0x3e (class/cl003e.h), its virtual mapping
+ * NV50_MEMORY_VIRTUAL 0x50a0 (class/cl50a0.h) and the 3D engine object
+ * TURING_A 0xc597 (class/clc597.h; kgrmgrGetGrObjectType maps TURING_A to
+ * GR_OBJECT_TYPE_3D, kernel_graphics_manager.c:127-134, and
+ * kgraphicsGetClassByType resolves the class from the GPU's GR class list).
+ */
+#define GSP_PHYS_RMCALL_CLASS_MEMORY_SYSTEM 0x003eu
+#define GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL_50 0x50a0u
+#define GSP_PHYS_RMCALL_CLASS_GR_3D 0xc597u
#define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN 0xc36f0108u
#define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au
#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u
@@ -485,8 +497,25 @@ bool gsp_phys_boot_requested(void)
* GSP_PHYS_RMCALL_HANDLE_CTX_FB (allocated by the ctx-buffer prep step).
*/
#define GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB 23u
+/*
+ * Mode 24 (2026-10-05, untested): the reference FWCLIENT non-3D promote set
+ * (kgrobjGetPromoteIds_FWCLIENT: MAIN(0), PM(1), PATCH(2), FECS_EVENT(9),
+ * PRIV_ACCESS_MAP(10), UNRESTRICTED_PRIV_ACCESS_MAP(11), in that order) as
+ * ONE GPU_PROMOTE_CTX on the V6 KMD-managed channel with the external VAS and
+ * the default registry. Each entry carries the combined kgrobjPromoteContext
+ * form (bInitialize=1 with PA+size+physAttr, gpuVirtAddr set, bNonmapped=0)
+ * where the ladder has a VA: MAIN at the PDB-mapped ctx VA, PATCH at
+ * VA_GR_PATCH and FECS/UNRESTRICTED at booking-only VAs (mode-19 precedent;
+ * not PDB-mapped). PM has no VA source in the ladder - the reference
+ * external-VAS client sends it init-only (bNonmapped=1) - and the captured
+ * stock shape for PRIV_ACCESS_MAP is init-only too, so both are emitted
+ * bInitialize=1/bNonmapped=1 with no VA and the deviation is logged per
+ * entry. Buffers that cannot be sourced are omitted with a warning and
+ * entryCount shrinks to the number of real entries.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT 24u
#define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
- (GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB)
+ (GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)
/*
* Work-submit token notifier slot. The default index is
@@ -518,6 +547,25 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB 0x4u
/*
+ * NVOS32 attribute/flag words the golden-image replay packs, exactly as the
+ * stock client's golden-channel allocation builds them
+ * (kernel_graphics.c:2207-2308):
+ * ATTR_LOCATION_PCI = 1 at field 26:25 -> 1 << 25 (nvos.h:1105-1107);
+ * ATTR2_MEMORY_PROTECTION_UNPROTECTED = 2 at field 26:25 -> 2 << 25
+ * (nvos.h:1332-1334);
+ * ATTR_ALLOCATE_FROM_RESERVED_HEAP_YES = 1 at field 14:14 (nvos.h:1036-1038);
+ * ATTR2_INTERNAL_YES = 1 at field 14:14 (nvos.h:1225-1227);
+ * ALLOC_FLAGS_VIRTUAL = 0x80000 (nvos.h:1490);
+ * ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN = 0x4 (nvos.h:1472).
+ */
+#define GSP_PHYS_RMCALL_OS32_ATTR_PCI (1u << 25)
+#define GSP_PHYS_RMCALL_OS32_ATTR2_PROT_UNPROTECTED (2u << 25)
+#define GSP_PHYS_RMCALL_OS32_ATTR_RESERVED_HEAP (1u << 14)
+#define GSP_PHYS_RMCALL_OS32_ATTR2_INTERNAL (1u << 14)
+#define GSP_PHYS_RMCALL_OS32_FLAG_VIRTUAL 0x00080000u
+#define GSP_PHYS_RMCALL_OS32_FLAG_FORCE_GROWS_DOWN 0x00000004u
+
+/*
* FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3
* (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is
* the smallest deterministic size this ladder needs and keeps vaBase 0. The
@@ -530,6 +578,12 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED BIT(3)
#define GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT BIT(2)
#define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS BIT(8)
+/*
+ * NV_VASPACE_ALLOCATION_FLAGS_PTETABLE_HEAP_MANAGED BIT(13) (nvos.h:2783):
+ * the only flag the golden-image VAS alloc sets
+ * (kernel_graphics.c:2195-2200); the ladder's own VAS never sets it.
+ */
+#define GSP_PHYS_RMCALL_VAS_FLAG_PTETABLE_HEAP_MANAGED BIT(13)
#define GSP_PHYS_RMCALL_VAS_SIZE 0x10000000ULL
/*
* Shared-management VAS size: the normal KMD/UVM client passes vaBase=0 and
@@ -690,6 +744,31 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_HANDLE_MEM_VIRT 0xcaf00030u
#define GSP_PHYS_RMCALL_HANDLE_CTX_FB 0xcaf00038u
#define GSP_PHYS_RMCALL_HANDLE_CTXSHARE 0xcaf0003cu
+/*
+ * Golden-image replay handles (gsp_phys_rmcall_golden=1): the six objects the
+ * replay creates, free in the same 0xcaf0xxxx client handle range as every
+ * other ladder object (no collision with the handles above, and below the
+ * 0xcaf01000 registered-buffer base).
+ */
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS 0xcaf00044u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS 0xcaf00048u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT 0xcaf0004cu
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD 0xcaf00050u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL 0xcaf00054u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ 0xcaf00058u
+#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER 0xcaf0005cu
+
+/*
+ * Golden-image replay sizes (kernel_graphics.c:2058-2060, 2273-2276): the
+ * GPFIFO pushbuffer is 32 entries x NVA06F_GP_ENTRY__SIZE (8 B,
+ * class/clc46f.h:264) = 256 B, and the client USERD is one 4 KB page (the
+ * reference allocates sizeof(Nvc46fControl), 0x200; one page covers it, the
+ * same size as the ladder's own USERD chunk).
+ */
+#define GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES 32u
+#define GSP_PHYS_RMCALL_GOLDEN_PB_SIZE \
+ (GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES * 8u)
+#define GSP_PHYS_RMCALL_GOLDEN_USERD_SIZE GSP_PHYS_PAGE_SIZE
/*
* NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225);
@@ -6676,15 +6755,30 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0)
}
/*
- * Allocate a server-owned FBMEM buffer through GSP_RM_ALLOC class
- * NV01_MEMORY_LOCAL_USER (0x40), same shape as the error-notifier
- * allocation. Mode 11 uses it for the GR context buffer so the KMD promote
- * path sees a VIDMEM context memdesc; returns the frame-buffer base offset
- * the server echoed in NV_MEMORY_ALLOCATION_PARAMS.offset.
+ * Shape of one generic memory-object allocation through GSP_RM_ALLOC:
+ * NV_MEMORY_ALLOCATION_PARAMS carries the caller's class and NVOS32
+ * attr/attr2/flags/hVASpace/internalflags. The class-0x40 FBMEM wrapper below
+ * (mode 11/15/23 GR context buffer) and the golden replay's 0x3e/0x50a0
+ * objects share this body; the echoed params carry the allocated FB offset
+ * in .offset.
*/
-static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle,
- u32 size, u64 *out_base,
- const char *what)
+struct gsp_phys_rmcall_mem_obj_shape {
+ u32 handle;
+ u32 h_class;
+ u64 size;
+ u32 attr;
+ u32 attr2;
+ u32 flags;
+ u32 h_va_space;
+ u32 internalflags;
+};
+
+static int gsp_phys_rmcall_alloc_mem_obj(void __iomem *bar0,
+ const struct gsp_phys_rmcall_mem_obj_shape *s,
+ u64 *out_base, u32 *out_rpc_result,
+ u32 *out_status, const char *what,
+ const char *kind,
+ const char *base_word)
{
struct gsp_phys_rmcall_memory_alloc_params mem;
struct gsp_phys_rmcall_rm_alloc_req *req;
@@ -6694,40 +6788,47 @@ static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle,
u64 base;
int ret;
+ if (out_status)
+ *out_status = 0;
memset(&mem, 0, sizeof(mem));
mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC);
mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE);
- mem.size = cpu_to_le64(size);
- mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED);
- mem.internalflags =
- cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB);
+ mem.size = cpu_to_le64(s->size);
+ mem.attr = cpu_to_le32(s->attr);
+ mem.attr2 = cpu_to_le32(s->attr2);
+ mem.flags = cpu_to_le32(s->flags);
+ mem.h_va_space = cpu_to_le32(s->h_va_space);
+ mem.internalflags = cpu_to_le32(s->internalflags);
req = kvzalloc(req_len, GFP_KERNEL);
if (!req)
return -ENOMEM;
req->h_client = cpu_to_le32(boot_stage.rmcall_client);
req->h_parent = cpu_to_le32(boot_stage.rmcall_device);
- req->h_object = cpu_to_le32(handle);
- req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER);
+ req->h_object = cpu_to_le32(s->handle);
+ req->h_class = cpu_to_le32(s->h_class);
req->params_size = cpu_to_le32(sizeof(mem));
memcpy(req->params, &mem, sizeof(mem));
ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
- req, req_len, 20, &rec, true, NULL);
+ req, req_len, 20, &rec, true,
+ out_rpc_result);
kvfree(req);
if (ret) {
- pr_err("vgsp-boot: rm-call: %s FBMEM alloc (hObject=0x%x class=0x40 size=0x%x) RPC failed: %d\n",
- what, handle, size, ret);
+ pr_err("vgsp-boot: rm-call: %s %s alloc (hObject=0x%x class=0x%x size=0x%x) RPC failed: %d\n",
+ what, kind, s->handle, s->h_class, (u32)s->size, ret);
gsp_phys_rx_record_free(&rec);
return ret;
}
status = gsp_phys_rmcall_reply_u32(&rec, 16);
+ if (out_status)
+ *out_status = status;
if (status != 0 ||
rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params,
offset) + 8) {
- pr_err("vgsp-boot: rm-call: %s FBMEM alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
- what, status, rec.payload_len,
+ pr_err("vgsp-boot: rm-call: %s %s alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
+ what, kind, status, rec.payload_len,
offsetof(struct gsp_phys_rmcall_memory_alloc_params,
offset) + 8);
gsp_phys_rx_record_free(&rec);
@@ -6738,13 +6839,41 @@ static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle,
offset));
gsp_phys_rx_record_free(&rec);
- *out_base = base;
- pr_info("vgsp-boot: rm-call: %s FBMEM allocated: hObject=0x%x class=0x40 size=0x%x FB base=0x%llx\n",
- what, handle, size, base);
+ if (out_base)
+ *out_base = base;
+ pr_info("vgsp-boot: rm-call: %s %s allocated: hObject=0x%x class=0x%x size=0x%x %s=0x%llx\n",
+ what, kind, s->handle, s->h_class, (u32)s->size,
+ base_word, base);
return 0;
}
/*
+ * Allocate a server-owned FBMEM buffer through GSP_RM_ALLOC class
+ * NV01_MEMORY_LOCAL_USER (0x40), same shape as the error-notifier
+ * allocation. Modes 11/15/23 use it for the GR context buffer so the KMD
+ * promote path sees a VIDMEM context memdesc; returns the frame-buffer base
+ * offset the server echoed in NV_MEMORY_ALLOCATION_PARAMS.offset.
+ */
+static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle,
+ u32 size, u64 *out_base,
+ const char *what)
+{
+ const struct gsp_phys_rmcall_mem_obj_shape shape = {
+ .handle = handle,
+ .h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER,
+ .size = size,
+ .attr = GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED,
+ .attr2 = 0,
+ .flags = 0,
+ .h_va_space = 0,
+ .internalflags = GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB,
+ };
+
+ return gsp_phys_rmcall_alloc_mem_obj(bar0, &shape, out_base, NULL,
+ NULL, what, "FBMEM", "FB base");
+}
+
+/*
* One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
* payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional)
* receives the server control status even when it is nonzero; the echoed
@@ -7110,6 +7239,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry bare VA+size promote";
case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT:
return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
+ case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT:
+ return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6";
default:
return "off (PROMOTE_CTX never sent)";
}
@@ -7291,7 +7422,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
boot_stage.rmcall_gr_promote ==
GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
boot_stage.rmcall_gr_promote ==
- GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT)) {
+ GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
+ boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)) {
vars = &gsp_phys_rmcall_channel_variant_skipctx;
attempts = 1;
}
@@ -7412,8 +7545,17 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
* NV_GR_ALLOCATION_PARAMETERS {version=2, flags=0, size=0x10, caps=0}; if
* the server rejects them the alloc helper dumps the failed reply and the
* ladder aborts with no retry (the H1 notes carry the bisection order).
+ *
+ * The class/handle/parent are parameters so the golden replay can allocate
+ * TURING_A (0xc597) on its own channel through the same body; the ladder
+ * caller below keeps the exact historical behavior.
*/
-static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
+static int gsp_phys_rmcall_alloc_gr_object_class(void __iomem *bar0,
+ u32 h_class, u32 handle,
+ u32 h_parent,
+ const char *what,
+ u32 *out_rpc_result,
+ u32 *out_status)
{
struct gsp_phys_rmcall_gr_alloc_params gr;
u32 status = 0;
@@ -7425,17 +7567,27 @@ static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
gr.caps = cpu_to_le32(0);
ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
- boot_stage.rmcall_channel,
- GSP_PHYS_RMCALL_HANDLE_GR_OBJ,
- GSP_PHYS_RMCALL_CLASS_COMPUTE,
- &gr, sizeof(gr), false, NULL,
- "TURING_COMPUTE_A", true, NULL, &status);
- pr_info("vgsp-boot: rm-call: TURING_COMPUTE_A alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n",
- GSP_PHYS_RMCALL_CLASS_COMPUTE, status,
- GSP_PHYS_RMCALL_HANDLE_GR_OBJ, boot_stage.rmcall_channel,
+ h_parent, handle, h_class, &gr, sizeof(gr),
+ false, NULL, what, true,
+ out_rpc_result, &status);
+ pr_info("vgsp-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n",
+ what, h_class, status, handle, h_parent,
GSP_PHYS_RMCALL_GR_PARAM_VERSION, 0u,
GSP_PHYS_RMCALL_GR_PARAM_SIZE, 0u,
ret ? " (rejected)" : "");
+ if (out_status)
+ *out_status = status;
+ return ret;
+}
+
+static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
+{
+ int ret;
+ ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
+ GSP_PHYS_RMCALL_CLASS_COMPUTE,
+ GSP_PHYS_RMCALL_HANDLE_GR_OBJ,
+ boot_stage.rmcall_channel,
+ "TURING_COMPUTE_A", NULL, NULL);
if (ret)
return ret;
@@ -7444,6 +7596,222 @@ static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
}
/*
+ * gsp_phys_rmcall_golden=1: allocate a TURING_A (0xc597) 3D object on the
+ * ladder's own channel before the compute object, mirroring the stock
+ * client's golden-image channel GR-object construct; the server-side
+ * GR-instance init state the engine-context commit needs is a by-product of
+ * that construct. Best-effort by design: a failure is logged and the compute
+ * object allocation still runs (the ladder's own promote path then keeps its
+ * historical behavior, no retry).
+ */
+static void gsp_phys_rmcall_golden_3d_on_ladder(void __iomem *bar0)
+{
+ u32 rpc = 0, status = 0;
+ int ret;
+
+ if (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled() ||
+ !boot_stage.rmcall_channel)
+ return;
+ ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
+ GSP_PHYS_RMCALL_CLASS_GR_3D,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER,
+ boot_stage.rmcall_channel,
+ "GOLDEN TURING_A on ladder channel", &rpc, &status);
+ if (ret)
+ pr_warn("vgsp-boot: rm-call: GOLDEN TURING_A 3D object on the ladder channel failed (%d, status=0x%x rpc_result=0x%x); continuing to the compute object with no retry\n",
+ ret, status, rpc);
+}
+
+/*
+ * Golden-image replay (gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1): the
+ * stock client creates the golden-image channel during FIFO post-load
+ * (kernel_graphics.c kgraphicsCreateGoldenImageChannel_IMPL:2042-2453) and
+ * that construct is what initializes the server-side GR-instance state the GR
+ * engine-context commit needs (the KernelGraphics byte +0x158 and
+ * graphics-context byte +0x135 the ladder observed as 0). Stage 3 replays the
+ * same allocations with the ladder's client and device under fresh handles
+ * before the ladder's own channel:
+ * 1. a separate FERMI_VASPACE_A with PTETABLE_HEAP_MANAGED
+ * (kernel_graphics.c:2194-2200);
+ * 2. the GPFIFO pushbuffer as a physical NV01_MEMORY_SYSTEM (0x3e) object,
+ * owner HEAP_OWNER_RM_CLIENT_GENERIC, type IMAGE, LOCATION_PCI,
+ * MEMORY_PROTECTION_UNPROTECTED, size 32 entries x 8 B
+ * (kernel_graphics.c:2202-2222);
+ * 3. its NV50_MEMORY_VIRTUAL (0x50a0) mapping on that VAS with
+ * NVOS32_ALLOC_FLAGS_VIRTUAL (kernel_graphics.c:2224-2235);
+ * 4. the client USERD (class 0x40), VIDMEM internal reserved-heap,
+ * FORCE_MEM_GROWS_DOWN (kernel_graphics.c:2237-2313);
+ * 5. a device-parented TURING_CHANNEL_GPFIFO_A (0xc46f) with hVASpace, the
+ * virtual PB as hObjectBuffer, 32 gpFifoEntries, gpFifoOffset 0, the
+ * USERD in hUserdMemory[0] and engineType GR0
+ * (kernel_graphics.c:2316-2363);
+ * 6. a TURING_A (0xc597, the GR_OBJECT_TYPE_3D class) object on that
+ * channel with the ladder's NV_GR_ALLOCATION_PARAMETERS shape
+ * (kernel_graphics.c:2409-2433).
+ * Each alloc logs one GOLDEN summary line with class/handle/status/rpc_result;
+ * the first failure aborts the whole ladder with no retry, like every other
+ * stage-3 pre-step. With the parameter 0 this function is never called.
+ */
+static void gsp_phys_rmcall_golden_log(const char *step, u32 handle,
+ u32 h_class, u32 status, u32 rpc,
+ int ret)
+{
+ pr_info("vgsp-boot: rm-call: GOLDEN %s: hObject=0x%x hClass=0x%x status=0x%x rpc_result=0x%x ret=%d\n",
+ step, handle, h_class, status, rpc, ret);
+}
+
+static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
+ u32 h_class, u32 status, u32 rpc,
+ int ret)
+{
+ pr_err("vgsp-boot: rm-call: GOLDEN replay aborted: %s failed: hObject=0x%x hClass=0x%x status=0x%x rpc_result=0x%x ret=%d; the ladder aborts with no retry\n",
+ step, handle, h_class, status, rpc, ret);
+}
+
+static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
+ const struct gsp_phys_rmcall_mem_obj_shape *shape,
+ const char *step)
+{
+ char what[64];
+ u32 rpc = 0, status = 0;
+ int ret;
+
+ scnprintf(what, sizeof(what), "GOLDEN %s", step);
+ ret = gsp_phys_rmcall_alloc_mem_obj(bar0, shape, NULL, &rpc, &status,
+ what, "mem", "base");
+ gsp_phys_rmcall_golden_log(step, shape->handle, shape->h_class,
+ status, rpc, ret);
+ if (ret)
+ gsp_phys_rmcall_golden_fail(step, shape->handle, shape->h_class,
+ status, rpc, ret);
+ return ret;
+}
+
+static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
+{
+ struct gsp_phys_rmcall_vas_params va;
+ struct gsp_phys_rmcall_channel_params ch;
+ struct gsp_phys_rmcall_mem_obj_shape shape;
+ u32 rpc = 0, status = 0;
+ int ret;
+
+ /* 1. Golden VAS: PTETABLE_HEAP_MANAGED only (kernel_graphics.c:2194-2200). */
+ memset(&va, 0, sizeof(va));
+ va.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_PTETABLE_HEAP_MANAGED);
+ ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+ boot_stage.rmcall_device,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+ GSP_PHYS_RMCALL_CLASS_VASPACE,
+ &va, sizeof(va), false, NULL,
+ "GOLDEN FERMI_VASPACE_A", true, &rpc,
+ &status);
+ gsp_phys_rmcall_golden_log("FERMI_VASPACE_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+ GSP_PHYS_RMCALL_CLASS_VASPACE, status,
+ rpc, ret);
+ if (ret) {
+ gsp_phys_rmcall_golden_fail("FERMI_VASPACE_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+ GSP_PHYS_RMCALL_CLASS_VASPACE,
+ status, rpc, ret);
+ return ret;
+ }
+
+ /* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
+ memset(&shape, 0, sizeof(shape));
+ shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS;
+ shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_SYSTEM;
+ shape.size = GSP_PHYS_RMCALL_GOLDEN_PB_SIZE;
+ shape.attr = GSP_PHYS_RMCALL_OS32_ATTR_PCI;
+ shape.attr2 = GSP_PHYS_RMCALL_OS32_ATTR2_PROT_UNPROTECTED;
+ ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "PB physical");
+ if (ret)
+ return ret;
+
+ /* 3. Its virtual mapping on the golden VAS (kernel_graphics.c:2224-2235). */
+ shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT;
+ shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL_50;
+ shape.flags = GSP_PHYS_RMCALL_OS32_FLAG_VIRTUAL;
+ shape.h_va_space = GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS;
+ ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "PB virtual");
+ if (ret)
+ return ret;
+
+ /* 4. Client USERD, internal reserved-heap VIDMEM (kernel_graphics.c:2237-2313). */
+ shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD;
+ shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER;
+ shape.size = GSP_PHYS_RMCALL_GOLDEN_USERD_SIZE;
+ shape.attr = GSP_PHYS_RMCALL_OS32_ATTR_RESERVED_HEAP;
+ shape.attr2 = GSP_PHYS_RMCALL_OS32_ATTR2_INTERNAL;
+ shape.flags = GSP_PHYS_RMCALL_OS32_FLAG_FORCE_GROWS_DOWN;
+ shape.h_va_space = 0;
+ ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "USERD");
+ if (ret)
+ return ret;
+
+ /*
+ * 5. Bare GR channel on the device with the golden VAS/PB/USERD
+ * (kernel_graphics.c:2316-2363). No TSG: the stock golden channel is
+ * device-parented and passes hVASpace explicitly (only TSG channels
+ * must pass hVASpace 0, kernel_channel.c:429-435).
+ */
+ memset(&ch, 0, sizeof(ch));
+ ch.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
+ ch.h_object_buffer = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT);
+ ch.gp_fifo_entries =
+ cpu_to_le32(GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
+ ch.gp_fifo_offset = cpu_to_le64(0);
+ ch.h_userd_memory[0] =
+ cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD);
+ ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+ ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
+ boot_stage.rmcall_device,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ GSP_PHYS_RMCALL_CLASS_CHANNEL,
+ &ch, sizeof(ch), false, NULL,
+ "GOLDEN TURING_CHANNEL_GPFIFO_A", true,
+ &rpc, &status);
+ gsp_phys_rmcall_golden_log("TURING_CHANNEL_GPFIFO_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ GSP_PHYS_RMCALL_CLASS_CHANNEL, status,
+ rpc, ret);
+ if (ret) {
+ gsp_phys_rmcall_golden_fail("TURING_CHANNEL_GPFIFO_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ GSP_PHYS_RMCALL_CLASS_CHANNEL,
+ status, rpc, ret);
+ return ret;
+ }
+
+ /* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
+ ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
+ GSP_PHYS_RMCALL_CLASS_GR_3D,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ "GOLDEN TURING_A", &rpc, &status);
+ gsp_phys_rmcall_golden_log("TURING_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
+ GSP_PHYS_RMCALL_CLASS_GR_3D, status,
+ rpc, ret);
+ if (ret) {
+ gsp_phys_rmcall_golden_fail("TURING_A",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
+ GSP_PHYS_RMCALL_CLASS_GR_3D,
+ status, rpc, ret);
+ return ret;
+ }
+
+ pr_info("vgsp-boot: rm-call: GOLDEN replay complete: 6 objects (VAS=0x%x PBphys=0x%x PBvirt=0x%x USERD=0x%x channel=0x%x GR=0x%x)\n",
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ);
+ return 0;
+}
+
+/*
* GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The
* single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and
* the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE
@@ -7946,6 +8314,26 @@ static void gsp_phys_rmcall_promote_entry_fill(
}
/*
+ * Mode-24 per-entry log: one line per emitted FWCLIENT promote entry with
+ * every wire field (bufferId, pa, va, size, physAttr, bInitialize,
+ * bNonmapped) plus the caller's note describing any deviation from the
+ * combined reference form.
+ */
+static void
+gsp_phys_rmcall_gr_promote_log_entry(u32 idx,
+ const struct gsp_phys_rmcall_promote_entry *e,
+ const char *note)
+{
+ pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s%s\n",
+ idx, le16_to_cpu(e->buffer_id),
+ le64_to_cpu(e->gpu_phys_addr),
+ le64_to_cpu(e->gpu_virt_addr),
+ le64_to_cpu(e->size), le32_to_cpu(e->phys_attr),
+ e->b_initialize, e->b_nonmapped,
+ (note && note[0]) ? " - " : "", note ? note : "");
+}
+
+/*
* Auxiliary GR context-buffer prepare (PATCH/PM): query the save size for
* `engine_id` (GET_ENGINE_CONTEXT_PROPERTIES), allocate a zeroed physically
* contiguous sysmem buffer and record its PA/size. The real client allocates
@@ -8097,6 +8485,54 @@ fail_patch:
}
/*
+ * Mode-24 helper: allocate the FWCLIENT non-3D set's small sysmem buffers
+ * that have no dedicated prepare helper - FECS_EVENT, PRIV_ACCESS_MAP and
+ * UNRESTRICTED_PRIV_ACCESS_MAP (one 4 KB contiguous zeroed page each; the
+ * reference client's global context buffers, reusing the same
+ * boot_stage.rmcall_{fecs,priv,unrestricted} chunks mode 19 prepares).
+ * PATCH is prepared separately with the queried engine size. Every buffer is
+ * best-effort: a failed or non-contiguous allocation leaves its _pa zero and
+ * the caller omits that entry with a warning; the promote still runs with the
+ * remaining buffers. Idempotent: a chunk already prepared this staging is
+ * reused.
+ */
+static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
+{
+ struct {
+ struct gsp_phys_boot_chunk *chunk;
+ u64 *pa;
+ const char *name;
+ } bufs[] = {
+ { &boot_stage.rmcall_fecs, &boot_stage.rmcall_fecs_pa,
+ "FECS_EVENT" },
+ { &boot_stage.rmcall_priv, &boot_stage.rmcall_priv_pa,
+ "PRIV_ACCESS_MAP" },
+ { &boot_stage.rmcall_unrestricted,
+ &boot_stage.rmcall_unrestricted_pa,
+ "UNRESTRICTED_PRIV_ACCESS_MAP" },
+ };
+ unsigned int i;
+
+ if (!boot_stage.pdev)
+ return;
+ for (i = 0; i < ARRAY_SIZE(bufs); i++) {
+ if (*bufs[i].pa)
+ continue;
+ if (gsp_phys_chunk_alloc(boot_stage.pdev, bufs[i].chunk, 1u) ||
+ bufs[i].chunk->page_dma) {
+ pr_warn("vgsp-boot: rm-call: FWCLIENT %s buffer allocation failed (not physically contiguous); omitting the entry from the mode-24 promote\n",
+ bufs[i].name);
+ gsp_phys_chunk_free(boot_stage.pdev, bufs[i].chunk);
+ continue;
+ }
+ gsp_phys_chunk_zero(bufs[i].chunk);
+ *bufs[i].pa = (u64)bufs[i].chunk->dma;
+ pr_info("vgsp-boot: rm-call: FWCLIENT %s buffer prepared: pa=0x%llx size=0x1000 (sysmem contiguous zeroed)\n",
+ bufs[i].name, *bufs[i].pa);
+ }
+}
+
+/*
* Fill one MAIN promote entry with the prepared PA/VA and the snapshot's
* entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5
* entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the
@@ -8224,6 +8660,12 @@ static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0)
* - mode 11: mode 10 with the context buffer backing swapped to a
* server-owned FBMEM allocation (VIDMEM physAttr 0x4, local-aperture
* leaf PTEs) instead of host sysmem.
+ * - mode 24: the reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/
+ * PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP, kgrobjGetPromoteIds_
+ * FWCLIENT order) as one control on the V6 KMD-managed channel, each
+ * entry in the combined kgrobjPromoteContext form where the ladder has
+ * a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers
+ * are omitted, all logged per entry.
* Mode 0 never reaches this function (stage 3 stops before it); the early
* return is a belt-and-braces no-op. Success means the server installed the
* context memdesc, added the VA to the engine-context vaList, committed the
@@ -8273,7 +8715,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE ||
mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
- mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) &&
+ mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
+ mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) &&
boot_stage.rmcall_channel_variant !=
gsp_phys_rmcall_channel_variant_skipctx.id) {
pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
@@ -8953,6 +9396,137 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
return ret;
}
boot_stage.rmcall_gr_promote_step = 1;
+ } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) {
+ /*
+ * Mode 24: the reference FWCLIENT non-3D set as ONE control.
+ * Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT
+ * (kernel_graphics_object.c:571-643): MAIN, PM, PATCH,
+ * FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP.
+ * The entry form follows kgrobjPromoteContext
+ * (kernel_graphics_object.c:90-139): zeroed params, then per
+ * buffer the init half (kgrctxPrepareInitializeCtxBuffer:
+ * PA+size+physAttr, bInitialize=1, bNonmapped=1) and the
+ * promote half (kgrctxPreparePromoteCtxBuffer: gpuVirtAddr,
+ * bNonmapped=0) combined into one entry where the ladder has
+ * a VA. PM has no VA source (the reference external-VAS
+ * client sends it init-only) and PRIV_ACCESS_MAP's captured
+ * stock shape is init-only too; both deviations are logged
+ * per entry. Auxiliary buffers that cannot be sourced are
+ * omitted and logged; entryCount is the number of real
+ * entries.
+ */
+ u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+ u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
+ u32 n = 0;
+
+ /*
+ * PATCH is always present in the reference set; a failed
+ * prepare aborts before any control is sent.
+ */
+ ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
+ if (ret) {
+ pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient aborted: PATCH prepare failed (%d); the reference set always carries PATCH, so no promote is sent and the ladder aborts with no retry\n",
+ ret);
+ kvfree(p);
+ return ret;
+ }
+ if (gsp_phys_rmcall_gr_pm_prepare(bar0))
+ pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PM prepare failed; omitting the PM entry and continuing (mode 24 treats PM as best-effort; the reference set pre-allocates it for an externally-owned VAS)\n");
+ gsp_phys_rmcall_gr_fwclient_small_prepare();
+
+ /* Entry 0: MAIN - combined PA+VA, PDB-mapped ctx VA. */
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
+ boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+ boot_stage.rmcall_ctx_size, attr, 1, 0);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "combined (PDB-mapped ctx VA)");
+ n++;
+
+ /* Entry 1: PM - init-only, no PM VA source in the ladder. */
+ if (boot_stage.rmcall_pm_prepared &&
+ boot_stage.rmcall_pm_pa && boot_stage.rmcall_pm_size) {
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM,
+ boot_stage.rmcall_pm_pa, 0,
+ boot_stage.rmcall_pm_size, attr, 1, 1);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "init-only, no PM VA source (reference external-VAS shape)");
+ n++;
+ } else {
+ pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PM omitted (no queried buffer; the reference external-VAS client pre-allocates it)\n");
+ }
+
+ /* Entry 2: PATCH - combined, booking-only VA. */
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
+ boot_stage.rmcall_patch_pa,
+ GSP_PHYS_RMCALL_VA_GR_PATCH,
+ boot_stage.rmcall_patch_size, attr, 1, 0);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "combined (booking-only VA, not PDB-mapped)");
+ n++;
+
+ /* Entry 3: FECS_EVENT - combined, booking-only VA. */
+ if (boot_stage.rmcall_fecs_pa) {
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
+ boot_stage.rmcall_fecs_pa, bk + 0x6000ULL,
+ 0x1000u, attr, 1, 0);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "combined (booking-only VA, not PDB-mapped)");
+ n++;
+ } else {
+ pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: FECS_EVENT omitted (allocation unavailable)\n");
+ }
+
+ /* Entry 4: PRIV_ACCESS_MAP - init-only, captured stock shape. */
+ if (boot_stage.rmcall_priv_pa) {
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
+ boot_stage.rmcall_priv_pa, 0, 0x1000u, attr,
+ 1, 1);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "init-only (captured stock shape; sysmem attr 0x6, reference FB uncached 0x4)");
+ n++;
+ } else {
+ pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PRIV_ACCESS_MAP omitted (allocation unavailable)\n");
+ }
+
+ /* Entry 5: UNRESTRICTED_PRIV_ACCESS_MAP - combined. */
+ if (boot_stage.rmcall_unrestricted_pa) {
+ gsp_phys_rmcall_promote_entry_fill(p, n,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
+ boot_stage.rmcall_unrestricted_pa,
+ bk + 0x7000ULL, 0x1000u, attr, 1, 0);
+ gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
+ "combined (booking-only VA, not PDB-mapped)");
+ n++;
+ } else {
+ pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: UNRESTRICTED_PRIV_ACCESS_MAP omitted (allocation unavailable)\n");
+ }
+
+ p->entry_count = cpu_to_le32(n);
+ rpc_result = 0;
+ status = 0;
+ ret = gsp_phys_rmcall_control(bar0,
+ boot_stage.rmcall_subdevice,
+ GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+ p, sizeof(*p), 24, NULL, 0,
+ &rpc_result, &status,
+ "GPU_PROMOTE_CTX (FWCLIENT set)");
+ pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient: entryCount=%u status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
+ n, status, rpc_result, boot_stage.rmcall_channel,
+ boot_stage.rmcall_client,
+ GSP_PHYS_RMCALL_ENGINE_GR0, mode);
+ if (ret) {
+ pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient failed (status=0x%x rpc_result=0x%x, %s, ret=%d); entries emitted=%u; the GR context is not committed and the ladder aborts before BIND with no retry\n",
+ status, rpc_result,
+ gsp_phys_rmcall_status_str(status), ret, n);
+ kvfree(p);
+ return ret;
+ }
+ boot_stage.rmcall_gr_promote_step = 1;
} else {
pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
mode);
@@ -9545,6 +10119,22 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;
/*
+ * Golden-image replay (gsp_phys_rmcall_golden=1 + gsp_phys_gpu=1):
+ * replay the stock client's post-load golden channel allocations after
+ * the ladder's VAS/TSG exist and before the ladder's own channel/GR
+ * object, so the server-side GR-instance state the GR engine-context
+ * commit needs is initialized first. The first failure aborts the whole
+ * ladder with no retry; with the parameter 0 this block is skipped
+ * entirely (no RPC, no log).
+ */
+ if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
+ ret = gsp_phys_rmcall_golden_replay(bar0);
+ if (ret)
+ pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
+ ret);
+ }
+
+ /*
* Mode 18: allocate the client context share (and with it the
* GraphicsSubcontext) after the TSG and before the channel, so the
* channel attaches to it via hContextShare instead of the legacy
@@ -9587,6 +10177,7 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
* this whole block is skipped entirely: no RPC, no log.
*/
if (gsp_phys_rmcall_engine_is_gr()) {
+ gsp_phys_rmcall_golden_3d_on_ladder(bar0);
ret = gsp_phys_rmcall_alloc_gr_object(bar0);
if (ret) {
pr_err("vgsp-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n",
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 0366380..cd04007 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -228,6 +228,28 @@ int gsp_phys_rmcall_channel_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_channel_value);
/*
+ * Golden-image replay (default 0). When set (and gsp_phys_gpu=1), the rm-call
+ * ladder's stage 3 replays the stock client's post-load golden-image channel
+ * allocations (kgraphicsCreateGoldenImageChannel: a separate
+ * FERMI_VASPACE_A, the GPFIFO pushbuffer as physical NV01_MEMORY_SYSTEM plus
+ * its NV50_MEMORY_VIRTUAL mapping, the client USERD, a device-parented
+ * TURING_CHANNEL_GPFIFO_A and a TURING_A 3D object) before the ladder's own
+ * channel, so the server-side GR-instance state the GR engine-context commit
+ * needs exists. With 0 the ladder is byte-identical to before.
+ */
+static bool gsp_phys_rmcall_golden_param;
+module_param_named(gsp_phys_rmcall_golden, gsp_phys_rmcall_golden_param,
+ bool, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_golden,
+ "Replay the stock golden-image channel allocations in stage 3 before the ladder channel (default 0; requires gsp_phys_gpu=1)");
+
+bool gsp_phys_rmcall_golden_enabled(void)
+{
+ return gsp_phys_rmcall_golden_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_enabled);
+
+/*
* P5 wave-1 shadow work forwarding (default 0). When set, the sema service
* requests one real CE copy on the debugfs ladder's channel for each decoded
* guest CE push, using only the ladder's module-owned buffers; the software
@@ -470,8 +492,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
return ret;
if (gsp_phys_rmcall_gr_promote_param < 0 ||
- gsp_phys_rmcall_gr_promote_param > 23) {
- pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..23; clamping to 0 (no PROMOTE_CTX)\n",
+ gsp_phys_rmcall_gr_promote_param > 24) {
+ pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..24; clamping to 0 (no PROMOTE_CTX)\n",
gsp_phys_rmcall_gr_promote_param);
gsp_phys_rmcall_gr_promote_param = 0;
}
@@ -490,7 +512,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
&gsp_phys_rmcall_gr_promote_ops,
&gsp_phys_rmcall_gr_promote_param, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
- "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate); values outside 0..23 clamp to 0 with a warning");
+ "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); values outside 0..24 clamp to 0 with a warning");
int gsp_phys_rmcall_gr_promote_value(void)
{
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index ad99c7d..8322725 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -99,6 +99,13 @@ int gsp_phys_srv_ctxcommit_mode(void);
int gsp_phys_rmcall_channel_value(void);
/*
+ * gsp_phys_rmcall_golden (default 0): replay the stock client's golden-image
+ * channel allocations in stage 3 before the ladder's own channel when set and
+ * gsp_phys_gpu=1. With 0, the ladder is byte-identical to before.
+ */
+bool gsp_phys_rmcall_golden_enabled(void);
+
+/*
* True when the gsp_phys_work_forward module parameter is set (default 0: the
* guest's CE pushes stay software-emulated exactly as before). When set, the
* sema service shadows each decoded CE push as one copy on the debugfs ladder's
@@ -229,8 +236,15 @@ int gsp_phys_rmcall_engine_value(void);
* PROMOTE_CTX with hVirtMemory, virtAddress/size and entryCount=0, so
* the server creates the memdesc/vaList entry itself;
* 7 = no-entry bare (virtAddress, size): PROMOTE_CTX with entryCount=0 and
- * no hVirtMemory (the other documented no-entry branch).
- * The parameter's set callback clamps any value outside 0..7 to 0 with a
+ * no hVirtMemory (the other documented no-entry branch);
+ * 8..23 = later KMD-channel/old-interface/hVirtMemory/FBMEM bisection
+ * shapes; see gsp_phys_gpu.c MODULE_PARM_DESC for the full list;
+ * 24 = the reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/
+ * PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the
+ * V6 KMD-managed channel: combined PA+VA entries where the ladder has
+ * a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted
+ * and logged.
+ * The parameter's set callback clamps any value outside 0..24 to 0 with a
* warning and warns on every nonzero enable (a failed promote can assert
* GSP-RM and wedge the device; recovery needs a reboot with human approval).
* The booter snapshots the value when the rm-call ladder's stage 3 starts
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 15:07:43 +0000