summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c345
-rw-r--r--kernel/gsp_phys_gpu.c6
-rw-r--r--kernel/gsp_phys_gpu.h8
3 files changed, 344 insertions, 15 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 0c2e75a..a25a0f0 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -330,6 +330,14 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707u
#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO 0x20801109u
#define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX 0x2080012bu
+/*
+ * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106), the golden
+ * client's VAS construct control. Mode 25 sends it only in the gated
+ * zero-level no-op-range form (see
+ * GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); the captured three level PAs
+ * are server-RM-internal and have no safe client source.
+ */
+#define GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES 0x90f10106u
#define GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX 0x2080012cu
/*
* NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063):
@@ -514,8 +522,33 @@ bool gsp_phys_boot_requested(void)
* entryCount shrinks to the number of real entries.
*/
#define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT 24u
+/*
+ * Mode 25 (2026-10-05, opt-in): the real golden-client construct captured
+ * from the 570.144 host client (REPORT-GOLDEN-CAPTURE-DECODED), replayed on
+ * the golden handles in gsp_phys_rmcall_golden_replay: the captured VAS
+ * COPY_SERVER_RESERVED_PDES control (0x90f10106, gated by the fwclient mask
+ * bit 8 because the level PAs are server-internal), the captured channel
+ * shape (flags 0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries
+ * 0x20, engineType 1, host-buffer memdescs) and the single atomic 9-entry
+ * GPU_PROMOTE_CTX on the golden channel (MAIN id0 combined, PATCH id2
+ * combined, ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined;
+ * sysmem physAttr 0x6 is the documented deviation). The golden VAS/channel/3D
+ * object stay alive and the ladder's own V6 FWCLIENT promote then follows on
+ * the golden-initialized GR-instance state. Requires gsp_phys_rmcall_golden=1
+ * plus gsp_phys_rmcall_engine=1; a failed golden step aborts the ladder with
+ * no retry and no follow-up promote.
+ */
+#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 25u
+/*
+ * Existing fwclient mask bit 8: mode-25 gate for the captured VAS
+ * COPY_SERVER_RESERVED_PDES control. The captured control copies three
+ * server-RM-internal page-table levels the ladder cannot source safely, so
+ * mode 25 sends only the zero-level no-op-range form and only when this bit
+ * is set; the default mask 0x3f leaves it off (skipped, logged).
+ */
+#define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES (1u << 8)
#define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
- (GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)
+ (GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)
/*
* Work-submit token notifier slot. The default index is
@@ -793,6 +826,18 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31)
/*
+ * Golden-image channel flag word captured from the real 570.144 host client
+ * (0x00200320): PRIVILEGED_CHANNEL (bit 5), USERD_INDEX_VALUE=3 (bits 10:8,
+ * the client's chId % 8) and USERD_INDEX_PAGE_FIXED (bit 21). SKIP_SCRUBBER
+ * (bit 23) and SKIP_CTXBUFFER_ALLOC (bit 31) are deliberately clear: the real
+ * golden channel is client-managed through the registry, not through the
+ * channel flags.
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9 \
+ (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (3u << 8) | \
+ (1u << 21))
+
+/*
* BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
* GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
* RPC never replied, every later RPC timed out, and the fail-safe held the
@@ -6173,6 +6218,31 @@ struct gsp_phys_rmcall_promote_params {
};
/*
+ * NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS (184 bytes): hSubDevice,
+ * subDeviceId, pageSize, virtAddrLo, virtAddrHi, numLevelsToCopy and six level
+ * entries {physAddr, size, aperture, pageShift}. The ladder has no safe source
+ * for the server-RM level PAs the real client copied, so the gated mode-25
+ * path sends only the zero-level no-op-range form.
+ */
+struct gsp_phys_rmcall_vas_copy_pdes_level {
+ __le64 phys_addr;
+ __le64 size;
+ __le32 aperture;
+ __le32 page_shift;
+};
+
+struct gsp_phys_rmcall_vas_copy_pdes_params {
+ __le32 h_sub_device;
+ __le32 sub_device_id;
+ __le64 page_size;
+ __le64 virt_addr_lo;
+ __le64 virt_addr_hi;
+ __le32 num_levels_to_copy;
+ __le32 pad;
+ struct gsp_phys_rmcall_vas_copy_pdes_level levels[6];
+};
+
+/*
* NV2080_CTRL_GPU_EVICT_CTX_PARAMS (20 bytes,
* ctrl/ctrl2080/ctrl2080gpu.h:1000-1006): engineType/hClient/ChID/
* hChanClient/hObject. Evicts the channel's virtual engine context; on the
@@ -7241,6 +7311,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT:
return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6";
+ case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9:
+ return "real golden-client construct on the golden handles (captured channel 0x00200320 + single atomic 9-entry PROMOTE_CTX) then the V6 FWCLIENT follow-on promote";
default:
return "off (PROMOTE_CTX never sent)";
}
@@ -7248,7 +7320,8 @@ static const char *gsp_phys_rmcall_gr_promote_name(void)
/*
* H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a
- * load-time module parameter (0..13, see gsp_phys_gpu.c); stage 3 snapshots it
+ * load-time module parameter (0..GSP_PHYS_RMCALL_GR_PROMOTE_MAX, see
+ * gsp_phys_gpu.c); stage 3 snapshots it
* next to the engine so every later step of this staging uses one shape. The
* module parameter's set callback already clamps out-of-range values with a
* warning; the check here is a second line of defense against a future
@@ -7424,7 +7497,9 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
boot_stage.rmcall_gr_promote ==
GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
boot_stage.rmcall_gr_promote ==
- GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT)) {
+ GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+ boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9)) {
vars = &gsp_phys_rmcall_channel_variant_skipctx;
attempts = 1;
}
@@ -7668,6 +7743,48 @@ static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
step, handle, h_class, status, rpc, ret);
}
+/*
+ * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
+ * The captured real-client control copied three server-RM-internal page-table
+ * levels; the ladder has no safe source for those PAs, so the only form this
+ * path can send is the zero-level no-op range (numLevelsToCopy=0) and it does
+ * so only when the gate is set. With the default mask the control is skipped
+ * and the deviation is logged; this sub-step still needs a live bisection
+ * (report open item 3).
+ */
+static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
+{
+ struct gsp_phys_rmcall_vas_copy_pdes_params *p;
+ u32 rpc_result = 0, status = 0;
+ int ret;
+
+ if (!(gsp_phys_rmcall_fwclient_mask_value() &
+ GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
+ pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped: the captured three level PAs are server-RM-internal and the ladder has no safe source (uncertain, report open item 3); set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the zero-level no-op-range form for bisection\n",
+ GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
+ return 0;
+ }
+
+ p = kvzalloc(sizeof(*p), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+ p->page_size = cpu_to_le64(1ULL << 29); /* GMMU_PD1_VADDR_BIT_LO */
+ p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
+ p->num_levels_to_copy = cpu_to_le32(0);
+ /* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
+ ret = gsp_phys_rmcall_control(bar0,
+ GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
+ GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
+ p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+ "VASPACE_COPY_SERVER_RESERVED_PDES (no-op range, gated)");
+ pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES: no-op range numLevels=0 pageSize=0x20000000 virtAddrHi=0x11fffffff status=0x%x rpc_result=0x%x ret=%d\n",
+ status, rpc_result, ret);
+ kvfree(p);
+ return ret;
+}
+
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);
+
static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
const struct gsp_phys_rmcall_mem_obj_shape *shape,
const char *step)
@@ -7718,6 +7835,22 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
return ret;
}
+ /*
+ * Mode 25: the captured VAS construct control follows the VAS alloc.
+ * It is gated off by default and aborts the replay only when the
+ * operator enabled the gate and the server rejected it.
+ */
+ if (boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+ gsp_phys_rmcall_engine_is_gr()) {
+ ret = gsp_phys_rmcall_golden9_vas_control(bar0);
+ if (ret) {
+ pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
+ ret);
+ return ret;
+ }
+ }
+
/* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
memset(&shape, 0, sizeof(shape));
shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS;
@@ -7774,6 +7907,29 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
ch.h_userd_memory[0] =
cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD);
ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+ /*
+ * Mode 25: the captured golden channel shape. The real client's
+ * instance/USERD/RAMFC/mthd blocks were client-local memmgr
+ * allocations (FBMEM in the capture); the ladder substitutes the
+ * registered host sysmem chunks and documents the deviation.
+ */
+ if (boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+ gsp_phys_rmcall_engine_is_gr()) {
+ ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
+ ch.h_object_buffer = cpu_to_le32(0);
+ ch.instance_mem = gsp_phys_rmcall_mem_desc(
+ GSP_PHYS_RMCALL_BUF_INSTANCE);
+ ch.userd_mem = gsp_phys_rmcall_mem_desc(
+ GSP_PHYS_RMCALL_BUF_USERD);
+ ch.ramfc_mem = gsp_phys_rmcall_mem_desc(
+ GSP_PHYS_RMCALL_BUF_RAMFC);
+ ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(
+ GSP_PHYS_RMCALL_BUF_MTHDBUF);
+ pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=host sysmem memdescs (capture used client-local FBMEM)\n",
+ GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
+ GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
+ }
ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
boot_stage.rmcall_device,
GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
@@ -7793,6 +7949,23 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
return ret;
}
+ /*
+ * Mode 25: the single atomic 9-entry GPU_PROMOTE_CTX on the golden
+ * channel, in the captured order/forms. It runs after the channel
+ * alloc and before the 3D object, matching the wire order of the real
+ * client (its GR-object constructor promotes before it allocates the
+ * object). A failure aborts the whole golden replay with no retry and
+ * no follow-up promote; the golden VAS/channel/3D objects stay alive
+ * for the ladder's own V6 FWCLIENT follow-on promote.
+ */
+ if (boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+ gsp_phys_rmcall_engine_is_gr()) {
+ ret = gsp_phys_rmcall_golden9_promote(bar0);
+ if (ret)
+ return ret;
+ }
+
/* 6. TURING_A (0xc597) 3D object on the golden channel (kernel_graphics.c:2409-2433). */
ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
GSP_PHYS_RMCALL_CLASS_GR_3D,
@@ -8543,6 +8716,123 @@ static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
}
/*
+ * Mode-25 golden 9-entry promote (0x2080012b): ONE atomic control on the
+ * golden channel with the captured entry order/forms - MAIN id0 combined,
+ * PATCH id2 combined, ids 3/4/5/6 VA-only globals (BUNDLE_CB, PAGEPOOL,
+ * ATTRIBUTE_CB, RTV_CB_GLOBAL) and ids 9/10/11 (FECS_EVENT, PRIV_ACCESS_MAP,
+ * UNRESTRICTED_PRIV_ACCESS_MAP) combined. The ladder's prepared sysmem
+ * context buffers back the combined entries (physAttr 0x6, the documented
+ * deviation from the capture's VIDMEM 0x4); the VA-only globals keep
+ * booking-only VAs above the context window. All nine buffers must be
+ * present; a short set or a rejected control aborts the whole replay with no
+ * retry and never a partial accept followed by another promote.
+ */
+static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
+{
+ struct gsp_phys_rmcall_promote_params *p;
+ u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
+ u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
+ u32 rpc_result = 0, status = 0;
+ unsigned int i;
+ int ret;
+
+ if (!boot_stage.rmcall_ctx_prepared ||
+ !boot_stage.rmcall_ctx_pa || !boot_stage.rmcall_ctx_size) {
+ pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: no prepared MAIN context buffer (prepared=%u pa=0x%llx size=0x%x)\n",
+ boot_stage.rmcall_ctx_prepared,
+ boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size);
+ return -EINVAL;
+ }
+ ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
+ if (ret)
+ return ret;
+ gsp_phys_rmcall_gr_fwclient_small_prepare();
+ if (!boot_stage.rmcall_patch_pa || !boot_stage.rmcall_fecs_pa ||
+ !boot_stage.rmcall_priv_pa ||
+ !boot_stage.rmcall_unrestricted_pa) {
+ pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: patch=0x%llx fecs=0x%llx priv=0x%llx unr=0x%llx (all nine buffers must be present); no promote is sent and the ladder aborts with no retry\n",
+ boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa,
+ boot_stage.rmcall_priv_pa,
+ boot_stage.rmcall_unrestricted_pa);
+ return -ENOMEM;
+ }
+
+ p = kvzalloc(sizeof(*p), GFP_KERNEL);
+ if (!p)
+ return -ENOMEM;
+
+ p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
+ p->h_client = cpu_to_le32(0);
+ p->ch_id = cpu_to_le32(0);
+ p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
+ p->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL);
+ p->entry_count = cpu_to_le32(9);
+
+ gsp_phys_rmcall_promote_entry_fill(p, 0,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
+ boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+ boot_stage.rmcall_ctx_size, attr, 1, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 1,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
+ boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
+ boot_stage.rmcall_patch_size, attr, 1, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 2,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
+ 0, bk + 0x2000ULL, 0, 0, 0, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 3,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
+ 0, bk + 0x3000ULL, 0, 0, 0, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 4,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
+ 0, bk + 0x4000ULL, 0, 0, 0, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 5,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
+ 0, bk + 0x5000ULL, 0, 0, 0, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 6,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
+ boot_stage.rmcall_fecs_pa, bk + 0x6000ULL, 0x1000u, attr, 1, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 7,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
+ boot_stage.rmcall_priv_pa, bk + 0x7000ULL, 0x1000u, attr, 1, 0);
+ gsp_phys_rmcall_promote_entry_fill(p, 8,
+ GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
+ boot_stage.rmcall_unrestricted_pa, bk + 0x8000ULL,
+ 0x1000u, attr, 1, 0);
+
+ for (i = 0; i < 9; i++)
+ pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
+ i, le16_to_cpu(p->entry[i].buffer_id),
+ le64_to_cpu(p->entry[i].gpu_phys_addr),
+ le64_to_cpu(p->entry[i].gpu_virt_addr),
+ le64_to_cpu(p->entry[i].size),
+ le32_to_cpu(p->entry[i].phys_attr),
+ p->entry[i].b_initialize, p->entry[i].b_nonmapped,
+ (i == 2 || i == 3 || i == 4 || i == 5) ?
+ " - VA-only booking, no PA" : "");
+
+ ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
+ GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
+ p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
+ "GPU_PROMOTE_CTX (golden9, 9 entries, atomic)");
+ pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9: entryCount=9 status=0x%x rpc_result=0x%x (hObject=golden channel 0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; gr_promote_param=%u)\n",
+ status, rpc_result, GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
+ boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0,
+ boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
+ boot_stage.rmcall_ctx_size,
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9);
+ if (ret) {
+ pr_err("vgsp-boot: rm-call: PROMOTE_CTX golden9 (9 entries) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the golden context is not committed and the ladder aborts with no retry and no follow-up promote\n",
+ status, rpc_result,
+ gsp_phys_rmcall_status_str(status), ret);
+ kvfree(p);
+ return ret;
+ }
+ boot_stage.rmcall_gr_promote_step = 1;
+ kvfree(p);
+ return 0;
+}
+
+/*
* Fill one MAIN promote entry with the prepared PA/VA and the snapshot's
* entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5
* entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the
@@ -8676,6 +8966,11 @@ static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0)
* entry in the combined kgrobjPromoteContext form where the ladder has
* a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers
* are omitted, all logged per entry.
+ * - mode 25: the same follow-on FWCLIENT promote on the ladder's V6
+ * channel, after gsp_phys_rmcall_golden_replay replayed the real golden
+ * construct (captured VAS control, captured channel shape and the single
+ * atomic 9-entry PROMOTE_CTX on the golden channel) so the server-side
+ * GR-instance state exists before this control runs.
* Mode 0 never reaches this function (stage 3 stops before it); the early
* return is a belt-and-braces no-op. Success means the server installed the
* context memdesc, added the VA to the engine-context vaList, committed the
@@ -8708,6 +9003,17 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF)
return -EOPNOTSUPP;
/*
+ * Mode 25's golden construct lives in the golden replay
+ * (gsp_phys_rmcall_golden_replay); without both gates the replay never
+ * ran, so the follow-on FWCLIENT promote would have no golden-initialized
+ * GR-instance state to consume. Abort instead of sending it blind.
+ */
+ if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+ (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
+ pr_err("vgsp-boot: rm-call: mode 25 requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n");
+ return -EINVAL;
+ }
+ /*
* Modes 8/9/10 must run on the V6 channel: without the
* SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping
* route that asserted GSP-RM in modes 1/2/4 (or rejects the
@@ -8726,7 +9032,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
- mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) &&
+ mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+ mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) &&
boot_stage.rmcall_channel_variant !=
gsp_phys_rmcall_channel_variant_skipctx.id) {
pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
@@ -9406,7 +9713,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
return ret;
}
boot_stage.rmcall_gr_promote_step = 1;
- } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT) {
+ } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
+ mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) {
/*
* Mode 24: the reference FWCLIENT non-3D set as ONE control.
* Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT
@@ -9423,7 +9731,9 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
* stock shape is init-only too; both deviations are logged
* per entry. Auxiliary buffers that cannot be sourced are
* omitted and logged; entryCount is the number of real
- * entries.
+ * entries. Mode 25 shares this follow-on promote on the
+ * ladder's V6 channel after its golden construct; the
+ * existing mask selects its entries too (bits 0-7).
*/
u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
@@ -10236,15 +10546,25 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
* replay the stock client's post-load golden channel allocations after
* the ladder's VAS/TSG exist and before the ladder's own channel/GR
* object, so the server-side GR-instance state the GR engine-context
- * commit needs is initialized first. The first failure aborts the whole
- * ladder with no retry; with the parameter 0 this block is skipped
- * entirely (no RPC, no log).
+ * commit needs is initialized first. With mode 25 the replay carries
+ * the single atomic 9-entry promote and is no longer best-effort: a
+ * failure aborts the whole ladder with no retry and no follow-up
+ * promote. For every other mode the replay keeps its historical
+ * best-effort continuation.
*/
if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
ret = gsp_phys_rmcall_golden_replay(bar0);
- if (ret)
+ if (ret) {
+ if (boot_stage.rmcall_gr_promote ==
+ GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 &&
+ gsp_phys_rmcall_engine_is_gr()) {
+ pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-25 golden replay failed (%d); the golden construct is atomic and is not retried\n",
+ ret);
+ return ret;
+ }
pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
ret);
+ }
}
/*
@@ -14500,6 +14820,11 @@ static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40
"NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40");
static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48,
"NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)");
+static_assert(sizeof(struct gsp_phys_rmcall_vas_copy_pdes_params) == 184,
+ "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS must be 184 bytes");
+static_assert(offsetof(struct gsp_phys_rmcall_vas_copy_pdes_params,
+ num_levels_to_copy) == 32,
+ "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.numLevelsToCopy must be at offset 32");
static_assert(sizeof(struct gsp_phys_rmcall_evict_ctx_params) == 20,
"NV2080_CTRL_GPU_EVICT_CTX_PARAMS must be 20 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 80,
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 5e6b264..21313a2 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -511,8 +511,8 @@ static int gsp_phys_rmcall_gr_promote_set(const char *val,
return ret;
if (gsp_phys_rmcall_gr_promote_param < 0 ||
- gsp_phys_rmcall_gr_promote_param > 24) {
- pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..24; clamping to 0 (no PROMOTE_CTX)\n",
+ gsp_phys_rmcall_gr_promote_param > 25) {
+ pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..25; clamping to 0 (no PROMOTE_CTX)\n",
gsp_phys_rmcall_gr_promote_param);
gsp_phys_rmcall_gr_promote_param = 0;
}
@@ -531,7 +531,7 @@ module_param_cb(gsp_phys_rmcall_gr_promote,
&gsp_phys_rmcall_gr_promote_ops,
&gsp_phys_rmcall_gr_promote_param, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
- "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); values outside 0..24 clamp to 0 with a warning");
+ "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; values outside 0..25 clamp to 0 with a warning");
int gsp_phys_rmcall_gr_promote_value(void)
{
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 9e809e3..a38067a 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -250,8 +250,12 @@ int gsp_phys_rmcall_engine_value(void);
* PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) as one control on the
* V6 KMD-managed channel: combined PA+VA entries where the ladder has
* a VA, PM/PRIV_ACCESS_MAP init-only, unsourceable buffers omitted
- * and logged.
- * The parameter's set callback clamps any value outside 0..24 to 0 with a
+ * and logged;
+ * 25 = the real golden-client construct on the golden replay handles
+ * (captured channel shape + single atomic 9-entry PROMOTE_CTX on the
+ * golden channel) followed by the V6 FWCLIENT promote; requires
+ * gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1.
+ * The parameter's set callback clamps any value outside 0..25 to 0 with a
* warning and warns on every nonzero enable (a failed promote can assert
* GSP-RM and wedge the device; recovery needs a reboot with human approval).
* The booter snapshots the value when the rm-call ladder's stage 3 starts
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 22:58:10 +0000