summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rm_emulation.c154
-rw-r--r--kernel/gsp_rm_tu104.h49
2 files changed, 155 insertions, 48 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index b9f8417..44de6c0 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -352,6 +352,10 @@ static u32 gsp_rm_checksum32(const void *data, u32 len)
*/
#define GSPRM_ELEM_HDR_SIZE 0x30
+/* rpc_gsp_rm_alloc_v03_00 fixed header: hClient, hParent, hObject, hClass,
+ * status, paramsSize, flags, reserved[4]. */
+#define GSPRM_ALLOC_HDR_SIZE 32
+
/*
* Max RPC data that fits in one queue element:
* 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes.
@@ -817,8 +821,15 @@ static void gsp_rm_build_static_info(u8 *buf)
val32 = 0;
memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4);
- /* engineCaps (offset 1204, 12 bytes) */
- val32 = 0x60003E02; /* GR0|CE0-4|NVDEC0|NVDEC1 */
+ /* engineCaps (offset 1204, 12 bytes)
+ *
+ * Bit N = RM_ENGINE_TYPE_<N> present (RM_ENGINE_TYPE_* equals the bit
+ * index, see gpu_engine_type.h). The async copy engine is
+ * RM_ENGINE_TYPE_COPY14 (0x17); clients (libcuda) refuse to create a
+ * context without it: the secondary/async engine record stays empty and
+ * the engine-context RM alloc is issued with class 0 (NV01_ROOT), which
+ * the driver must reject. */
+ val32 = 0x60803E02; /* GR0|CE0-4|COPY14|NVDEC0|NVDEC1 */
memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4);
val32 = 0x00362020; /* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */
memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4);
@@ -1308,8 +1319,19 @@ static void gsp_rm_handle_ce_physical_caps(struct gsp_emu_context *ctx,
if (ce_type >= TU104_ENGINE_TYPE_COPY0 &&
ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES) {
+ u32 idx = ce_type - TU104_ENGINE_TYPE_COPY0;
+
out_params[4] = TU104_CE_CAPS0;
out_params[5] = TU104_CE_CAPS1;
+
+ /* The GR-synchronous CEs (GRCE) are the first ones: the client
+ * picks the last GRCE as its compute companion ("ceForCompute",
+ * rm_device.c) and every non-GRCE CE becomes an async CE. With
+ * no GRCE at all the client's context-creation walk ends with an
+ * empty secondary-engine record and the engine RM alloc is issued
+ * with class 0. */
+ if (idx == TU104_GRCE_IDX)
+ out_params[4] |= TU104_CE_CAPS0_GRCE;
} else {
*out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */
*out_params_size = req_size;
@@ -1342,6 +1364,8 @@ static void gsp_rm_handle_ce_all_caps(struct gsp_emu_context *ctx,
for (i = 0; i < TU104_NUM_CES; i++) {
out_params[i * 2 + 0] = TU104_CE_CAPS0;
out_params[i * 2 + 1] = TU104_CE_CAPS1;
+ if (i == TU104_GRCE_IDX)
+ out_params[i * 2 + 0] |= TU104_CE_CAPS0_GRCE;
present |= 1ULL << i;
}
memcpy(out_params + 128, &present, 8);
@@ -1898,6 +1922,87 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
}
}
+static bool gsp_rm_alloc_interesting(u32 class_id);
+
+/*
+ * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by
+ * the class-specific allocation parameters.
+ *
+ * The reply MUST echo the request payload back: the client copies the reply
+ * params over the caller's params buffer using the class-expected size (not
+ * the size the caller reserved), so replying with zeroed params smashes
+ * whatever sits above the caller's params buffer. The alloc params are
+ * in/out for the classes we do not model in detail, so echoing the input is
+ * both the safe and the correct "unchanged" answer. status (offset 16) is
+ * the only field we deliberately rewrite (to NV_OK).
+ *
+ * NOTE: the RPC header 'length' only counts the fixed struct size, so it
+ * must NOT be used to bound the params - read the paramsSize field from the
+ * alloc header instead (see handoff PART 6).
+ */
+static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx,
+ const u8 *elem, u32 sequence)
+{
+ struct gsp_rm_emu *rm = ctx->rm_emu;
+ const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
+ sizeof(struct gsprm_rpc_header);
+ const u8 *params = p + GSPRM_ALLOC_HDR_SIZE;
+ u32 hc = 0, hp = 0, ho = 0, class_id = 0;
+ u32 params_size = 0, flags = 0;
+ u32 echo_len, reply_len;
+ u8 *reply;
+
+ memcpy(&hc, p + 0, 4);
+ memcpy(&hp, p + 4, 4);
+ memcpy(&ho, p + 8, 4);
+ memcpy(&class_id, p + 12, 4);
+ memcpy(&params_size, p + 20, 4);
+ memcpy(&flags, p + 24, 4);
+
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x",
+ class_id, params_size, flags, hc, hp, ho);
+
+ /* Full param dump for memory/channel classes and any class we have not
+ * classified yet (rate-limited). %phN is capped at 64 bytes per print,
+ * so chunk it. */
+ if (gsp_rm_alloc_interesting(class_id) && rm->alloc_param_dumps < 60) {
+ u32 shown = min(params_size, (u32)384);
+ u32 off;
+
+ rm->alloc_param_dumps++;
+ for (off = 0; off < shown; off += 64) {
+ u32 n = min(shown - off, (u32)64);
+
+ nv_vgpu_info(ctx->vgpu,
+ "GSPRM: alloc class=0x%x params[%u..%u]=%*phN",
+ class_id, off, off + n, (int)n,
+ params + off);
+ }
+ }
+
+ /* Echo the fixed alloc header + as many param bytes as are contiguous
+ * in the first queue element. */
+ echo_len = min(params_size,
+ (u32)GSPRM_MAX_SINGLE_ELEM_DATA - GSPRM_ALLOC_HDR_SIZE);
+ reply_len = GSPRM_ALLOC_HDR_SIZE + echo_len;
+
+ reply = kmalloc(reply_len, GFP_KERNEL);
+ if (!reply) {
+ nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_RM_ALLOC reply alloc failed");
+ return;
+ }
+
+ memcpy(reply, p, reply_len);
+ memset(reply + 16, 0, 4); /* status = NV_OK */
+
+ if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_ALLOC, sequence, 0,
+ reply, reply_len))
+ nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_RM_ALLOC reply send failed");
+
+ kfree(reply);
+}
+
static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
const u8 *elem, u32 sequence)
{
@@ -2200,6 +2305,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
gsp_rm_handle_get_static_info(ctx, sequence);
return;
+ case GSPRM_RPC_FUNC_GSP_RM_ALLOC:
+ gsp_rm_handle_rm_alloc(ctx, elem, sequence);
+ return;
+
case GSPRM_RPC_FUNC_GSP_RM_CONTROL:
gsp_rm_handle_rm_control(ctx, elem, sequence);
return;
@@ -2217,47 +2326,6 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem)
nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u cmd_read=%u stat_write=%u stat_seq=%u",
gsp_rm_rpc_name(function), function, sequence,
rm->cmd_read, rm->stat_write, rm->stat_seq);
- } else if (function == GSPRM_RPC_FUNC_GSP_RM_ALLOC) {
- /* GSP_RM_ALLOC (103): rpc_gsp_rm_alloc_v03_00 at
- * elem+GSPRM_ELEM_HDR_SIZE. NOTE: the RPC header
- * 'length' only counts the fixed struct size (64), so
- * it must NOT be used to bound the params - read the
- * alloc header directly. See handoff PART 6. */
- const u8 *p = elem + GSPRM_ELEM_HDR_SIZE +
- sizeof(struct gsprm_rpc_header);
- const u8 *params = p + 32;
- u32 hc = 0, hp = 0, ho = 0, class_id = 0;
- u32 params_size = 0, flags = 0;
-
- memcpy(&hc, p + 0, 4);
- memcpy(&hp, p + 4, 4);
- memcpy(&ho, p + 8, 4);
- memcpy(&class_id, p + 12, 4);
- memcpy(&params_size, p + 20, 4);
- memcpy(&flags, p + 24, 4);
-
- nv_vgpu_info(ctx->vgpu,
- "GSPRM: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x",
- class_id, params_size, flags, hc, hp, ho);
-
- /* Full param dump for memory/channel classes and any
- * class we have not classified yet (rate-limited).
- * %phN is capped at 64 bytes per print, so chunk it. */
- if (gsp_rm_alloc_interesting(class_id) &&
- rm->alloc_param_dumps < 60) {
- u32 shown = min(params_size, (u32)384);
- u32 off;
-
- rm->alloc_param_dumps++;
- for (off = 0; off < shown; off += 64) {
- u32 n = min(shown - off, (u32)64);
-
- nv_vgpu_info(ctx->vgpu,
- "GSPRM: alloc class=0x%x params[%u..%u]=%*phN",
- class_id, off, off + n, (int)n,
- params + off);
- }
- }
} else {
nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)",
gsp_rm_rpc_name(function), function, sequence,
diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h
index 6a19726..74d1ba3 100644
--- a/kernel/gsp_rm_tu104.h
+++ b/kernel/gsp_rm_tu104.h
@@ -142,6 +142,7 @@ struct tu104_get_constructed_falcon_info_params {
#define TU104_ENG_CE2 0x793ceb02
#define TU104_ENG_CE3 0x793ceb03
#define TU104_ENG_CE4 0x793ceb04
+#define TU104_ENG_CES 0x793ceb0e /* classId(OBJCE) inst 14 (async) */
#define TU104_ENG_NVDEC0 0x8f99e100 /* classId(OBJBSP)=0x8f99e1, inst 0 */
#define TU104_ENG_NVDEC1 0x8f99e101
#define TU104_ENG_NVENC0 0xe97b6c00 /* classId(OBJMSENC)=0xe97b6c, inst 0 */
@@ -159,6 +160,7 @@ struct tu104_get_constructed_falcon_info_params {
#define TU104_RM_ENGINE_CE2 0x0b
#define TU104_RM_ENGINE_CE3 0x0c
#define TU104_RM_ENGINE_CE4 0x0d
+#define TU104_RM_ENGINE_CES 0x17 /* RM_ENGINE_TYPE_COPY14 */
#define TU104_RM_ENGINE_NVDEC0 0x1d
#define TU104_RM_ENGINE_NVDEC1 0x1e
#define TU104_RM_ENGINE_NVENC0 0x25
@@ -178,6 +180,7 @@ struct tu104_get_constructed_falcon_info_params {
#define TU104_MC_ENGINE_CE2 17
#define TU104_MC_ENGINE_CE3 18
#define TU104_MC_ENGINE_CE4 19
+#define TU104_MC_ENGINE_CES 29 /* MC_ENGINE_IDX_CE14 */
#define TU104_MC_ENGINE_NVDEC0 65
#define TU104_MC_ENGINE_NVDEC1 66
#define TU104_MC_ENGINE_NVENC0 38
@@ -193,6 +196,7 @@ struct tu104_get_constructed_falcon_info_params {
#define TU104_FAULT_ID_CE2 17
#define TU104_FAULT_ID_CE3 18
#define TU104_FAULT_ID_CE4 19
+#define TU104_FAULT_ID_CES 29
#define TU104_FAULT_ID_NVDEC0 10
#define TU104_FAULT_ID_NVDEC1 25
#define TU104_FAULT_ID_NVENC0 11
@@ -241,6 +245,7 @@ struct tu104_get_constructed_falcon_info_params {
#define TU104_TAG_CE2 3
#define TU104_TAG_CE3 4
#define TU104_TAG_CE4 5
+#define TU104_TAG_CES 6
#define TU104_TAG_NVDEC0 10
#define TU104_TAG_NVDEC1 11
#define TU104_TAG_NVENC0 13
@@ -282,7 +287,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
.numPbdmas = 1,
.engineName = "GRAPHICS",
},
- /* [1] CE0 */
+ /* [1] CE0 - the GR-synchronous CE (GRCE); shares the GR PBDMA */
{
.engineData = {
[EI_ENG_DESC] = TU104_ENG_CE0,
@@ -302,8 +307,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
[EI_CHRAM_PRI_BASE] = 0,
[EI_KERNEL_RM_MAX] = 0,
},
- .pbdmaIds = {1, 0},
- .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 1, 0},
+ .pbdmaIds = {0, 0},
+ .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 0, 0},
.numPbdmas = 1,
.engineName = "CE0",
},
@@ -407,7 +412,36 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
.numPbdmas = 1,
.engineName = "CE4",
},
- /* [6] NVDEC0 */
+ /* [6] CES - async copy engine (RM COPY14 / OBJCE instance 14).
+ * Clients (libcuda) refuse to finish context creation without it: the
+ * async/secondary engine record stays empty and the engine-context RM
+ * alloc is issued with class 0. Not on a runlist of its own; shares the
+ * CE runlist. */
+ {
+ .engineData = {
+ [EI_ENG_DESC] = TU104_ENG_CES,
+ [EI_FIFO_TAG] = TU104_TAG_CES,
+ [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CES,
+ [EI_RUNLIST] = 5,
+ [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CES,
+ [EI_RC_MASK] = TU104_RC_CE4,
+ [EI_RESET] = 5,
+ [EI_INTR] = 5,
+ [EI_MC] = TU104_MC_ENGINE_CES,
+ [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE,
+ [EI_INSTANCE_ID] = 14,
+ [EI_RUNLIST_PRI_BASE] = 0,
+ [EI_IS_HOST_DRIVEN_ENGINE] = 1,
+ [EI_RUNLIST_ENGINE_ID] = 0,
+ [EI_CHRAM_PRI_BASE] = 0,
+ [EI_KERNEL_RM_MAX] = 0,
+ },
+ .pbdmaIds = {5, 0},
+ .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 5, 0},
+ .numPbdmas = 1,
+ .engineName = "CES",
+ },
+ /* [7] NVDEC0 */
{
.engineData = {
[EI_ENG_DESC] = TU104_ENG_NVDEC0,
@@ -559,7 +593,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = {
},
};
-#define TU104_ENGINE_TABLE_SIZE 12
+#define TU104_ENGINE_TABLE_SIZE 13
/* ============================================================
* TU104 Interrupt Table (from intr_phys_tu102.c, kernel table)
@@ -706,6 +740,7 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = {
#define TU104_CTRL_CE_GET_CE_PCE_MASK 0x20802a02
/* CE capability bits (NV2080_CTRL_CE_CAPS_*, ctrl2080ce.h: byte:bit) */
+#define TU104_CE_CAPS0_GRCE 0x01 /* synchronous with GR (GRCE) */
#define TU104_CE_CAPS0_SYSMEM_READ 0x04
#define TU104_CE_CAPS0_SYSMEM_WRITE 0x08
#define TU104_CE_CAPS0_SYSMEM 0x20
@@ -721,6 +756,10 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = {
/* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */
#define TU104_ENGINE_TYPE_COPY0 0x00000009
#define TU104_NUM_CES 5
+/* The GR-synchronous CE (GRCE): the one sharing the GR PBDMA. Clients pick
+ * it as their compute companion (ceForCompute); the driver accepts a CE on
+ * the GR runqueue only when it shares GR's PBDMA ids. */
+#define TU104_GRCE_IDX 0
#define TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO 0x20800a59
#define TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER 0x20800a22
#define TU104_CTRL_STATIC_KGR_GET_INFO 0x20800a2a
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 23:37:41 +0000