diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 72 |
1 files changed, 69 insertions, 3 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 49e8be9..2c0fdc0 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -12777,8 +12777,34 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_SEND_PCAS_A 0x2b4u #define GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B 0x2bcu #define GSP_PHYS_RMCALL_GR_PCAS_INVALIDATE_SCHEDULE 0x3u -/* Method block bound: inline run(67) + PCAS(4) + REPORT_SEM(8). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 96u +/* + * Per-channel compute init (the vendor turingInitComputeState block): the + * stock 570 compute channel receives it once at channel creation, long before + * any launch, and the V6 channel never did. Emit it as a preamble so the + * SKED/CWD state a QMD needs exists before the inline-QMD run. Values are the + * TU104 constants: SPA 7.5, CWD refcount = 48 SMs * 32 warps, shared window + * base 0x1000000 (CU_TURING_SHARED_WINDOW_BASE), 24 non-throttled local + * memory SMs, SKEDCHECK 48 KB check disabled, cut collector enabled. + */ +#define GSP_PHYS_RMCALL_GR_INIT_NO_OPERATION 0x100u +#define GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_A 0x2a0u +#define GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_B 0x2a4u +#define GSP_PHYS_RMCALL_GR_INIT_SPA_VERSION 0x310u +#define GSP_PHYS_RMCALL_GR_INIT_CWD_REF_COUNTER 0x248u +#define GSP_PHYS_RMCALL_GR_INIT_SKEDCHECK 0x23cu +#define GSP_PHYS_RMCALL_GR_INIT_CUT_COLLECTOR 0x254u +#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_A 0x200u +#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_B 0x204u +#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_C 0x208u +#define GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C 0x2ecu +#define GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL 0x250u +#define GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE 0x1000000u +#define GSP_PHYS_RMCALL_GR_SPA_VERSION 0x0705u +#define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u +#define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u +#define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; @@ -12803,6 +12829,45 @@ void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters); /* + * Vendor per-channel compute init subset (turingInitComputeState): the state + * a QMD needs before SKED can schedule it. Emitted once before the launch. + */ +static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) +{ + u32 i; + + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT, + GSP_PHYS_RMCALL_CLASS_COMPUTE); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_NO_OPERATION, + 0u); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_A, + (u32)((u64)GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE >> 32)); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_B, + GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SPA_VERSION, + GSP_PHYS_RMCALL_GR_SPA_VERSION); + for (i = GSP_PHYS_RMCALL_GR_CWD_LEVELS; i-- > 0;) + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_INIT_CWD_REF_COUNTER, + (i & 0x3fu) | + (GSP_PHYS_RMCALL_GR_CWD_REFCOUNT << 8)); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SKEDCHECK, + 1u); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_CUT_COLLECTOR, + 1u); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_A, 0u); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_B, + (u32)GSP_PHYS_RMCALL_VA_DST); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_C, + GSP_PHYS_PAGE_SIZE); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C, + GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, + 0u); + return p; +} + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -12815,6 +12880,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const u32 *qmd, { u32 *p = pb; + p = gsp_phys_rmcall_build_gr_init(p); *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) | ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | @@ -15962,7 +16028,7 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, - "the GR launch method block (SET_OBJECT + inline-QMD run + REPORT_SEMAPHORE) must fit the bound"); + "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= GSP_PHYS_RMCALL_COPY_SIZE, "the GR launch method block must fit one module pushbuffer page"); |