diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 38 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 18 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 7 |
3 files changed, 61 insertions, 2 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 8b78d9a..dc8e961 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -13827,6 +13827,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); + int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); p = gsp_phys_rmcall_build_gr_init(p); if (qmdvirt) @@ -13882,7 +13883,16 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, (u32)(qmd_va >> 8)); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B, + two_step ? 1u : GSP_PHYS_RMCALL_GR_PCAS_INVALIDATE_SCHEDULE); + if (two_step) { + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SEND_PCAS_A, + (u32)(qmd_va >> 8)); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B, + 2u); + } } p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, (u32)(sem_va >> 32)); @@ -14305,6 +14315,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, if (!epoch) epoch = ++boot_stage.rmcall_gr_submit_epoch; memcpy(qmd, req->qmd, sizeof(qmd)); + /* + * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's + * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not + * schedule and the trailing schedule-only PCAS does. + */ + if (gsp_phys_rmcall_gr_pcas_two_step_value()) + qmd[4] |= 1u << 12; /* * QMD release0: when the launch has a one-word absolute release and @@ -14814,6 +14831,7 @@ gr_page_unmapped: : 0; u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0; u32 q48 = 0, q49 = 0, q50 = 0; + u32 q51 = 0, q52 = 0, q53 = 0; u32 q63 = 0; u32 relw = 0; @@ -14832,6 +14850,9 @@ gr_page_unmapped: q48 = get_unaligned_le32(qp + 192); q49 = get_unaligned_le32(qp + 196); q50 = get_unaligned_le32(qp + 200); + q51 = get_unaligned_le32(qp + 204); + q52 = get_unaligned_le32(qp + 208); + q53 = get_unaligned_le32(qp + 212); q63 = get_unaligned_le32( qp + GSP_PHYS_GR_QMD_SENTINEL_WORD * @@ -14842,13 +14863,26 @@ gr_page_unmapped: relw = READ_ONCE( *(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off)); - pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w48..50=%08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n", + pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w48..53=%08x %08x %08x %08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n", sub.gpget, gpget_post, q0, q1, q23, q24, - q25, q48, q49, q50, q63, relw, + q25, q48, q49, q50, q51, q52, q53, q63, + relw, q63 == GSP_PHYS_GR_QMD_SENTINEL ? "not-run (sentinel intact)" : "ran (page overwritten)", qmdwin_note); + if (bar0) + pr_info("vgsp-gr: FE readback raw 0x40a500..0x40a50c = %08x %08x %08x %08x (subctx-1 control at 0x40a504)\n", + ioread32(bar0 + 0x0040a500u), + ioread32(bar0 + 0x0040a504u), + ioread32(bar0 + 0x0040a508u), + ioread32(bar0 + 0x0040a50cu)); + if (bar0) + pr_info("vgsp-gr: SKED readback: HWW_ESR=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x FECS_CTX=0x%08x\n", + ioread32(bar0 + 0x00407020u), + ioread32(bar0 + 0x00407060u), + ioread32(bar0 + 0x00407064u), + ioread32(bar0 + 0x00409b00u)); } poll = w1; ret = -ETIMEDOUT; diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9100959..5ec97e2 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -638,6 +638,24 @@ int gsp_phys_rmcall_gr_pcas_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value); /* + * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting + * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy + * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the + * dependent-QMD flow (cuda_device.h:1108-1113). + */ +static int gsp_phys_rmcall_gr_pcas_two_step_param; +module_param_named(gsp_phys_rmcall_gr_pcas_two_step, + gsp_phys_rmcall_gr_pcas_two_step_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas_two_step, + "Two-step PCAS: set QMD REQUIRE_SCHEDULING_PCAS and emit copy PCAS then schedule-only PCAS (default 0; requires gsp_phys_rmcall_gr_pcas=1)"); + +int gsp_phys_rmcall_gr_pcas_two_step_value(void) +{ + return gsp_phys_rmcall_gr_pcas_two_step_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_two_step_value); + +/* * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin; default 0). When set, * a forwarded GR launch relocates its inline-QMD destination from the guest * QMD VA to a dedicated module page mapped at the ladder's own window-0 VA diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index d37f5d5..8cfb5e9 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -301,6 +301,13 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void); int gsp_phys_rmcall_gr_pcas_value(void); /* + * gsp_phys_rmcall_gr_pcas_two_step (default 0): set the QMD + * REQUIRE_SCHEDULING_PCAS bit and emit copy PCAS followed by a schedule-only + * PCAS; requires gsp_phys_rmcall_gr_pcas=1. + */ +int gsp_phys_rmcall_gr_pcas_two_step_value(void); + +/* * gsp_phys_rmcall_gr_qmdwin (default 0): 1 relocates the forwarded GR * inline-QMD destination from the guest QMD VA to a dedicated module page * mapped at the proven engine-reachable window-0 VA 0x107000 (the PTE is |