diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 74 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 19 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 1 |
3 files changed, 94 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 46060b1..d43430b 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16744,6 +16744,61 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -16776,6 +16831,25 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 74062dd..e9c0c5a 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -617,6 +617,25 @@ int gsp_phys_rmcall_gr_prelude_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value); /* + * r597: emit the SEM_EXECUTE ACQUIRE groups found in the captured guest + * prelude ahead of the rebuilt inline-QMD window (default 0). The verbatim + * prelude replay breaks the launch (qmd_rel=0 after an 8.9 s stall, r599), + * so the acquire cannot be taken by replaying the prelude; this gate takes + * only the wait method groups and leaves the proven window in place. + */ +static int gsp_phys_rmcall_gr_acq_param; +module_param_named(gsp_phys_rmcall_gr_acq, gsp_phys_rmcall_gr_acq_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_acq, + "Emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt GR window (default 0)"); + +int gsp_phys_rmcall_gr_acq_value(void) +{ + return gsp_phys_rmcall_gr_acq_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_acq_value); + +/* * QMD virtualization control gate (default 0 after m26 falsified the * hypothesis: on Turing the context init value is 0, native CUDA never * writes 0x29c, and the nonzero enable alone does not schedule). When set, diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 36c2220..3993c6e 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -295,6 +295,7 @@ int gsp_phys_rmcall_gr_promote_value(void); * init+window+PCAS+sem shape); 1 replays the verbatim prelude. */ int gsp_phys_rmcall_gr_prelude_value(void); +int gsp_phys_rmcall_gr_acq_value(void); /* * gsp_phys_rmcall_gr_qmdvirt (default 1): emit |