summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c89
1 files changed, 71 insertions, 18 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ea52845..8fd602c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1861,6 +1861,20 @@ void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
*/
static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS;
module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644);
+
+/*
+ * SESSION 127 C5C0 translator gate. When 1, the sweep additionally
+ * translates continuation entries into launch work (tail-aware method
+ * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5
+ * copy-out capture, candidate-owned release withholding). Default 0 keeps
+ * the historical sweep behaviour for the CUDA batteries; the Vulkan
+ * translator cycles pass sema_gr_xlate=1. Flip the default once the CUDA
+ * battery is proven clean with it on.
+ */
+static bool gsp_sema_gr_xlate;
+module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644);
+MODULE_PARM_DESC(sema_gr_xlate,
+ "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)");
MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)");
static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
@@ -10173,8 +10187,27 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
keepit = true;
break;
}
- if (!keepit)
+ /*
+ * SESSION 127 (gated): a stale session-wide
+ * rec whose page resolves nowhere made the
+ * fill refuse the whole launch; keep it when
+ * it resolves (the CUDA inline-served
+ * pages), drop only unresolvable unnamed
+ * ones. With the translator off, the
+ * historical keep-all behaviour stands.
+ */
+ if (!keepit && gsp_sema_gr_xlate) {
+ u64 rpa = 0;
+ u32 rap = SVC_APERTURE_SYS;
+
+ if (svc_resolve_va(vgpu, rpage, &rpa,
+ &rap) &&
+ svc_walk_vaspaces(vgpu, rpage,
+ &rpa, &rap))
+ continue;
+ } else if (!keepit) {
continue;
+ }
recs[n].dst_va = rec->dst_va;
recs[n].len = rec->len;
memcpy(recs[n].data, rec->data, rec->len);
@@ -18902,15 +18935,21 @@ static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu)
}
}
-/* Is this release VA owned by a live, not-yet-authored launch candidate? */
-static bool svc_gr_ahead_owned_rel_va(u64 va)
+/*
+ * Is this release VA owned by a live, not-yet-authored launch candidate
+ * captured from THIS push? (Only the launch's own push's release is
+ * withheld; a continuation fence entry elsewhere in the ring keeps the
+ * historical immediate-forward behaviour.)
+ */
+static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va)
{
u32 ri;
for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
if (svc_gr_cand_ring[ri].valid &&
!svc_gr_cand_ring[ri].rel_done &&
- svc_gr_cand_ring[ri].rel_va == va)
+ svc_gr_cand_ring[ri].rel_va == va &&
+ svc_gr_cand_ring[ri].push_va == push_va)
return true;
}
return false;
@@ -20462,7 +20501,8 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
}
if (svc_resolve_va(vgpu, page, &pa, &ap)) {
/*
- * SESSION 127 C5C0 translator: a client-VAS page. The Vulkan
+ * SESSION 127 C5C0 translator (gated): a client-VAS page. The
+ * Vulkan
* QMD's program, parameter and descriptor pages live in the
* client's own page tables (vidmem aperture) and the trusted
* sources - pins, learned cache, discovered map - do not
@@ -20473,7 +20513,8 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
u64 wpa = 0;
u32 wap = SVC_APERTURE_SYS;
- if (svc_walk_vaspaces(vgpu, page, &wpa, &wap))
+ if (!gsp_sema_gr_xlate ||
+ svc_walk_vaspaces(vgpu, page, &wpa, &wap))
return;
pa = wpa;
ap = svc_ap_of(wap);
@@ -22240,10 +22281,11 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
u32 qap = SVC_APERTURE_SYS;
if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) ||
- !svc_walk_vaspaces(vgpu,
- svc_gr_cand.qmd_dev_va &
- ~(u64)(SEMA_PAGE_SIZE - 1),
- &qpa, &qap)) {
+ (gsp_sema_gr_xlate &&
+ !svc_walk_vaspaces(vgpu,
+ svc_gr_cand.qmd_dev_va &
+ ~(u64)(SEMA_PAGE_SIZE - 1),
+ &qpa, &qap))) {
svc_fwd_req.gr_qmd_pa =
qpa & ~(u64)(SEMA_PAGE_SIZE - 1);
svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap);
@@ -24006,6 +24048,7 @@ module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644);
MODULE_PARM_DESC(sema_gr_ahead,
"GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)");
+
static u32 svc_gr_ahead_log;
static u64 svc_gr_ahead_entries;
static u64 svc_gr_ahead_rels;
@@ -24142,7 +24185,13 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
u32 cp_subch = ~0u;
bool used_tail = false;
bool forward = gsp_phys_gr_forward_enabled();
+ bool xlate = gsp_sema_gr_xlate;
+ if (!xlate) {
+ /* Historical sweep: releases from exactly-covered entries. */
+ tail = NULL;
+ taillen = 0;
+ }
methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail);
if (!methods)
return 0;
@@ -24187,7 +24236,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
* C5C0 inline-QMD window (mirrors the main decode's intercept):
* only accumulate once the run declares itself a QMD window.
*/
- if (forward && subch >= 1 && addr >= SVC_QMD_ADDR_A &&
+ if (xlate && forward && subch >= 1 &&
+ addr >= SVC_QMD_ADDR_A &&
addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
(stq.qmd_have ||
(addr <= SVC_QMD_ADDR_B &&
@@ -24211,7 +24261,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
* the bytes into the VA shadow so the forwarded launch's
* operand pages - parameter banks - are real).
*/
- if (op == 3 && subch >= 1 && addr == GRCE_M_LOAD_INLINE_DATA &&
+ if (xlate && op == 3 && subch >= 1 &&
+ addr == GRCE_M_LOAD_INLINE_DATA &&
have_out && line_len) {
u64 nbytes = (u64)line_len *
(line_count ? line_count : 1);
@@ -24273,7 +24324,7 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
}
/* TURING_DMA_COPY_A operand runs (subch >= 2). */
- if (subch >= 2) {
+ if (xlate && subch >= 2) {
u32 k, n = count > 4 ? 4 : count;
u32 vv[4] = { w0, w1, w2, w3 };
@@ -24357,8 +24408,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
cp_subch = subch;
}
}
- } else if (subch == 1 && addr >= 0x180 && addr <= 0x18c &&
- count) {
+ } else if (xlate && subch == 1 && addr >= 0x180 &&
+ addr <= 0x18c && count) {
u32 k, n = count > 4 ? 4 : count;
u32 vv[4] = { w0, w1, w2, w3 };
@@ -24404,7 +24455,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
* here would clear the client's fence before
* the engine output landed.
*/
- if (forward && svc_gr_ahead_owned_rel_va(va)) {
+ if (xlate && forward &&
+ svc_gr_ahead_owned_rel_va(va, push_va)) {
svc_gr_ahead_owned++;
} else {
svc_pending_add(va, w2,
@@ -25055,7 +25107,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* verdict, so a bounded per-pass retry here is the
* sweep-captured launch's entire dispatch path.
*/
- if (gsp_phys_gr_forward_enabled()) {
+ if (gsp_sema_gr_xlate &&
+ gsp_phys_gr_forward_enabled()) {
if (svc_gr_cand.valid &&
svc_gr_cand.h_object == ch->h_object &&
svc_gr_ahead_req_log < 8) {
@@ -25074,7 +25127,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
* they cannot sit forever (the completion edge is
* the primary path).
*/
- if (!svc_gr_forward_pending())
+ if (gsp_sema_gr_xlate && !svc_gr_forward_pending())
svc_gr_copy_pend_run(vgpu);
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-26 11:34:05 +0000