summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c98
1 files changed, 80 insertions, 18 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b1ad7ba..587fb96 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4710,7 +4710,7 @@ struct svc_gr_data_win {
u64 reloc_base; /* relocated window base */
};
-struct svc_gr_cand {
+struct svc_gr_cand_rec {
bool valid;
bool offered; /* consumed by a request or a fallback */
/*
@@ -4746,7 +4746,19 @@ struct svc_gr_cand {
u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */
};
-static struct svc_gr_cand svc_gr_cand;
+/*
+ * fail4: the capture was a single slot; two concurrent launches (streams2's
+ * vadd pair on two channels) overwrote each other's candidate while the
+ * first was kept for a retry (the candidate-overwrite witness) and the
+ * engines ran crossed parameters (c1 == a2+b2). Keep up to four candidates
+ * side by side in a ring; the accessor macro addresses the current slot,
+ * so every existing use keeps working on the selected candidate, and
+ * capture/consume select the slot instead of clobbering it.
+ */
+#define SVC_GR_CAND_RING 4
+static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING];
+static u32 svc_gr_cand_pos;
+#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos])
static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
"the sema GR QMD word count must match the backend ABI");
@@ -4782,7 +4794,7 @@ static_assert(sizeof(struct svc_gr_page) == 24,
static u64 svc_push_hash(const u8 *p, u32 len);
/* QMD field decode: the exact shape the host CUDA bridge uses. */
-static void svc_gr_qmd_decode(struct svc_gr_cand *c)
+static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c)
{
const u32 *q = c->qmd;
u32 i;
@@ -4827,6 +4839,25 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
* overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined
* later in the file and this site may not reference it).
*/
+ /*
+ * fail4: never overwrite a DIFFERENT launch's unconsumed candidate -
+ * move to a free ring slot instead (same-push re-captures still
+ * refresh in place).
+ */
+ if (svc_gr_cand.valid && !svc_gr_cand.offered &&
+ svc_gr_cand.push_va != push_va) {
+ u32 ri;
+
+ for (ri = 1; ri < SVC_GR_CAND_RING; ri++) {
+ u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING;
+
+ if (!svc_gr_cand_ring[rp].valid ||
+ svc_gr_cand_ring[rp].offered) {
+ svc_gr_cand_pos = rp;
+ break;
+ }
+ }
+ }
if (svc_gr_cand.valid && svc_cand_ovw_log < 64) {
svc_cand_ovw_log++;
pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n",
@@ -4932,8 +4963,14 @@ static bool svc_fwd_defer_arm(void)
{
if (!gsp_phys_guest_mem_enabled())
return false;
- if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled())
- return true;
+ if (gsp_phys_gr_forward_enabled()) {
+ u32 ri;
+
+ for (ri = 0; ri < SVC_GR_CAND_RING; ri++)
+ if (svc_gr_cand_ring[ri].valid &&
+ !svc_gr_cand_ring[ri].offered)
+ return true;
+ }
return svc_fwd_cand.valid &&
gsp_phys_work_defer_release_enabled();
}
@@ -6186,7 +6223,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
}
if (gsp_sema_dbg &&
((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) ||
- (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64)))
+ (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 2048)))
pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n",
m->type == 1 ? "memcopy" : "memset",
m->dst, m->value, m->elem, m->count,
@@ -6204,7 +6241,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
* read-back signature: always keep room for it past the
* general trace budget so the DtoH's src/dst/rc is on record.
*/
- if ((svc.op_trace_log < 160 &&
+ if ((svc.op_trace_log < 2048 &&
(time_before(jiffies, svc.op_trace_until) ||
m->dst >= SVC_VAS_FLOOR ||
(m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) ||
@@ -7424,7 +7461,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
svc_fwd_defer_armed = false;
svc.pages_decoded++;
/* Diagnostic: name every push decoded in the post-completion window. */
- if (svc.op_trace_log < 160 &&
+ if (svc.op_trace_log < 2048 &&
time_before(jiffies, svc.op_trace_until)) {
svc.op_trace_log++;
pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n",
@@ -8178,7 +8215,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
}
/* Diagnostic: full operand state in the post-completion
* window (or for high device VAs), bounded. */
- if (svc.op_trace_log < 160 && ch->h_object &&
+ if (svc.op_trace_log < 2048 && ch->h_object &&
(time_before(jiffies, svc.op_trace_until) ||
st.offset_out >= SVC_VAS_FLOOR ||
st.offset_in >= SVC_VAS_FLOOR)) {
@@ -8300,16 +8337,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
* executor's population is on record even when the
* 64 KB copy witness below never fires for it.
*/
- if (n_ops != n_ops_before && svc.opadd_log < 256) {
+ if (n_ops != n_ops_before) {
struct svc_memset_op *lm = &ops[n_ops - 1];
- svc.opadd_log++;
- pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n",
- ch->h_object, lm->type, lm->dst,
- lm->src, lm->bytes, lm->elem,
- lm->count, lm->aperture,
- lm->src_aperture, lm->phys,
- lm->src_phys);
+ /*
+ * fail4 A/B note: a CE-style auto-advance of
+ * st.offset_out per op was tried here and
+ * reverted - it did not change the memset
+ * coverage and the driver's per-op re-sets are
+ * already decoded, so the executed dsts match
+ * the stream.
+ */
+ if (svc.opadd_log < 2048) {
+ svc.opadd_log++;
+ pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n",
+ ch->h_object, lm->type,
+ lm->dst, lm->src, lm->bytes,
+ lm->elem, lm->count,
+ lm->aperture, lm->src_aperture,
+ lm->phys, lm->src_phys);
+ }
}
/*
@@ -16329,8 +16376,23 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
* The candidate is per-channel: a capture left unconsumed by an
* earlier pass (e.g. the entry was held before its request) must never
* be queued against a different channel's handle by that channel's
- * own end-of-pass request call.
+ * own end-of-pass request call. fail4: with several reserved ring
+ * slots, select THIS channel's oldest unconsumed candidate first.
*/
+ if (gsp_phys_gr_forward_enabled()) {
+ u32 ri;
+
+ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+ u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING;
+
+ if (svc_gr_cand_ring[rp].valid &&
+ !svc_gr_cand_ring[rp].offered &&
+ svc_gr_cand_ring[rp].h_object == ch->h_object) {
+ svc_gr_cand_pos = rp;
+ break;
+ }
+ }
+ }
if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
svc_gr_cand.h_object != ch->h_object)
return false;
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-20 20:50:12 +0000