diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 181 |
1 files changed, 138 insertions, 43 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bafe5c1..9b03cc5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -344,6 +344,10 @@ struct sema_svc { u64 ring_ops; /* op runs from ring-decoded pushes */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 copy_watch_log; /* sema_copy_watch log budget */ + u64 copy_pat_log; /* sema_copy_pat log budget */ + u64 push_pat_log; /* push-content pattern budget */ + u64 dma_skip_log; /* unrecognized-DMA budget */ + u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ @@ -2024,9 +2028,12 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * Data-path watch: for copies of exactly sema_copy_watch bytes, log * the first 8 source and destination bytes so a broken link in the * chain (wrong source page, wrong destination, silent wipe) is - * visible directly in the log. + * visible directly in the log. With sema_copy_pat set, any copy whose + * source or destination starts with that 8-byte value is logged too, + * whatever its size - this finds the push carrying a known payload. */ - if (gsp_sema_copy_watch && total == gsp_sema_copy_watch) { + if ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) || + gsp_sema_copy_pat) { u8 sb[8] = {0}, db[8] = {0}; if (src_aperture == CE_APERTURE_VID) { @@ -2045,7 +2052,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else { svc_read(vgpu, dst, db, 8); } - if (svc.copy_watch_log < 64) { + if (gsp_sema_copy_watch && total == gsp_sema_copy_watch && + svc.copy_watch_log < 64) { svc.copy_watch_log++; pr_info("sema-copy-watch: total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8_before=%02x %02x %02x %02x %02x %02x %02x %02x\n", total, dst, dst_aperture, src, src_aperture, @@ -2053,6 +2061,23 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, sb[6], sb[7], db[0], db[1], db[2], db[3], db[4], db[5], db[6], db[7]); } + if (gsp_sema_copy_pat && svc.copy_pat_log < 64) { + u64 sv = 0, dv = 0; + + memcpy(&sv, sb, 8); + memcpy(&dv, db, 8); + if (sv == gsp_sema_copy_pat || dv == gsp_sema_copy_pat) { + svc.copy_pat_log++; + pr_info("sema-copy-pat: %s h=0x%x total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8=%02x %02x %02x %02x %02x %02x %02x %02x\n", + sv == gsp_sema_copy_pat ? "SRC" : "DST", + svc.cur_ch, total, dst, dst_aperture, + src, src_aperture, + sb[0], sb[1], sb[2], sb[3], sb[4], + sb[5], sb[6], sb[7], db[0], db[1], + db[2], db[3], db[4], db[5], db[6], + db[7]); + } + } } for (o = 0; o < total; ) { @@ -2110,10 +2135,11 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops) + u32 n_ops, u32 ch_h) { u32 i; + svc.cur_ch = ch_h; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; int rc; @@ -2134,11 +2160,12 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, svc.memsets_fail++; if ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 8)) - pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d\n", + pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, - m->src_phys, m->src_aperture, rc); + m->src_phys, m->src_aperture, rc, + svc.cur_ch); } } @@ -2470,6 +2497,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, len >= 32 ? *(const u32 *)(buf + 28) : 0); } + /* + * Payload watch: with sema_copy_pat set, scan the push bytes for the + * 8-byte value and name the push that contains it (channel, VA, offset + * in the push). This finds the submission carrying a known payload + * even when its method encoding is not (yet) decoded into an op. + */ + if (gsp_sema_copy_pat && svc.push_pat_log < 32) { + u32 o; + + for (o = 0; o + 8 <= len; o += 4) { + u64 v; + + memcpy(&v, buf + o, 8); + if (v == gsp_sema_copy_pat) { + svc.push_pat_log++; + pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n", + ch_h, ch_eng, push_va, len, o, + allow_ops, trusted); + break; + } + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -2572,7 +2622,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* The push's work always runs; see the * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { - svc_run_ops(vgpu, ops, n_ops); + svc_run_ops(vgpu, ops, n_ops, ch_h); svc.ring_ops++; ops_runs++; } @@ -2668,7 +2718,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * cannot run swept-RAM false positives. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { - svc_run_ops(vgpu, ops, n_ops); + svc_run_ops(vgpu, ops, n_ops, ch_h); svc.ring_ops++; ops_runs++; } @@ -2756,7 +2806,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); - if (ch_h && svc.lda_log < 48) { + if (ch_h && svc.lda_log < + (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", ch_h, ch_eng, dma, transfer, @@ -2807,6 +2858,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; + } else if (svc.dma_skip_log < 32) { + /* + * A launch that looks like a transfer but + * carries no branch we execute: no source + * offset and no remap constant. Log the + * operand state once to expose an + * encoding we do not decode yet (a + * missing OFFSET_IN run, or a phys-mode + * value that never arrived). + */ + svc.dma_skip_log++; + pr_info("sema-dma-skip: h=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n", + ch_h, dma, st.line_length, + st.offset_in, st.offset_out, + st.have_in, st.have_out, + st.have_remap, st.have_sem); } } @@ -2844,17 +2911,24 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* - * Apply the release BEFORE running the push's - * collected work. The client's tracker only - * observes the release, and its context-init - * deadline is what the whole run races against; - * running the memsets first added their full - * execution time to every push's visible - * completion latency. Ordering is still safe: - * this is the only writer of both the semaphore - * and the vidmem shadow, and the next push is - * not decoded until this call returns. + * The push's collected work runs BEFORE the + * release is applied: the client reads the + * copied data as soon as it observes the + * release, so a release-first order lets it + * observe completion of a copy that has not + * executed yet (observed: cuMemcpyDtoH + * returning zeros). The work is small (the + * payload copies, <= 4 KB each) and runs on + * this thread before the release write, so + * the visible completion latency grows by + * only the copy itself. */ + if (allow_ops && n_ops && !gsp_sema_skip_ops) { + svc_run_ops(vgpu, ops, n_ops, ch_h); + svc.ring_ops++; + ops_runs++; + } + if (pending) { svc.pushes_pending++; if (DMA_SEM_REDUCTION_EN(dma) && @@ -2880,28 +2954,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, sem_va, cur)) { svc.releases_applied++; + svc_rel_va_seen = sem_va; svc.post_hint_until = jiffies + 10 * HZ; } } /* - * The push's collected work always runs on real - * hardware; the release only orders observers. - * Execute it here, independently of whether the - * semaphore is translatable or still behind - - * gating the work on the semaphore drops the - * page-table writes that create those very - * translations (UVM's CE memsets into vidmem). - * Ops are only ever collected from pushes read - * through a trusted ring translation. - */ - if (allow_ops && n_ops && !gsp_sema_skip_ops) { - svc_run_ops(vgpu, ops, n_ops); - svc.ring_ops++; - ops_runs++; - } - - /* * Exact-slot fallback: the releasing channel's * completion counter was enumerated from the * client's pagemap, so the release can be @@ -3741,6 +3799,22 @@ unsigned gsp_sema_ops_log = 8; /* Data-path watch: copy size whose first bytes are logged (0 = off). */ unsigned gsp_sema_copy_watch; +/* + * Data-path watch: 8-byte value matched against a copy's first source and + * destination words (either end), logged regardless of size (0 = off). Used + * to find the push that carries a known payload when its size is unknown. + */ +u64 gsp_sema_copy_pat; + +/* + * Grace window (ms) an untranslatable CE entry stays pending before the + * fallback acknowledges its release and consumes it. The push's work (data + * copies the client will read back) must run before the release the client + * synchronizes on; the translation usually lands within a service pass, so + * keep the entry pending inside the window instead of dropping the work. + */ +unsigned gsp_sema_ce_grace_ms = 8000; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, @@ -4487,16 +4561,37 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_log_stalled_va(va); /* - * With the fallback enabled the rescue - * is pointless for CE: it never - * executes the work anyway (only the - * release), and its multi-second RAM - * scan is what made the driver's - * progress waits expire. Acknowledge - * the release and move on. + * The push's work (the data copies + * the client reads back) must run + * before the release the client + * synchronizes on: acknowledging the + * release without executing the copy + * lets the client read unstaged + * memory (observed: cuMemcpyDtoH + * returning zeros after its HtoD push + * was skipped unreadable). The push + * page's translation usually lands + * within one service pass (content + * discovery or the host watcher), so + * keep the entry pending for the grace + * window first and only then fall back + * to acknowledging the release. */ + if (gsp_sema_fake_rel && ch->engine_type) { + if (!ch->defer_since) + ch->defer_since = + jiffies; + if (time_before(jiffies, + ch->defer_since + + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_nomap++; + kfree(pb); + break; + } + } if (gsp_sema_fake_rel && svc_fake_release(vgpu, ch)) { + ch->defer_since = 0; ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; |