summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c293
1 files changed, 275 insertions, 18 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8fd602c..f77b75d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1875,6 +1875,27 @@ static bool gsp_sema_gr_xlate;
module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644);
MODULE_PARM_DESC(sema_gr_xlate,
"C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)");
+/*
+ * SESSION 128 utility-channel inline-download materialization
+ * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload
+ * channels (the MEM2MEM/A140 + 2D utilities, registered but with an
+ * unresolved USERD so the pump's put-bounded entry loop never decodes
+ * them) carry the engine-side code/data downloads: the
+ * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180..
+ * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose
+ * payload the engine fetches from the next GPFIFO entry (the
+ * source-view call). Without this pass those destinations stay zero in
+ * the vidmem shadow - observed: the Vulkan client's code-heap/SASS
+ * upload (12 KB) and command-stream staging (256 B) never land, so the
+ * forwarded launch has no program content. Bounded: scan up to N
+ * entries per channel per pass, materialize once-only keyed by
+ * (push_va, content hash, dst, bytes) via the same
+ * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate.
+ */
+static u32 gsp_sema_gr_util_scan;
+module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644);
+MODULE_PARM_DESC(sema_gr_util_scan,
+ "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)");
MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)");
static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
@@ -24057,6 +24078,8 @@ static u64 svc_gr_ahead_skips;
static u64 svc_gr_ahead_owned;
/* SESSION 127: bounded witness for the sweep-caller's launch request. */
static u32 svc_gr_ahead_req_log;
+/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */
+static u64 svc_gr_ahead_inline_writes;
/*
* SESSION 127 C5C0 translator, bounded tail materialization buffer: a
@@ -24065,10 +24088,98 @@ static u32 svc_gr_ahead_req_log;
* entry plus a dedicated data slot entry). 4 KB covers the observed
* param-bank write (1568 B) in full and the first page of the big uploads.
*/
-#define SVC_AHEAD_TAIL_INLINE_MAX 4096
+#define SVC_AHEAD_TAIL_INLINE_MAX 32768
static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX];
static u64 svc_ahead_inline_trunc;
+/*
+ * SESSION 128: one-copy materialization bound for the utility/CE sweep.
+ * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch
+ * copy from a client-owned staging buffer into the vidmem shadow) moves
+ * the driver's code/constant uploads; the observed SASS/code-heap class is
+ * tens of KB, so 4 MB is a generous cap that still refuses a bogus
+ * geometry before it can walk the shadow.
+ */
+#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20)
+
+/*
+ * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an
+ * unprocessed copy-class channel carried - src bytes live in the client's
+ * staging buffer (guest RAM, resolved through the trusted walk like any
+ * operand page), dst is the vidmem page the driver staged the upload for
+ * (code heap / constant store). This is the emulated engine write for the
+ * copy the real CE would perform; it is once-only keyed by (push_va,
+ * content hash, dst, bytes) so re-scans of a stable ring are free.
+ */
+static u64 svc_ahead_copy_in_skip;
+static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h,
+ u64 push_va, u64 hash, u64 src_va, u64 dst_va,
+ u32 row, u32 rows, u32 dma)
+{
+ static u8 cb[65536];
+ u64 bytes = (u64)row * (rows ? rows : 1);
+ u64 done = 0;
+ u64 spa = 0, dpa = 0;
+ int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS;
+ int rc = 0;
+ bool src_ok, dst_ok;
+
+ if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI ||
+ dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI)
+ return;
+ /*
+ * Only the upload direction is materialized: the source must be a
+ * resolved client page in sysmem/guest RAM (the staging buffer the
+ * client CPU-wrote), and the destination must be vidmem (resolved to
+ * the FB aperture) or a not-yet-resolved VA whose bytes the VA
+ * shadow will hold. A resolved SYS destination is a copy-OUT (the
+ * engine reading vidmem into a client buffer): replaying it here
+ * would overwrite the client's buffer with shadow garbage before the
+ * producing launch ran, so it is skipped - the GR channels' copy-out
+ * machinery owns that direction.
+ */
+ src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB;
+ dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) ||
+ dap == SVC_APERTURE_FB;
+ if (!src_ok || !dst_ok) {
+ svc_ahead_copy_in_skip++;
+ if (svc_ahead_copy_in_skip <= 8)
+ pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n",
+ src_ok, dst_ok, push_va, src_va, dst_va,
+ bytes);
+ return;
+ }
+ if (bytes > SVC_AHEAD_COPY_IN_MAX) {
+ svc_ahead_inline_trunc++;
+ if (svc_ahead_inline_trunc <= 8)
+ pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n",
+ push_va, src_va, dst_va, bytes,
+ SVC_AHEAD_COPY_IN_MAX);
+ return;
+ }
+ if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes))
+ return;
+ svc_inline_once_store(push_va, hash, dst_va, (u32)bytes);
+ while (done < bytes) {
+ u32 chunk = bytes - done > sizeof(cb) ?
+ (u32)sizeof(cb) : (u32)(bytes - done);
+
+ rc = svc_read_va(vgpu, src_va + done, cb, chunk);
+ if (rc) {
+ rc = -EIO;
+ break;
+ }
+ rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk,
+ ch_h);
+ if (rc)
+ break;
+ done += chunk;
+ }
+ svc_gr_ahead_inline_writes++;
+ pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n",
+ src_va, dst_va, bytes, dma, done, rc, ch_h);
+}
+
/* One 4-byte word of a method stream that may continue into the tail. */
static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen,
u32 off)
@@ -24172,7 +24283,7 @@ done:
*/
static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
const u8 *buf, u32 len, const u8 *tail,
- u32 taillen, u64 push_va)
+ u32 taillen, u64 push_va, bool inline_only)
{
static struct svc_push_state stq; /* service thread only */
static u32 dw[66]; /* QMD window words */
@@ -24236,7 +24347,7 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
* C5C0 inline-QMD window (mirrors the main decode's intercept):
* only accumulate once the run declares itself a QMD window.
*/
- if (xlate && forward && subch >= 1 &&
+ if (xlate && forward && !inline_only && subch >= 1 &&
addr >= SVC_QMD_ADDR_A &&
addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
(stq.qmd_have ||
@@ -24292,6 +24403,7 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
bytes)) {
svc_inline_once_store(push_va, h, off_out,
bytes);
+ svc_gr_ahead_inline_writes++;
(void)svc_grce_inline_write(vgpu, off_out,
svc_ahead_inline_buf,
bytes,
@@ -24394,26 +24506,44 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
* movement, nothing to record).
*/
last_dma = dma;
- if (forward && (dma & 0x3) && have_in &&
- have_out && line_len && line_count &&
+ if ((dma & 0x3) && have_in && have_out &&
+ line_len && line_count &&
off_in && off_out) {
- svc_gr_copy_pend_add(ch->h_object,
- push_va, svc_push_hash(buf,
- len),
- off_in, off_out,
- line_len * 4, line_count,
- pitch_out ? pitch_out :
+ if (inline_only)
+ svc_ahead_copy_in(vgpu,
+ ch->h_object, push_va,
+ svc_push_hash(buf,
+ len),
+ off_in, off_out,
+ line_len * 4,
+ line_count, dma);
+ else if (forward) {
+ svc_gr_copy_pend_add(ch->h_object,
+ push_va, svc_push_hash(buf,
+ len),
+ off_in, off_out,
line_len * 4,
- pitch_in, dma, last_remap);
- cp_subch = subch;
+ line_count,
+ pitch_out ? pitch_out :
+ line_len * 4,
+ pitch_in, dma,
+ last_remap);
+ cp_subch = subch;
+ }
}
}
- } else if (xlate && subch == 1 && addr >= 0x180 &&
- addr <= 0x18c && count) {
+ } else if (xlate && (subch == 1 || subch == 2) &&
+ addr >= 0x180 && addr <= 0x18c && count) {
u32 k, n = count > 4 ? 4 : count;
u32 vv[4] = { w0, w1, w2, w3 };
- /* GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c). */
+ /*
+ * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c).
+ * The mem2mem subchannel (2, the A140/KEPLER_INLINE_
+ * TO_MEMORY_B upload lane the utility channels drive)
+ * carries the same operand methods as the compute
+ * subchannel (1).
+ */
for (k = 0; k < n; k++) {
switch (addr + 4 * k) {
case 0x180:
@@ -24439,7 +24569,7 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch
}
}
- if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
+ if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) {
u64 va;
va = ((u64)(w1 & 0xff) << 32) |
@@ -24548,7 +24678,7 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
}
swept++;
nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf,
- tail_len, va);
+ tail_len, va, false);
}
svc_gr_ahead_entries += swept;
svc_gr_ahead_rels += nrel;
@@ -24561,6 +24691,115 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu,
}
}
+/*
+ * SESSION 128: utility-channel inline-download materialization.
+ *
+ * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D
+ * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered
+ * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in
+ * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA
+ * downloads they carry (the client's code-heap/SASS upload, command-stream
+ * staging) never land anywhere the emulation can see: the destinations
+ * stay zero in the vidmem shadow.
+ *
+ * This pass scans a bounded window of the channel's GPFIFO directly (the
+ * gpfifo VA resolves through the trusted walk even when the USERD does
+ * not) and feeds each entry + its successor (the source-view call carrying
+ * the payload) through the proven sweep decoder in inline-only mode: the
+ * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2),
+ * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by
+ * svc_grce_inline_write; releases, QMD windows and copy-pend records are
+ * skipped so nothing about the client's fence/completion state changes.
+ * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring
+ * costs only the reads. Requires sema_gr_xlate; bounded by
+ * sema_gr_util_scan entries per channel per pass (0 = off).
+ */
+static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu,
+ struct svc_channel *ch)
+{
+/*
+ * The observed utility-channel pushes are the ~17 KB setup block and the
+ * two split uploads (256 B command stream + 12 KB code/heap payload); a
+ * 64 KB window covers every one of them with room to spare while keeping
+ * the two static buffers modest.
+ */
+#define SVC_UTIL_BUF (64 * 1024)
+ static u8 ubuf[SVC_UTIL_BUF];
+ static u8 nbuf[SVC_UTIL_BUF];
+ static u64 svc_gr_util_entries;
+ static u64 svc_gr_util_writes;
+ static u32 svc_gr_util_log;
+ u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64);
+ u32 swept = 0, nmade = 0;
+
+ if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan ||
+ !ch->gpfifo_va || !ch->gpfifo_entries)
+ return;
+ for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) {
+ u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE];
+ u32 e0, e1, len, rlen, tail_len = 0;
+ u64 va, before;
+
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)i * SVC_GPFIFO_ENTRY_SIZE,
+ ent, sizeof(ent)))
+ break;
+ memcpy(&e0, ent, 4);
+ memcpy(&e1, ent + 4, 4);
+ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+ len = ((e1 >> 10) & 0x1fffff) << 2;
+ if (!va && !len)
+ continue; /* unwritten slot */
+ if (!va || !len || len > SVC_PUSH_MAX)
+ continue;
+ if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI)
+ continue;
+ rlen = min_t(u32, len, sizeof(ubuf));
+ if (svc_read_va_push(vgpu, ch, va, ubuf, rlen))
+ continue;
+ /*
+ * The payload of a trailing LOAD_INLINE_DATA run rides the
+ * next GPFIFO entry (the driver's source-view GPFIFO call,
+ * observed beef0100 idx00->idx01 and idx08->idx09): read it
+ * as the tail exactly like svc_gr_ahead_sweep does.
+ */
+ if (svc_read_va(vgpu,
+ ch->gpfifo_va +
+ (u64)((i + 1) % ch->gpfifo_entries) *
+ SVC_GPFIFO_ENTRY_SIZE,
+ nent, sizeof(nent)) == 0) {
+ u32 ne0, ne1, nlen;
+ u64 nva;
+
+ memcpy(&ne0, nent, 4);
+ memcpy(&ne1, nent + 4, 4);
+ nva = ((u64)(ne1 & 0xff) << 32) |
+ (u64)(ne0 & 0xfffffffc);
+ nlen = ((ne1 >> 10) & 0x1fffff) << 2;
+ if (nva && nlen && nlen < SVC_PUSH_MAX) {
+ tail_len = min_t(u32, nlen, sizeof(nbuf));
+ if (svc_read_va_push(vgpu, ch, nva, nbuf,
+ tail_len))
+ tail_len = 0;
+ }
+ }
+ before = svc_gr_ahead_inline_writes;
+ (void)svc_gr_ahead_push(vgpu, ch, ubuf, rlen, nbuf,
+ tail_len, va, true);
+ if (svc_gr_ahead_inline_writes != before)
+ nmade++;
+ swept++; }
+ svc_gr_util_entries += swept;
+ svc_gr_util_writes += nmade;
+ if (swept && (nmade || svc_gr_util_log < 24)) {
+ svc_gr_util_log++;
+ pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n",
+ ch->h_object, swept, nmade, svc_gr_util_entries,
+ svc_gr_util_writes);
+ }
+}
+
static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
if (svc_pdb_disc_pending) {
@@ -24871,6 +25110,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
jiffies + 27 * HZ;
}
+ /*
+ * SESSION 128: utility/upload channels carry the engine's
+ * inline code/data downloads (LOAD_INLINE_DATA on the
+ * mem2mem subchannel) and the copy-class channels (eng=11/12)
+ * carry the driver's pitch copy-ins (staging -> code heap /
+ * constant store). With an unresolved USERD the put-bounded
+ * entry loop below never decodes them (the !userd_ok branch
+ * continues before it); with a resolved USERD the main
+ * decode's LOAD_INLINE_DATA/operand arms still only accept
+ * subchannel 1, so the subchannel-2 downloads are missed
+ * either way. Scan every channel's GPFIFO directly here and
+ * materialize the downloads (see svc_gr_util_sweep); the
+ * inline-only mode plus the once-only key make the overlap
+ * with the main loop and the GR sweep a no-op.
+ */
+ if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate)
+ svc_gr_util_sweep(vgpu, ch);
+
if (!ch->userd_ok) {
if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
rrc ? -1 : ring_ap, &put, &get,
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-26 08:33:08 +0000