diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 188 |
1 files changed, 140 insertions, 48 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 41a5c58..ef89a52 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -8409,6 +8409,51 @@ static bool svc_gr_cand_push_seen(u64 push_va) return false; } +/* + * r391f: once-only identity for GRCE inline writes. A push's + * LOAD_INLINE_DATA runs author bytes into the vidmem shadow; a re-decode of + * the same push (replay/rescue/rehold) must not re-run them - the target + * page may have been superseded in between (victory_reduce: the client's + * cudaMemset inline write re-zeroed the accumulator after the launch's + * copy-out, the sum=0 class). Keyed by (push_va, push content hash, dst, + * bytes): the release payload inside every push makes the hash unique per + * push, so a new client's identical-looking write never collides. Session + * scoped (cleared by svc_session_flush). Service-thread only. + */ +#define SVC_INLINE_ONCE_SLOTS 64 +static struct { + u64 push_va; + u64 hash; + u64 dst; + u32 bytes; +} svc_inline_once[SVC_INLINE_ONCE_SLOTS]; +static u32 svc_inline_once_next; + +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_INLINE_ONCE_SLOTS; i++) + if (svc_inline_once[i].push_va == push_va && + svc_inline_once[i].hash == hash && + svc_inline_once[i].dst == dst && + svc_inline_once[i].bytes == bytes) + return true; + return false; +} + +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i = svc_inline_once_next++ % SVC_INLINE_ONCE_SLOTS; + + svc_inline_once[i].push_va = push_va; + svc_inline_once[i].hash = hash; + svc_inline_once[i].dst = dst; + svc_inline_once[i].bytes = bytes; +} + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -8540,6 +8585,8 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } +static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -8548,6 +8595,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, { struct svc_push_state st; + /* r391f: the inline-write once-only key needs the push's identity. */ + svc_pass_hash = svc_push_hash(buf, len); + /* * r316: pending releases record the channel they were decoded for; * the verified pool resolver is channel-keyed (see svc_pending_add @@ -8777,27 +8827,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, slot_bytes = 0x8000 - 4; } /* - * r391f once-only (the sum=0 class): a REPLAYED - * entry must not re-run its inline writes. The ops - * are already gated by allow_ops (entry_replay - * clears it), but the inline-write gate used - * `trusted || allow_ops` - and a replay is trusted - * by construction - so every re-decode of an - * already-processed push re-wrote its inline data - * into the vidmem shadow. victory_reduce's + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the inline-write + * gate `trusted || allow_ops` re-ran the write on + * every re-decode (a replay is trusted by + * construction): victory_reduce's * cudaMemset(d_sum,0,8) is exactly such an inline * write (observed: `inline write * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 * data8=0` firing 86 us before the D2H readback, * zeroing the accumulator the launch had just - * written - the readback then reads zeros). The - * r383 replay-capture path only needs to CAPTURE - * the launch; the inline bytes were authored on the - * first (non-replay) pass. + * written - the client then read zeros). A coarse + * !replay gate is wrong (a push whose first pass + * was untrusted must still author its bytes on the + * later trusted pass); key the write by + * (push_va, push content hash, dst, bytes) - a new + * push's release payload makes the hash unique - + * and skip only a genuine duplicate. The table is + * session-scoped (a new client starts clean). */ - if ((trusted || allow_ops) && !replay && - !gsp_sema_skip_ops && - st.have_out && st.line_length) { + { u32 bytes = slot_run ? slot_bytes : st.line_length * (st.gr_line_count ? @@ -8806,42 +8857,81 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (bytes > have) bytes = have; - svc_grce_inline_write(vgpu, st.offset_out, - buf + off + 4, bytes, - ch->h_object); - /* - * fail4: hardware semantics - a LOAD_INLINE_DATA - * run advances the destination cursor by the - * bytes it wrote. Without this, the consecutive - * runs of a split 64 KB upload all land at the - * first destination and the page ends up with - * only the last run's words (the RAWA junk). - */ - st.offset_out += bytes; /* - * fail4: name the header that follows each big - * LID run - the split 64 KB upload's other - * payload carriers sit exactly there. + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the + * inline-write gate `trusted || allow_ops` + * re-ran the write on every re-decode (a replay + * is trusted by construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an + * inline write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H + * readback, zeroing the accumulator the launch + * had just written - the client then read + * zeros). A coarse !replay gate is wrong (a push + * whose first pass was untrusted must still + * author its bytes on the later trusted pass); + * key the write by (push_va, push content hash, + * dst, bytes) - the release payload inside every + * push makes the hash unique per push - and skip + * only a genuine duplicate. Session-scoped: a new + * client starts clean. */ - if (gsp_sema_dbg && len >= 4096 && - svc.next_hdr_log < 24) { - u32 noff = off + 4 + count * 4; - u32 nh = 0, nw = 0; - - if (noff + 4 <= len) - memcpy(&nh, buf + noff, 4); - if (noff + 8 <= len) - memcpy(&nw, buf + noff + 4, 4); - svc.next_hdr_log++; - pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n", - ch->h_object, push_va, off, - count, nh, nw, st.offset_out); + if ((trusted || allow_ops) && !gsp_sema_skip_ops && + st.have_out && st.line_length && bytes && + !svc_inline_once_seen(push_va, svc_pass_hash, + st.offset_out, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + st.offset_out, + bytes); + svc_grce_inline_write(vgpu, st.offset_out, + buf + off + 4, + bytes, + ch->h_object); + /* + * fail4: hardware semantics - a + * LOAD_INLINE_DATA run advances the + * destination cursor by the bytes it + * wrote. Without this, the consecutive + * runs of a split 64 KB upload all land + * at the first destination and the page + * ends up with only the last run's + * words (the RAWA junk). + */ + st.offset_out += bytes; + /* + * fail4: name the header that follows + * each big LID run - the split 64 KB + * upload's other payload carriers sit + * exactly there. + */ + if (gsp_sema_dbg && len >= 4096 && + svc.next_hdr_log < 24) { + u32 noff = off + 4 + count * 4; + u32 nh = 0, nw = 0; + + if (noff + 4 <= len) + memcpy(&nh, buf + noff, 4); + if (noff + 8 <= len) + memcpy(&nw, buf + noff + 4, + 4); + svc.next_hdr_log++; + pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n", + ch->h_object, push_va, + off, count, nh, nw, + st.offset_out); + } + } else if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", + trusted, allow_ops, st.have_out, + st.line_length, ch->h_object); } - } else if (gsp_sema_dbg && svc.grce_log < 64) { - svc.grce_log++; - pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", - trusted, allow_ops, st.have_out, - st.line_length, ch->h_object); } /* * Record the destination and the payload bytes for @@ -10819,6 +10909,8 @@ static void svc_session_flush(void) memset(svc_retry_q, 0, sizeof(svc_retry_q)); memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); memset(svc_fwd_rel, 0, sizeof(svc_fwd_rel)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; memset(svc_grce_dst_ring, 0, sizeof(svc_grce_dst_ring)); for (i = 0; i < SVC_VAS_SLOTS; i++) { if (svc_vas[i].data) |