diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 295 |
1 files changed, 258 insertions, 37 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f77b75d..c1426d4 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -24082,6 +24082,35 @@ static u32 svc_gr_ahead_req_log; static u64 svc_gr_ahead_inline_writes; /* + * SESSION 129 util-channel witness: the utility channel's subchannel-2 + * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm + * fired for in-push payloads on the GR channel - i.e. the failure sits in + * the split (tail-served) shape. This struct carries the per-entry story + * out of the translator for one bounded, deduplicated witness line: + * which read source answered (attempt A = the channel path, 1 = the direct + * registered-VAS walk), whether the strict parse accepted the entry, and + * what the LID arm did (saw/fired/refused a still-zero payload). + */ +struct svc_util_wit { + u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */ + u32 pa_ok; /* attempt-A parse (method count) */ + u32 pb_ok; /* attempt-B parse (method count) */ + u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */ + u32 t_len; /* tail length served */ + bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */ + bool fired; /* ... and authored its write */ + bool zero; /* ... but the tail payload was still all-zero (pending) */ + u32 dst; + u32 line_len; + u32 line_lc; + u32 count; +}; +static struct svc_util_wit svc_util_wit; +static u64 svc_util_pending_zero; +static u32 svc_util_dec_log; +static struct { u64 va; u64 h; } svc_util_dec_keys[32]; + +/* * SESSION 127 C5C0 translator, bounded tail materialization buffer: a * LOAD_INLINE_DATA run whose payload overruns the entry end is served from * the next ring entry (the driver splits large inline runs into a methods @@ -24380,6 +24409,8 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch u32 cap = count * 4; u32 bytes, k, words; u64 h = svc_push_hash(buf, len); + bool tail_served = end > len; + bool refuse_zero = false; if (nbytes > cap) nbytes = cap; @@ -24399,11 +24430,39 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch taillen, off + 4 + 4 * k) }, 4); - if (bytes && !svc_inline_once_seen(push_va, h, off_out, - bytes)) { + /* + * SESSION 129: a tail-served payload whose head is still + * all zeros means the tail source answered with an empty + * page (a stale pin shadow, or the driver has not staged + * the bytes yet) - authoring zeros over the destination + * would corrupt the code heap. Leave the write pending + * (no once-store, the sweep retries every pass) until a + * nonzero payload arrives. In-push payloads are never + * gated: the bytes are part of the flushed submit. + */ + if (inline_only && tail_served && bytes && + !memchr_inv(svc_ahead_inline_buf, 0, + min_t(u32, bytes, 16))) + refuse_zero = true; + if (inline_only) { + svc_util_wit.saw_lid = true; + svc_util_wit.dst = (u32)off_out; + svc_util_wit.line_len = line_len; + svc_util_wit.line_lc = line_count; + svc_util_wit.count = count; + if (refuse_zero) + svc_util_wit.zero = true; + } + if (refuse_zero) { + svc_util_pending_zero++; + } else if (bytes && !svc_inline_once_seen(push_va, h, + off_out, + bytes)) { svc_inline_once_store(push_va, h, off_out, bytes); svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; (void)svc_grce_inline_write(vgpu, off_out, svc_ahead_inline_buf, bytes, @@ -24422,12 +24481,19 @@ static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; - rec->dst_va = off_out; - rec->len = min_t(u32, bytes, - GSP_BRIDGE_INLINE_BYTES); - memcpy(rec->data, svc_ahead_inline_buf, - rec->len); - svc_grce_dst_ring_next++; + /* + * A refused (still-zero) tail payload is not + * recorded either: the next pass re-reads the + * tail and the real bytes land then. + */ + if (!refuse_zero) { + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } } if (end > len) break; @@ -24692,6 +24758,63 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, } /* + * SESSION 129: read a VA straight through the registered-VAS walks only + * (newest registration first), one page at a time - the exact path + * walk_probe and sema_probe prove. The util sweep's channel read can be + * shadowed by a stale injected pin / window guess for the utility VAs + * (observed r737: the subchannel-2 LID arm never fired although the push + * page content is real and the VAS walk resolves both the methods entry + * and its payload slot); this helper is the arbitration fallback that + * bypasses those sources entirely. + */ +static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t off = va & (SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used + * by the bounded util-sweep witness to select only the entries that can + * exercise the arm (one per distinct content, so a stable ring logs once). + */ +static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch) +{ + u32 k, n = 0; + + *first_subch = 0xff; + for (k = 0; k + 4 <= len; k += 4) { + u32 hdr; + + memcpy(&hdr, buf + k, 4); + if (((hdr >> 29) & 0x7) == 3 && + ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) { + if (*first_subch == 0xff) + *first_subch = (hdr >> 13) & 0x7; + n++; + } + } + return n; +} + +/* * SESSION 128: utility-channel inline-download materialization. * * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D @@ -24713,6 +24836,14 @@ static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring * costs only the reads. Requires sema_gr_xlate; bounded by * sema_gr_util_scan entries per channel per pass (0 = off). + * + * SESSION 129 read arbitration: attempt A is the channel path (unchanged); + * when A fails to read, or reads content the strict parse rejects, attempt + * B re-reads the push and the tail through svc_util_walk_read and the + * parse arbitrates - a source that yields a method stream wins. The r737 + * evidence (arm never fired while the content is real) pointed at exactly + * this: a shadowed channel read consumed the entry without a parseable + * buffer while the walk resolves the true pages. */ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch) @@ -24726,6 +24857,7 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, #define SVC_UTIL_BUF (64 * 1024) static u8 ubuf[SVC_UTIL_BUF]; static u8 nbuf[SVC_UTIL_BUF]; + static u8 bbuf[SVC_UTIL_BUF]; static u64 svc_gr_util_entries; static u64 svc_gr_util_writes; static u32 svc_gr_util_log; @@ -24737,7 +24869,7 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, return; for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; - u32 e0, e1, len, rlen, tail_len = 0; + u32 e0, e1, len, rlen; u64 va, before; if (svc_read_va(vgpu, @@ -24756,40 +24888,129 @@ static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) continue; rlen = min_t(u32, len, sizeof(ubuf)); - if (svc_read_va_push(vgpu, ch, va, ubuf, rlen)) - continue; /* - * The payload of a trailing LOAD_INLINE_DATA run rides the - * next GPFIFO entry (the driver's source-view GPFIFO call, - * observed beef0100 idx00->idx01 and idx08->idx09): read it - * as the tail exactly like svc_gr_ahead_sweep does. + * SESSION 129 read arbitration + bounded witness (see the + * sweep's header comment). Attempt A: the channel path, + * unchanged. Attempt B: the direct registered-VAS walk, used + * when A cannot read or cannot parse; the strict parse then + * arbitrates which content is the method stream. */ - if (svc_read_va(vgpu, - ch->gpfifo_va + - (u64)((i + 1) % ch->gpfifo_entries) * - SVC_GPFIFO_ENTRY_SIZE, - nent, sizeof(nent)) == 0) { - u32 ne0, ne1, nlen; - u64 nva; + { + u32 push_rc, parse_a = 0, parse_b = 0; + u32 tl_a = 0, tl_b = 0; + u32 lidn, lidsc = 0xff, slot; + u64 nva = 0; + u32 nlen = 0; + u64 hkey; + u8 *pbuf = ubuf; + bool ut = false, seen = false; + + push_rc = svc_read_va_push(vgpu, ch, va, ubuf, + rlen); + /* + * The payload of a trailing LOAD_INLINE_DATA run rides + * the next GPFIFO entry (the driver's source-view + * GPFIFO call, observed beef0100 idx00->idx01 and + * idx08->idx09): read it as the tail exactly like + * svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1; - memcpy(&ne0, nent, 4); - memcpy(&ne1, nent + 4, 4); - nva = ((u64)(ne1 & 0xff) << 32) | - (u64)(ne0 & 0xfffffffc); - nlen = ((ne1 >> 10) & 0x1fffff) << 2; - if (nva && nlen && nlen < SVC_PUSH_MAX) { - tail_len = min_t(u32, nlen, sizeof(nbuf)); + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + } + if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) { + tl_a = min_t(u32, nlen, sizeof(nbuf)); if (svc_read_va_push(vgpu, ch, nva, nbuf, - tail_len)) - tail_len = 0; + tl_a)) + tl_a = 0; + } + if (!push_rc) + parse_a = svc_ahead_parse(ubuf, rlen, nbuf, + tl_a, &ut); + if (push_rc || !parse_a) { + bool bpush = false; + + if (nva && nlen && nlen < SVC_PUSH_MAX) { + u32 want = min_t(u32, nlen, + sizeof(nbuf)); + + if (!svc_util_walk_read(vgpu, nva, + nbuf, + want)) + tl_b = want; + } + if (!svc_util_walk_read(vgpu, va, bbuf, + rlen)) { + bpush = true; + parse_b = svc_ahead_parse(bbuf, rlen, + nbuf, tl_b, + &ut); + /* + * A's buffer is authoritative only + * when A actually read it; otherwise + * (or when only B parses) B's walk + * content is the stream. + */ + if (push_rc || + (parse_b && !parse_a)) { + pbuf = bbuf; + tl_a = tl_b; + } + } + if (push_rc && !bpush) + continue; /* unreadable both ways */ } + memset(&svc_util_wit, 0, sizeof(svc_util_wit)); + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf, + tl_a, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; + /* + * Bounded witness, one line per distinct LID-carrying + * (or unparseable) entry content: which attempt was + * used, the parses, the tail resolution and what the + * LID arm did. + */ + lidn = svc_util_lid_scan(pbuf, rlen, &lidsc); + if (!lidn && parse_a) + continue; + hkey = svc_push_hash(pbuf, rlen); + for (slot = 0; slot < 32; slot++) + if (svc_util_dec_keys[slot].h == hkey && + svc_util_dec_keys[slot].va == va) { + seen = true; + break; + } + if (seen || svc_util_dec_log >= 96) + continue; + slot = svc_util_dec_log & 31; + svc_util_dec_keys[slot].va = va; + svc_util_dec_keys[slot].h = hkey; + svc_util_dec_log++; + pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n", + ch->h_object, i, va, rlen, + pbuf == bbuf ? 1u : 0u, parse_a, parse_b, + nva, tl_a, lidn, lidsc, + svc_util_wit.saw_lid ? 1 : 0, + svc_util_wit.fired ? 1 : 0, + svc_util_wit.zero ? 1 : 0, + svc_util_wit.dst, svc_util_wit.line_len, + svc_util_wit.line_lc, + rlen >= 4 ? *(const u32 *)pbuf : 0, + rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0); } - before = svc_gr_ahead_inline_writes; - (void)svc_gr_ahead_push(vgpu, ch, ubuf, rlen, nbuf, - tail_len, va, true); - if (svc_gr_ahead_inline_writes != before) - nmade++; - swept++; } + } svc_gr_util_entries += swept; svc_gr_util_writes += nmade; if (swept && (nmade || svc_gr_util_log < 24)) { |