diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 140 |
1 files changed, 110 insertions, 30 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 90ea545..c50832e 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3023,6 +3023,10 @@ static u32 svc_pushalt_log; /* r707x: per-client push-read attempt witness (pump read site). */ static u32 svc_rdw_log; static u8 svc_read_last_src; +/* r715: PA the winning source of the last push read resolved to. */ +static u64 svc_read_last_pa; +/* r715: per-client push-decode branch witness (pump read site). */ +static u32 svc_pushdec_log; /* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ static u32 svc_pushfail_log; @@ -8889,18 +8893,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len) memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* - * UVM's large page-table pushes begin with a - * zero-filled header region (observed 0x1004 - * bytes) before the first method, so zeros before - * the first method are leading padding; zeros - * after one are trailing padding and end the - * stream. + * NVC06F_DMA_NOP is 0x00000000: a zero word is + * NOP padding. UVM's large page-table pushes begin + * with a zero-filled header region (observed 0x1004 + * bytes) and the GR submit pushes carry zero gaps + * between method groups (observed r715: the 1432- + * and 17316-byte pushes pad between the context + * setup and the dispatch groups), so zeros are + * skipped anywhere in the stream, not only before + * the first method or at the tail. The exact-end + * requirement and the valid-header walk below are + * the false-positive arbiters. */ - if (!methods) { - off += 4; - continue; - } - break; /* trailing zero padding */ + off += 4; + continue; } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; @@ -8920,16 +8926,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len) return false; off += 4 + count * 4; break; + case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */ + case 2: /* GRP2_USE_TERT */ case 4: /* IMMD_DATA_METHOD: header only */ - off += 4; - break; case 7: /* END_PB_SEGMENT: header only */ + /* + * r715: the class header defines op 0 as + * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT - + * 4-byte header-only formats. The GR submit + * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK, + * value 0xfff) around their semaphore groups; the + * old parser rejected the first one on every big + * push, so the whole push read implausible and + * the entries were skipped without decode + * (session 124's consume-without-decode). + */ off += 4; break; default: return false; } - if (++methods > 1024) + if (++methods > 8192) return false; } while (off < len && buf[off] == 0) @@ -12699,6 +12716,7 @@ static void svc_session_flush(void) /* r707x: per-client witness budgets. */ svc_pushfail_log = 0; svc_rdw_log = 0; + svc_pushdec_log = 0; unsigned long flags; u32 i; @@ -13879,7 +13897,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; - if (op == 4 || op == 7) /* header-only methods */ + /* + * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2 + * (GRP2_USE_TERT) are 4-byte header-only formats per the class + * header; the GR submit pushes start with 0x0001fff0 + * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page + * was rejected here and the read fell through to the walk. + */ + if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */ return true; if (op != 1 && op != 3 && op != 5) return false; @@ -14076,6 +14101,7 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; svc_read_last_src = (u8)src; + svc_read_last_pa = pa; done = true; } if (!done) { @@ -24426,8 +24452,12 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int rescue_rc; int rd_rc; bool unreadable; + bool zero_nop = false; + bool displaced = false; + bool have_ipin = false; svc_read_last_src = 0xff; + svc_read_last_pa = 0; rd_rc = svc_read_va_push(vgpu, ch, va, pb, len); unreadable = rd_rc != 0; @@ -24463,8 +24493,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 ipa; int iap; - injected = (svc_ipin_lookup(va, &ipa, - &iap) == 0); + have_ipin = + (svc_ipin_lookup(va, &ipa, + &iap) == 0); + /* + * r715: the entry's treatment + * follows the source that + * ACTUALLY answered the read, + * not a fresh page-granularity + * ipin lookup. The read falls + * through to the trusted walk + * (src 4) whenever an ipin's + * page fails its first-chunk + * guard, and taking the + * injected branch then skipped + * the only source that could + * answer: the entry was + * consumed without ever + * consulting the registered + * VAS walk (session 124). + */ + injected = have_ipin && + (svc_read_last_src == 0); /* * r496e: an all-zero push on a * content-scanned-ring channel is the @@ -24473,7 +24523,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * translation: accept it and let the * decode find no ops. */ - bool zero_nop = ch->gr_ram_ring && + zero_nop = ch->gr_ram_ring && len >= 4 && !memchr_inv(pb, 0, len); @@ -24597,18 +24647,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 *db = kmalloc(len, GFP_KERNEL); - if (db && - svc_push_read_displaced(vgpu, - ch->h_object, va, - db, len)) { - memcpy(pb, db, len); - injected = false; - unreadable = false; - } - kfree(db); - } - } - } + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + displaced = true; + } + kfree(db); + } + } + } /* * The entry's page was unreadable through @@ -24627,11 +24678,40 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, db, len); injected = false; unreadable = false; + displaced = true; } kfree(db); } /* + * r715 pushdec witness: name the branch + * the entry takes straight after the + * read, before the rescue/hold paths. + * Session 124 had an entry consumed + * between the read (rc=0 src=4) and the + * decode with no row naming why; this + * line makes the branch and the content + * (first 8 bytes) visible on every read. + * Budget: 512 rows per client (reset in + * svc_session_flush). + */ + if (gsp_sema_dbg && svc_pushdec_log < 512) { + svc_pushdec_log++; + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + svc_method_stream_plausible(pb, len) ? 1 : 0, + injected ? 1 : 0, + have_ipin ? 1 : 0, + zero_nop ? 1 : 0, + displaced ? 1 : 0, + unreadable ? 1 : 0, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + + /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA |