diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 181 |
1 files changed, 146 insertions, 35 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1ea8038..58778c2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -149,6 +149,14 @@ struct svc_channel { * entries, and an INC release must not advance * the tracking semaphore twice for one push */ u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */ + u32 entry_want; /* expected_sem_payload when the pending GPFIFO + * entry was first examined: if the pool has + * advanced past it since, the entry's release + * was already delivered by another path (the + * content sweep) and the entry can be consumed + * immediately instead of waiting out the grace + * window (observed: one 8 s grace per push, + * serializing the runtime init) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ @@ -2429,23 +2437,72 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } /* - * Does this buffer start like a push? The discovered VA->PA map is + * Parse a buffer as a complete push method stream. + * + * The B06F/C5B5 method encoding carries four method shapes: SEC_OP 1 + * (INC_METHOD), 3 (NON_INC_METHOD), 4 (IMMD_DATA_METHOD, the immediate data + * lives in the header's count field so it has no payload words), 5 + * (ONE_INC) and 7 (END_PB_SEGMENT, header only); a zero word is padding. + * UVM emits INC for almost everything but a CE push can *start* with a + * NON_INC method (observed: a runtime channel's second push begins + * `0x60080002` = NON_INC count 8), so a parser that only accepts op 1 + * rejects real, readable pushes and no content rescue can validate their + * page. + * + * Returns true when every word is covered by a method (trailing zero + * padding allowed) and at least one method was seen. Arbitrary data + * (firmware, page tables, allocator lists) practically never parses to an + * exact method boundary, which is what keeps the content rescue safe. + */ +static bool svc_stream_parse(const u8 *buf, u32 len) +{ + u32 off = 0, methods = 0; + + for (;;) { + u32 hdr, op, count; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) + break; /* trailing zero padding */ + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 64 || + off + 4 + count * 4 > len) + return false; + off += 4 + count * 4; + break; + case 4: /* IMMD_DATA_METHOD: header only */ + off += 4; + break; + case 7: /* END_PB_SEGMENT: header only */ + off += 4; + break; + default: + return false; + } + if (++methods > 1024) + return false; + } + while (off < len && buf[off] == 0) + off++; + return methods > 0 && off == len; +} + +/* + * Does this buffer read as a push? The discovered VA->PA map is * content-scanned and can hand a push VA an unrelated page (observed: GSP - * firmware and kernel data pages). A real push starts with a method at - * address 0 (SET_OBJECT / INC_METHOD class bind) with a count that fits - * the buffer. + * firmware and kernel data pages), so a readable page is only trusted when + * the whole entry parses as a method stream. */ static bool svc_method_stream_plausible(const u8 *buf, u32 len) { - u32 hdr = 0, op, count; - - if (len < 8) - return false; - memcpy(&hdr, buf, 4); - op = (hdr >> 29) & 0x7; - count = (hdr >> 16) & 0xfff; - return op == 1 && count && count <= 64 && - 4 + count * 4 <= len && ((hdr & 0x1fff) << 2) == 0; + return svc_stream_parse(buf, len); } /* @@ -2821,12 +2878,32 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if (op != 1 || !count || count > 64 || + if (op == 4 || op == 7) { + /* Header-only methods (immediate data in the header + * for op 4; end-of-push segment for op 7). */ + off += 4; + continue; + } + + if ((op != 1 && op != 3 && op != 5) || !count || count > 64 || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } + if (op == 3 || op == 5) { + /* + * NON_INC / ONE_INC methods carry their own data + * words. Skip the method whole: resyncing word by + * word through the data misreads payload words as + * headers (observed: a NON_INC block whose data + * held INC-shaped words queued a bogus LAUNCH_DMA). + * No release is decoded from these encodings. + */ + off += 4 + count * 4; + continue; + } + if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) @@ -4486,27 +4563,7 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, */ static bool svc_stream_full_ok(const u8 *buf, u32 len) { - u32 off = 0, methods = 0; - - while (off + 4 <= len) { - u32 hdr, op, count; - - memcpy(&hdr, buf + off, 4); - if (hdr == 0) - break; /* trailing zero padding */ - op = (hdr >> 29) & 0x7; - count = (hdr >> 16) & 0xfff; - if (op != 1 || !count || count > 64 || - off + 4 + count * 4 > len) - return false; - off += 4 + count * 4; - if (++methods > 1024) - return false; - } - /* Trailing zero padding is common in recycled push slots. */ - while (off < len && buf[off] == 0) - off++; - return methods > 0 && off == len; + return svc_stream_parse(buf, len); } /* @@ -5015,6 +5072,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 va, idx = ch->last_get; u32 len, e0, e1, g; + /* + * Remember the payload this entry should deliver the + * first time it is examined. If the pool advances past + * it later, the release was already delivered (the + * content sweep applies releases from RAM pages before + * the pump reads their entries) and the entry must be + * consumed instead of waiting out the grace window - + * otherwise every push costs one full grace period. + */ + if (!ch->defer_since) + ch->entry_want = ch->expected_sem_payload; + if (rrc) { stalled = true; /* ring unreadable: sweep RAM */ break; @@ -5158,6 +5227,27 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!rel_seen && gsp_sema_fake_rel) svc_fake_release(vgpu, ch); else if (!rel_seen) { + /* + * Already delivered by the + * content sweep: the pool + * advanced past this entry's + * payload while it was pending + * (or decoded no release). The + * entry has nothing left to + * deliver; consume it now, do + * not pay the grace window. + */ + if (ch->entry_want && + ch->expected_sem_payload > + ch->entry_want) { + ch->defer_since = 0; + ch->rescue_fail = 0; + ch->last_get = (idx + 1) % + ch->gpfifo_entries; + svc.gpfifo_skipped++; + kfree(pb); + continue; + } if (!ch->defer_since) ch->defer_since = jiffies; @@ -5326,6 +5416,27 @@ if (gsp_sema_fake_rel && * - that consumed the client's * never-delivered wait. */ + if (ch->entry_want && + ch->expected_sem_payload > + ch->entry_want) { + /* + * Already delivered + * (the content sweep + * applied this entry's + * release from RAM): + * consume now instead + * of waiting the grace + * window per push. + */ + ch->rescue_fail = 0; + ch->defer_since = 0; + ch->last_get = + (idx + 1) % + ch->gpfifo_entries; + svc.gpfifo_skipped++; + kfree(pb); + continue; + } if (!ch->defer_since) ch->defer_since = jiffies; |