diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 762 |
1 files changed, 748 insertions, 14 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4d497d9..bd87d4c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -44,6 +44,8 @@ #include <linux/workqueue.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" @@ -476,6 +478,15 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ + unsigned long gr_setup_hold_until; /* SESSION 153-CONTINUED-54: absolute + * deadline of the GR setup-push hold: + * an unreadable/all-zero MME-setup + * (or >= 12 KB) GR push stays pending + * past the 1-failed-rescue consume + * until the demanded guest pin lands + * (0 = none) */ + u64 gr_setup_hold_va; /* entry VA the setup hold was armed for */ + u32 gr_setup_hold_idx; /* GPFIFO slot the setup hold was armed for */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -1511,6 +1522,19 @@ struct sema_svc { #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; + + /* + * SESSION 153-CONTINUED-56: the armed setup-hold's demand VA set + * (page-aligned, deduped per boot), exposed read-only to the demand + * tool via /sys/kernel/debug/vgsp/sema_setup_demand. State, not a + * log row: under vgsp-sema spam the `stalled VA #N` dmesg rows + * rotate out of the ring before the 3 s harvest reads them, so the + * setup-push pages were lost (r1090 needed a manual demand); the pin + * tool now reads this file directly each pass instead. + */ +#define SVC_SETUP_DEMAND_MAX 64 + u64 setup_demand_vas[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_count; }; static struct sema_svc svc; @@ -1544,6 +1568,107 @@ static void svc_log_stalled_va(u64 va) } /* + * SESSION 153-CONTINUED-56: record one page-aligned VA of the armed + * setup-hold demand set. Same caller as svc_log_stalled_va (the service + * thread); the debugfs reader publishes entries with a write barrier on the + * count, so a reader can never observe a torn entry (the count never + * decreases and an entry is stable once its slot is covered by the count). + */ +static void svc_setup_demand_record(u64 va) +{ + u32 i, n = svc.setup_demand_count; + + if (!va) + return; + for (i = 0; i < n; i++) + if (svc.setup_demand_vas[i] == va) + return; + if (n >= SVC_SETUP_DEMAND_MAX) + return; + svc.setup_demand_vas[n] = va; + smp_wmb(); + svc.setup_demand_count = n + 1; +} + +/* + * Read-only view of the setup-hold demand set for the pin tool: + * # sema setup-hold demand VAs (page-aligned): N + * 0x4021000 + * ... + */ +static int svc_setup_demand_show(struct seq_file *m, void *v) +{ + u32 i, n = READ_ONCE(svc.setup_demand_count); + + seq_printf(m, "# sema setup-hold demand VAs (page-aligned): %u\n", n); + for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) + seq_printf(m, "0x%llx\n", + (unsigned long long)svc.setup_demand_vas[i]); + return 0; +} + +static int svc_setup_demand_open(struct inode *inode, struct file *file) +{ + return single_open(file, svc_setup_demand_show, NULL); +} + +static const struct file_operations svc_setup_demand_fops = { + .owner = THIS_MODULE, + .open = svc_setup_demand_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* True when this module created the vgsp debugfs root (mirrors cuda bridge). */ +static bool svc_setup_demand_root_owned; + +static int svc_setup_demand_debugfs_init(void) +{ + struct dentry *root, *f; + + svc_setup_demand_root_owned = false; + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR(root)) + root = NULL; + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("sema-svc: debugfs dir vgsp unavailable; sema_setup_demand not exposed\n"); + return -ENOENT; + } + svc_setup_demand_root_owned = true; + } + f = debugfs_create_file("sema_setup_demand", 0444, root, NULL, + &svc_setup_demand_fops); + if (IS_ERR_OR_NULL(f)) { + pr_warn("sema-svc: debugfs sema_setup_demand creation failed\n"); + if (svc_setup_demand_root_owned) + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + return -ENOMEM; + } + dput(root); + pr_info("sema-svc: debugfs /sys/kernel/debug/vgsp/sema_setup_demand ready\n"); + return 0; +} + +static void svc_setup_demand_debugfs_fini(void) +{ + struct dentry *root; + + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) + return; + debugfs_lookup_and_remove("sema_setup_demand", root); + dput(root); + if (svc_setup_demand_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + } +} + +/* * Deferred readability probe for unreadable CE pushes that were consumed via * the fake-release fallback. The pump fakes immediately (work-before-release * cannot hold the channel); this queue re-attempts the read later in the @@ -8022,8 +8147,24 @@ static void svc_gl_add_ptr(u64 va) if (!va) return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ for (k = 0; k < svc_gl.n_ptr; k++) - if (svc_gl.ptr[k] == va) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) return; if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) svc_gl.ptr[svc_gl.n_ptr++] = va; @@ -8049,6 +8190,26 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA * can be typed (VTG vs PS) for the program-bytes delivery. */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { u32 j = (addr - 0x2000) / 0x40; @@ -8056,6 +8217,37 @@ static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); return; } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine @@ -8905,7 +9097,7 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, { static u32 chunkdump_log; - if (chunkdump_log < 4 && svc_gl.start_off + 8u <= at_off) { + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { u32 cur2 = svc_gl.start_off, end2, k2, wcnt; u32 cw2 = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : @@ -13676,14 +13868,240 @@ static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ */ unsigned gsp_sema_push_raw; unsigned gsp_sema_push_raw_small; +static u32 svc_mme_census_flag; module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); MODULE_PARM_DESC(sema_push_raw_small, "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); +module_param_named(sema_mme_census, svc_mme_census_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_census, + "Census MME-family method ids (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc, 0x3800..0x39fc) in beef pushes, bounded (default 0)"); static u64 svc_pushraw_last_va; static u32 svc_pushraw_last_len; static u32 svc_pushraw_n; static u32 svc_pushraw_small_n; +/* + * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D + * client) channel. The replayed draw windows raise a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) - the MME executed a macro whose code it + * could not fetch. The macro calls (CALL_MME_MACRO/DATA 0x3800..0x39fc) carry + * macro code addresses, so the macro CONTEXT state - the MME Instruction/ + * Start-address RAM loads (0x114/0x118/0x11c/0x120), the shadow RAM control + * (0x124), MME version (0x234), the MME DMA window (0x550..0x568) and the + * shadow scratch (0x3400..0x34fc) - must have reached the ladder context too. + * If those setup methods never appear in ANY client push the MME can never + * have its macro library and the replay is doomed; if they do appear, the + * next question is which push forwards them. This census answers that: it + * scans every beef push for those method ids and logs a bounded one-line + * summary (first-hit offsets + first data word). Read-only; bounded. + */ +static u32 svc_mme_census_n; + +static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + u32 nhit = 0; + char line[512]; + int lc = 0; + + if (!READ_ONCE(svc_mme_census_flag) || svc_mme_census_n >= 128) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4 <= len && nhit < 16) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if (op == 1u || op == 3u || op == 5u) + /* multi-word run: classify by its method id */ + ; + else if (op == 4u || op == 7u) + cnt = 0; + else + cnt = 0; + if ((addr >= 0x114u && addr <= 0x124u) || + addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu) || + (addr >= 0x3800u && addr <= 0x39fcu)) { + u32 v0 = 0; + + if (off + 8 <= len) + memcpy(&v0, buf + off + 4, 4); + lc += scnprintf(line + lc, sizeof(line) - lc, + "%s0x%x@0x%x(%08x)", + nhit ? "," : "", addr, off, v0); + nhit++; + } + off += (op == 1u || op == 3u || op == 5u) && cnt ? + (1u + cnt) * 4u : 4u; + } + if (nhit) { + svc_mme_census_n++; + pr_info("sema-mme-census: h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, line); + } +} + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude buffer. + * + * The draw-window replay raises a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with + * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program + * loaded. The client DOES upload the program (the census records the + * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the + * replay only ever forwards draw-window slices, so the ladder context + * never sees the setup. This buffer captures the setup runs verbatim - + * every method run whose id is in the C597 MME setup families: + * + * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER + * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words) + * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER + * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc) + * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER]) + * 0x234 SET_MME_VERSION + * 0x550..0x568 MME mem address / data RAM / DMA / reduction + * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH + * + * - in first-encounter order, so the booter can preplay them ahead of the + * first 3D window replay of a capture generation (see + * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob). + * Identical runs (a re-upload of the same program) are skipped by an + * exact-run signature table, so repeated setup pushes do not grow the + * buffer; genuinely new content appends and bumps the generation so the + * next window re-preplays. The writer is the single service thread and + * appends strictly in order; the generation only increments after a + * capture that appended, giving the booter a seqcheck against torn views. + */ +#define SVC_MME_PRELUDE_WORDS 512u +#define SVC_MME_PRELUDE_SIGS 256u +static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; +static u32 svc_mme_prelude_n; +static u32 svc_mme_prelude_gen; +static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; +static u32 svc_mme_prelude_sig_n; +static u32 svc_mme_prelude_dump_n; +static u32 svc_mme_prelude_flag = 1; +module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_prelude_cap, + "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); + +static bool svc_mme_setup_run_addr(u32 addr) +{ + return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu); +} + +static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr) +{ + u64 h = 1469598103934665603ULL ^ hdr; + u32 i; + + for (i = 1u; i < span; i++) { + u32 v; + + memcpy(&v, p + 4u * i, 4); + h ^= v; + h *= 1099511628211ULL; + } + return h; +} + +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_mme_prelude_n); + if (gen) + *gen = READ_ONCE(svc_mme_prelude_gen); + return svc_mme_prelude_buf; +} + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_mme_prelude_n; + + if (!READ_ONCE(svc_mme_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_mme_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_mme_prelude_sig_n; i++) { + if (svc_mme_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS || + svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, + span * 4u); + svc_mme_prelude_n += span; + svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1); + if (svc_mme_prelude_dump_n < 4) { + u32 k; + + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n", + h_object, push_va, len, added, + svc_mme_prelude_n, svc_mme_prelude_gen, + words0); + for (k = 0; k + 4u <= svc_mme_prelude_n && + k < 512u; k += 4u) + pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_mme_prelude_buf[k], + svc_mme_prelude_buf[k + 1], + svc_mme_prelude_buf[k + 2], + svc_mme_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_mme_prelude_n, dups); + } else if (dups && svc_mme_prelude_dump_n < 8) { + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n", + h_object, push_va, dups, svc_mme_prelude_n); + } +} + static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { @@ -14410,6 +14828,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, len >= 16 && len < 512) svc_push_raw_dump(buf, len, push_va, ch->h_object); + /* SESSION 153-CONTINUED-52: MME-family census for the beef channel. */ + if (ch) + svc_mme_census(buf, len, push_va, ch->h_object); + + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a + * trusted resolution feeds the buffer (the bytes are replayed into a + * hardware context); identical re-uploads are deduped inside. + */ + if (ch && trusted) + svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -18857,6 +19287,101 @@ static u64 svc_push_pa_ceiling(void) return top; } +/* + * SESSION 153-CONTINUED-51: deterministic push-page resolution. + * + * The direct GL client's 5-7 KB window pushes routinely read back all + * zero through the trusted chain on some boots ("sema-pushzero ... + * rescued=0", r1033-class) while the SAME VA pages answer real content + * on other boots - the learned cache (lat) can shadow a page-table walk + * that has since moved, and the client reuses/reallocates its push VAs + * across frames. The launch-page chain (svc_gr_page_note) resolves + * through the ledger then a FRESH walk; the push read chain consults the + * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk + * forever. When the trusted read answered an all-zero page, consult the + * same ledger/walk chain the launch pages use and, when a DIFFERENT page + * with non-zero content answers, adopt it AND heal the read sources + * (validated pin + lat) so every later read of this page is deterministic + * instead of a per-boot lottery. + * + * Only a page whose content differs from all-zero is adopted: a + * legitimately-zero page (padding) must never steal a correct + * translation. Returns 0 when a replacement page was adopted (pa_out / + * ap_out updated and `p` refilled), -ENOENT when nothing better exists. + */ +static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u64 net_pa, int net_ap, u8 *p, + size_t chunk, u64 *pa_out, int *ap_out) +{ + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 lpa = 0, wpa = 0; + u32 lap = 0; + int wap = SVC_APERTURE_SYS; + static u32 heal_log; + + /* 1. The ledger: the launch-page chain's landing authority. A page a + * host write or an earlier launch resolution landed on is the + * page the client's own chain agreed about - the read chain does + * not consult it at all. */ + if (!svc_ledger_lookup(page_va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of((int)lap); + + if (svc_read_ap(vgpu, lpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, lpa + off, (int)map_ap); + *pa_out = lpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(lpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + + /* 2. A FRESH walk, bypassing the learned cache: the guest may have + * remapped the VA after the lat entry was written. */ + if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 && + wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of(wap); + + if (svc_read_ap(vgpu, wpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, wpa + off, (int)map_ap); + *pa_out = wpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(wpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + /* Nothing better: restore the zero state exactly (the caller's read + * filled p with zeros and must see zeros back). */ + memset(p, 0, chunk); + return -ENOENT; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -19037,11 +19562,12 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, svc_read_last_pa = rpa; svc_lat_store(page_va, ppa, pap); - if (pushzero_log < 32) { + if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); @@ -19050,25 +19576,58 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, /* promo page unusable: put the * zero state back exactly. */ memset(p, 0, chunk); - if (!rrc && pushzero_log < 32) { + if (!rrc && pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", ch->h_object, (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa, (unsigned long long)ppa); } } - } else if (pushzero_log < 32) { + } else if (pushzero_log < 1024) { pushzero_log++; - pr_info("sema-pushzero: h=0x%x va=0x%llx src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", ch->h_object, - (unsigned long long)va, src, + (unsigned long long)va, + (unsigned)len, src, (unsigned long long)pa); } } /* + * SESSION 153-CONTINUED-51: when the zero-content + * rescue above did not produce non-zero bytes, the + * page must still be resolved deterministically - + * run the SAME ledger/fresh-walk chain the launch + * pages use (svc_gr_page_note) for this page. A + * different page with non-zero content is adopted, + * tagged src=10, and pinned so every later read of + * the page (continuation chunks and future frames) + * answers from the healed source instead of the + * learned cache. This is the r1069 class: every + * window push read zeros for a whole boot while the + * launch pages resolved fine. + */ + if (src >= 2 && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + (first ? len >= 512u : chunk >= 512u) && + !memchr_inv(p, 0, chunk)) { + u64 hpa = pa; + int hap = ap; + + if (!svc_push_page_heal(vgpu, ch->h_object, + va, pa, ap, p, + chunk, &hpa, &hap)) { + pa = hpa; + svc_read_last_src = 10; + svc_read_last_pa = hpa; + } + /* Not healed: the original (zero) source and + * its label stay; nothing was adopted. */ + } + /* * r820: name whether the trusted answer came from a * driver-declared PROMOTE_CTX buffer; the decode uses * it for the zero-NOP acceptance (see the flag). @@ -21591,8 +22150,126 @@ unsigned gsp_sema_ce_grace_ms = 25000; */ unsigned gsp_sema_push_hold_ms = 2500; +/* + * SESSION 153-CONTINUED-54: GR setup-push hold. + * + * The client's 12288 B C597 MME-setup push ("LOAD_MME_* program upload") + * reads unreadable/all-zero on the first-read lottery boots and used to be + * consumed ("skipping stale entry ... after 1 failed rescues") before the + * demand channel's injected pin could land, so the MME prelude capture + * missed the macro program for the whole boot. When such a push's first + * read fails, demand every 4K VA page of the push through the stalled-VA + * demand channel (pin_gr_stream harvests the rows and the guest resolves + + * injects the pages) and hold the entry pending PAST the rescue-consume + * until the pin lands (the cheap-wait guard re-reads when it does) or this + * window expires. Bounded by an absolute per-(va,slot) deadline; 0 + * disables the hold (the demand rows are still emitted). + * + * r1083-r1087: DEFAULT 0 (off). A 20 s hold blocks the whole ring behind + * the held entry, and on the three hold-active boots (r1083/r1084/r1085) + * the GL client derailed into the black-dud/churn class, while the r1087 + * boot (hold=0) read the beef windows fine (the wall moved back to the + * incomplete MME prelude + dispatch stall). The demand channel also cannot + * yet resolve the GL pushbuffer VA families (0x4000000+ / 0x42xxxxx), so + * the pin the hold waits for never lands and the hold only delays. Re-enable + * (e.g. sema_gr_setup_hold_ms=20000) once the guest content resolver covers + * those VA families. + */ +unsigned gsp_sema_gr_setup_hold_ms; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +/* + * Is this a GR push whose readability the MME-setup capture depends on? + * The client's C597 setup push is 12288 B; any GR push of that size is a + * candidate. A smaller push qualifies only when it is READABLE and its + * method runs hit the MME setup families (the unreadable case has no bytes + * to scan, hence the length rule alone). + */ +static bool svc_gr_setup_push(u32 len, const u8 *buf) +{ + u32 off = 0; + + if (len >= 12288u) + return true; + if (!buf) + return false; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + svc_mme_setup_run_addr(addr)) + return true; + off += ((op == 1u || op == 3u || op == 5u) && cnt) ? + (1u + cnt) * 4u : 4u; + } + return false; +} + +/* + * Demand every 4K page of the push and arm the per-(va,slot) hold. The + * demand is the existing stalled-VA channel: svc_log_stalled_va rows are + * deduped and harvested by tools/guest-diag/pin_gr_stream.py, which asks + * the guest scanner to resolve the VA set and inject the pages. Only one + * arm line per (va,slot) is logged. The loop is bounded by `page_cap` (the + * 12288 B setup push needs 3; the cap keeps a bulk push from eating the + * 256-entry stalled-VA budget). + * + * SESSION 153-CONTINUED-55: the cap was 0x8000 (8 pages). The GL client's + * observed setup push is 60216 B (16 pages); pages 1..8 were demanded but + * pages 9..15 never were, so the host's per-chunk read of the entry had no + * source for the tail chunks and the whole read failed (unreadable=1, the + * hold re-armed forever). 0x10000 covers the full 60216 B push within the + * stalled-VA budget. + */ +static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, + u32 len) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 poff; + + if (!va || !gsp_sema_gr_setup_hold_ms) + return; + if (ch->gr_setup_hold_va != va || ch->gr_setup_hold_idx != idx) { + ch->gr_setup_hold_va = va; + ch->gr_setup_hold_idx = idx; + ch->gr_setup_hold_until = + jiffies + msecs_to_jiffies(gsp_sema_gr_setup_hold_ms); + pr_info("sema-chan: GR setup hold h=0x%x entry=%u va=0x%llx len=%u armed (demanding 4K pages; pending until the pin lands, window %u ms)\n", + ch->h_object, idx, va, len, gsp_sema_gr_setup_hold_ms); + } + for (poff = 0; poff < len && poff < 0x10000u; poff += SEMA_PAGE_SIZE) { + svc_log_stalled_va(page + poff); + /* + * SESSION 153-CONTINUED-56: also publish the page in the + * debugfs demand set the pin tool reads directly, so the + * demand no longer depends on the dmesg row surviving the + * ring rotation under vgsp-sema spam. + */ + svc_setup_demand_record(page + poff); + } +} + +static bool svc_gr_setup_hold_active(const struct svc_channel *ch, u64 va, + u32 idx) +{ + return ch->gr_setup_hold_until && + ch->gr_setup_hold_va == va && + ch->gr_setup_hold_idx == idx && + time_before(jiffies, ch->gr_setup_hold_until); +} + +static void svc_gr_setup_hold_clear(struct svc_channel *ch) +{ + ch->gr_setup_hold_until = 0; + ch->gr_setup_hold_va = 0; + ch->gr_setup_hold_idx = ~0u; +} + static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out) { @@ -33002,6 +33679,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) c->unread_va = 0; c->unread_until = 0; c->defer_since = 0; + svc_gr_setup_hold_clear(c); } } /* @@ -33843,6 +34521,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; @@ -34065,7 +34744,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ - if (ch->engine_type && ch->unread_va == va && + if ((ch->engine_type || + svc_gr_setup_hold_active(ch, va, (u32)idx)) && + ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; @@ -34564,6 +35245,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; + svc_gr_setup_hold_clear(ch); /* * The entry words repeated, but the * push bytes may not have: a client @@ -35433,13 +36115,33 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) 1 : 0; ch->gr_nocap_since = jiffies; + /* + * SESSION 153-CONTINUED-54: an + * all-zero setup push gets the + * same 4K-page demand + pending + * hold as the unreadable one. + */ + if (svc_gr_setup_push(len, pb)) + svc_gr_setup_hold_arm(ch, + va, + (u32)idx, + len); } - if (gsp_sema_gr_nocap_hold_ms && + { + u32 nocap_ms = + gsp_sema_gr_nocap_hold_ms; + + if (svc_gr_setup_push(len, pb) && + gsp_sema_gr_setup_hold_ms > + nocap_ms) + nocap_ms = + gsp_sema_gr_setup_hold_ms; + if (nocap_ms && !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( - gsp_sema_gr_nocap_hold_ms)) && + nocap_ms)) && !svc_hold_bypass()) { /* Force a fresh capture on * the retry: a replay decode @@ -35458,7 +36160,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_cand_push_owed(va) ? 1 : 0, svc_gr_qmd_seen, - gsp_sema_gr_nocap_hold_ms); + nocap_ms); } kfree(pb); break; @@ -35479,6 +36181,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len, owed_done); + } } } /* @@ -35723,6 +36426,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * SESSION 153-CONTINUED-54: + * demand the setup push's 4K + * pages and hold it pending. + */ + if (!ch->engine_type && + svc_gr_setup_push(len, NULL)) + svc_gr_setup_hold_arm(ch, va, + (u32)idx, + len); + /* * GR stale-skip clock: the * rescue-failure skip gate below * requires defer_since, yet every @@ -35771,6 +36484,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->unread_ipin_pa = ipa; } /* + * SESSION 153-CONTINUED-54: a held + * GR setup push waits cheaply for + * the whole hold window (the fresh- + * pin check still releases it the + * moment the demanded pin lands). + */ + if (svc_gr_setup_hold_active(ch, va, + (u32)idx) && + time_before(ch->unread_until, + ch->gr_setup_hold_until)) + ch->unread_until = + ch->gr_setup_hold_until; + /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's @@ -36091,7 +36817,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since && time_after(jiffies, ch->defer_since + - msecs_to_jiffies(gsp_sema_gr_grace_ms))) { + msecs_to_jiffies(gsp_sema_gr_grace_ms)) && + !svc_gr_setup_hold_active(ch, va, + (u32)idx)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -36136,6 +36864,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* * m177: deliver any * withheld set even @@ -36235,6 +36964,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* GRFIX FIX1c (r214): do NOT reset the per-launch * bound record here - a consume of an unrelated entry * would re-arm the held launch's bound every pass @@ -37786,6 +38516,7 @@ static void svc_stall_watch_check(void) continue; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); ch->gr_hold_idx = ~0u; ch->gr_hold_until = 0; ch->gr_nocap_idx = ~0u; @@ -38144,6 +38875,8 @@ int gsp_sema_service_start(void) } if (gsp_cuda_bridge_init()) pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); + if (svc_setup_demand_debugfs_init()) + pr_warn("sema-svc: setup-demand debugfs unavailable\n"); return 0; } @@ -38179,6 +38912,7 @@ void gsp_sema_service_stop(void) } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ gsp_cuda_bridge_fini(); + svc_setup_demand_debugfs_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ svc_vas_free_all(); } |