diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 165 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 336 |
2 files changed, 375 insertions, 126 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 535c8ad..f60f6c5 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17034,34 +17034,24 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * noinline so this rarely-taken wait does not grow the submit frame, which * is already near the warning bound. */ +/* + * m180: scan the request's unique tracked bounce pages and return how many + * differ from the copy-in checksum baseline. `hash_out` (optional) receives + * an FNV-1a fold of the per-page checksums - the content fingerprint the + * settle rule needs so a store ramp with a constant changed-page COUNT can + * never read as settled again. Static scratch: the only caller is the + * single-inflight ordered gsp-phys-work item (same discipline as the other + * submit-scope statics). + */ static noinline u32 -gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) +gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, + const u32 *csum_in, u32 *n_tracked, u64 *hash_out) { - /* - * B-2: one entry per page of the grown region (64 KB), static like the - * other submit-scope scratch: the only caller is the single-inflight - * ordered work item (see the static scratch note in - * gsp_phys_backend_guest_gr_submit). 64 KB must not sit on the kernel - * stack (this frame was already near the warning bound at 205 pages). - */ static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 n_all = 0, di, iter; - u32 changed = 0, prev = 0xffffffffu; - const char *why = " (TIMEOUT)"; - unsigned long wait_start = jiffies; + u32 n_all = 0, di, j, changed = 0; + u64 hash = 1469598103934665603ULL; - /* - * m179: report how many pages were actually trackable. "wrote == 0" - * from a launch with n_tracked == 0 (no data entry resolves to a - * bounce page) is not a no-execution signature and must not trigger - * the no-write retry/accept path. - */ - if (n_tracked) - *n_tracked = 0; - if (!req->n_data) - return 0; for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); u32 k; @@ -17077,41 +17067,68 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, if (!dup) checked[n_all++] = bi; } - if (!n_all) - return 0; if (n_tracked) *n_tracked = n_all; - for (iter = 0; iter < 300u; iter++) { - u32 j; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); + u32 ci, csum = 0; - changed = 0; - for (j = 0; j < n_all; j++) { - const u8 *src = gsp_phys_chunk_page_va( - dchunk, checked[j]); - u32 ci, csum = 0; + if (!src) + continue; + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(src + ci); + if (csum != csum_in[checked[j]]) + changed++; + hash ^= csum; + hash *= 1099511628211ULL; + } + if (hash_out) + *hash_out = hash; + return changed; +} - if (!src) - continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) - changed++; - } +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u32 *csum_in, u32 *n_tracked) +{ + u32 n_all = 0, iter; + u32 changed = 0; + u64 prev_hash = 0; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + /* + * m179: report how many pages were actually trackable. "wrote == 0" + * from a launch with n_tracked == 0 (no data entry resolves to a + * bounce page) is not a no-execution signature and must not trigger + * the no-write retry/accept path. + */ + if (n_tracked) + *n_tracked = 0; + if (!req->n_data) + return 0; + n_all = gsp_phys_gr_scan_pages(req, csum_in, n_tracked, NULL); + if (!n_all) + return 0; + for (iter = 0; iter < 300u; iter++) { + u64 hash = 0; + + changed = gsp_phys_gr_scan_pages(req, csum_in, NULL, &hash); if (changed >= n_all) { why = ""; break; } /* - * m162: a partial count must be stable for ~300 ms before it - * counts as settled. The old 3-iteration (~18 ms) window - * accepted a stalled store ramp or an un-dispatched grid as - * "settled": observed 1024-page vecadd outputs copied out with - * only 2 pages written (the client then verified 32736 bad - * elements). A genuinely masked/short kernel costs at most - * 300 ms extra; a ramping engine gets the time its stores - * need. + * m180 (H3): a partial count must be CONTENT-stable for + * ~300 ms before it counts as settled. The m162 count-only + * rule accepted a continuing store ramp whose page count + * happened to be constant (the always-failing class's + * "17/32 settled" copy-outs), and accepted an + * un-dispatched grid as settled. A genuinely masked/short + * kernel costs at most 300 ms extra; a ramping engine gets + * the time its stores need. */ - if (changed > 0 && changed == prev && iter >= 60u) { + if (changed > 0 && hash == prev_hash && iter >= 60u) { why = " (settled)"; break; } @@ -17119,7 +17136,7 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, why = " (no writes)"; break; } - prev = changed; + prev_hash = hash; usleep_range(4000, 6000); } pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", @@ -19231,6 +19248,56 @@ gr_page_unmapped: n_tracked, nw_attempts, want_qmd, fetched ? "OK" : "NONE"); + /* + * m180 (lottery discriminators, agent-order H1/H2). + * + * (a) Deposit identity without changing the sentinel + * contract: read BOTH qmdwin banks' sentinel words. + * Ours intact + the other bank overwritten means an + * OLDER deposit was consumed instead of this one + * (stale/mis-targeted deposit, H2); ours overwritten + * means the engine really fetched THIS deposit + * (lost-dispatch vs read-only kernel, H1); both + * intact means no deposit ever reached the engine. + */ + if (qmdwin_on && boot_stage.rmcall_gr_qmdwin.pages && + qmd_dst_mem) { + u8 *om = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank ^ 1u); + + pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x other-bank=%u sentinel=%s\n", + qw_bank, + get_unaligned_le32( + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4), + qw_bank ^ 1u, + om ? (gsp_phys_gr_qmd_fetched(om) ? + "overwritten (older deposit consumed)" : + "intact") : "n/a"); + } + /* + * (b) Late-store check: a store ramp that retires after + * the host's window is indistinguishable from a lost + * dispatch at the accept moment. Re-scan once at +1 s + * and report whether the tracked pages changed in the + * meantime; the copy-out below picks up any stores + * that arrived late, so this both classifies and heals + * the late sub-case. + */ + { + u32 late_tracked = 0; + u32 late; + + msleep(1000); + late = gsp_phys_gr_scan_pages(req, bounce_csum_in, + &late_tracked, NULL); + pr_warn("vgsp-gr: zero-write late-check: %u/%u pages changed after +1000 ms (%s)\n", + late, late_tracked, + late ? "LATE-STORES - copy-out will ship them" : + "NO-STORES - dispatch lost or read-only kernel"); + } + if (qmd_rel_hpa) { u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index aa56c5e..1cb771a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4270,6 +4270,7 @@ static bool svc_dst_watched(u64 dst) * selected. Returns 0 on success. */ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */ +static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) @@ -4291,12 +4292,19 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, if (!phys) { /* Virtual destination: the VA shadow first (its pages stay - * authoritative), then the pinned map and the VA->PA map. */ + * authoritative), then the unified landing table (the page a + * previous write of this VA landed on - always the + * authoritative location of the content), then the pinned + * map and the VA->PA map. */ u64 pa; u32 aper; if (svc_vas_has(addr)) { shadow = true; + } else if (!svc_ledger_lookup(addr, &pa, &aper)) { + addr = pa; + aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; } else if (svc_op_translate(vgpu, addr, &pa, &aper)) { if (!svc_vas_can_shadow(addr)) return -ENOENT; @@ -4323,6 +4331,22 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, if (shadow) return svc_vas_fill(addr, pat, elem, total); + /* + * C2 (data class) + m180 unification: a large virtual fill is a + * write/read pair with the memset's readers like any H2D copy - + * record its landing pages (SYS and VID alike) so the engine fill + * and the D2H read of this VA use exactly the page the fill wrote. + */ + if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && + total >= (64u << 10)) { + u32 cap = (aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + u64 po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(va0 + po, addr + po, cap); + } + if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); @@ -4333,20 +4357,6 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); - /* - * C2 (data class): a large virtual fill is a write/read pair - * with the memset's readers like any H2D copy - record its - * landing pages so the fill/D2H side reads exactly what was - * filled (memset2d was fully bypassed before this). - */ - if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && - total >= (64u << 10)) { - u64 po; - - for (po = 0; po < total; po += SEMA_PAGE_SIZE) - svc_ledger_store(va0 + po, addr + po, - SVC_APERTURE_FB); - } return 0; } @@ -4759,56 +4769,107 @@ struct svc_ledger_ent { u32 ap; }; /* - * C4 (stall/data class): a 4 MiB H2D is 1024 pages; the old 256 kept only - * the tail (early pages evicted before the launch fill ran). 2048 covers a - * single 4-8 MiB operand without eviction. Lookup is linear and bounded. + * Unified VA->landing-page authority ("R2 ledger"). + * + * m172 grew the original 256-entry ring to 2048, but a ring still evicts + * unrelated VAs wholesale: one 4 MiB H2D plus one launch's data-page + * record set is 4-6 K inserts, so an 8 MiB machine operand's early pages + * were evicted before the launch fill looked them up, and the fill fell + * back to its own (possibly stale) resolution - the exact "write path and + * read path pick different pages" class. + * + * m180: keyed hash table instead of a ring. One entry per VA page, O(1) + * update/lookup, no cross-VA eviction, and the insert of a page already + * present rewrites it in place (a write is a new landing and is always + * the truth). 16384 slots cover a full multi-buffer client's working set + * (8 MB in + 8 MB out + engine records ~ 12 K pages); hash collisions + * only occur between VA pages 64 MiB apart or across clients whose pages + * do not co-reside in time. */ -#define SVC_LEDGER_MAX 2048 +#define SVC_LEDGER_MAX (1u << 14) +#define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u) +#define SVC_LEDGER_PROBE 16u static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX]; static u32 svc_ledger_n; -static u32 svc_ledger_next; static u64 svc_ledger_diverges; static u64 svc_ledger_stores; +static u64 svc_ledger_rewrites; static u32 svc_ledger_store_log; +static inline u32 svc_ledger_home(u64 page) +{ + return (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & + SVC_LEDGER_MASK; +} + static void svc_ledger_store(u64 va, u64 pa, u32 ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); - u32 i; + u32 home = svc_ledger_home(page); + u32 p; + + for (p = 0; p < SVC_LEDGER_PROBE; p++) { + struct svc_ledger_ent *e = &svc_ledger[(home + p) & + SVC_LEDGER_MASK]; - for (i = 0; i < svc_ledger_n; i++) { - if (svc_ledger[i].va_page == page) { - svc_ledger[i].pa_page = pa_page; - svc_ledger[i].ap = ap; + if (e->va_page == page) { + /* + * A write path moving an existing page: its landing + * is the new truth. Logged (bounded) because a move + * is the write-side half of the DIVERGE class. + */ + if (e->pa_page != pa_page) { + svc_ledger_rewrites++; + if (svc_ledger_rewrites <= 16) + pr_info("sema-ledger: rewrite va=0x%llx old=0x%llx new=0x%llx ap=%u\n", + page, e->pa_page, pa_page, + ap); + } + e->pa_page = pa_page; + e->ap = ap; + return; + } + if (!e->va_page) { + e->va_page = page; + e->pa_page = pa_page; + e->ap = ap; + svc_ledger_n++; + svc_ledger_stores++; + if (svc_ledger_store_log < 64) { + svc_ledger_store_log++; + pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n", + page, pa_page, ap, svc_ledger_n); + } return; } } - if (svc_ledger_n < SVC_LEDGER_MAX) - i = svc_ledger_n++; - else - i = svc_ledger_next++ % SVC_LEDGER_MAX; - svc_ledger[i].va_page = page; - svc_ledger[i].pa_page = pa_page; - svc_ledger[i].ap = ap; + /* + * Cluster full (16 same-home VA pages co-resident): replace the home + * slot. Bounded, rare, and only affects VAs whose pages hash to the + * identical home slot. + */ + svc_ledger[home].va_page = page; + svc_ledger[home].pa_page = pa_page; + svc_ledger[home].ap = ap; svc_ledger_stores++; - if (svc_ledger_store_log < 64) { - svc_ledger_store_log++; - pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n", - page, pa_page, ap, svc_ledger_n); - } } static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); - u32 i; + u32 home = svc_ledger_home(page); + u32 p; - for (i = 0; i < svc_ledger_n; i++) { - if (svc_ledger[i].va_page == page) { - *pa = svc_ledger[i].pa_page | - (va & (SEMA_PAGE_SIZE - 1)); - *ap = svc_ledger[i].ap; + for (p = 0; p < SVC_LEDGER_PROBE; p++) { + const struct svc_ledger_ent *e = + &svc_ledger[(home + p) & SVC_LEDGER_MASK]; + + if (!e->va_page) + return -ENOENT; + if (e->va_page == page) { + *pa = e->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + *ap = e->ap; return 0; } } @@ -4819,7 +4880,7 @@ static void svc_ledger_clear(void) { memset(svc_ledger, 0, sizeof(svc_ledger)); svc_ledger_n = 0; - svc_ledger_next = 0; + svc_ledger_rewrites = 0; } static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, @@ -4844,17 +4905,40 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc_vas_has(dst)) { dst_shadow = true; + } else if (!svc_ledger_lookup(dst, &pa, &aper)) { + /* + * m180 unification: a previous write of this VA is + * the authority for where the content lives (and so + * for where a new write lands); it also spares the + * m169 stale-walk hold below, which exists precisely + * because no pin was known. + */ + dst = pa; + dst_aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + dst_prov = 1; + dst_virt = true; } else if (svc_op_translate(vgpu, dst, &pa, &aper)) { - if (!svc_vas_can_shadow(dst)) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst, &lpa, &lap)) { + dst = lpa; + dst_aperture = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + dst_prov = 1; + dst_virt = true; + } else if (!svc_vas_can_shadow(dst)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", dst, svc.cur_ch); } return -ENOENT; + } else { + dst_shadow = true; + svc.vas_shadowed++; } - dst_shadow = true; - svc.vas_shadowed++; } else { dst = pa; dst_aperture = aper; @@ -4894,12 +4978,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc_vas_has(src)) { src_shadow = true; } else if (svc_op_translate(vgpu, src, &pa, &aper)) { - if (gsp_sema_dbg && svc.op_tr_log < 64) { - svc.op_tr_log++; - pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", - src, svc.cur_ch); + u64 lpa = 0; + u32 lap = 0; + + /* + * m180 unification: read the page the last host write + * of this VA landed on when the resolver can no longer + * answer (same rule as the per-page loop below), instead + * of failing the whole copy. + */ + if (!svc_ledger_lookup(src, &lpa, &lap)) { + src = lpa; + src_aperture = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + src_prov = 1; + src_virt = true; + } else { + if (gsp_sema_dbg && svc.op_tr_log < 64) { + svc.op_tr_log++; + pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", + src, svc.cur_ch); + } + return -ENOENT; } - return -ENOENT; } else { src = pa; src_aperture = aper; @@ -4994,6 +5095,17 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, svc_fwd_cand.bytes = (u32)total; /* + * m180 unification: record the destination landing + * even when the copy itself is deferred to the + * forwarded-request path (the software copy below + * never runs then, so the in-loop store would be + * skipped and the page would have no authority for + * the fill/D2H readers). + */ + if (dst_orig >= SVC_VAS_FLOOR) + svc_ledger_store(dst_orig, dst_page, dst_kind); + + /* * P6b wave 2: with the hardware-primary path armed * (parameter on and all three prerequisites on, so the * request can queue as a guest copy and its releases @@ -5136,19 +5248,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 lpa = 0; u32 lap = 0; - if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) - return -ENOENT; - /* - * m171 (AGENT-F R2): read-after-write - when a large - * H2D wrote this device VA page through a different - * translation, read the page the write landed on. - */ - if (src_orig >= SVC_VAS_FLOOR && - !svc_ledger_lookup(src_orig + o, &lpa, &lap) && - ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != - (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || - (lap == SVC_APERTURE_FB) != - (aper == CE_APERTURE_VID))) { + if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { + /* + * m180 unification: a per-page resolution miss + * must not fail the whole copy when the unified + * table knows where this VA's content landed. + */ + if (svc_ledger_lookup(src_orig + o, &lpa, &lap)) + return -ENOENT; + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (lap == SVC_APERTURE_FB) != + (aper == CE_APERTURE_VID))) { + /* + * m171 (AGENT-F R2): read-after-write - when + * a large H2D wrote this device VA page + * through a different translation, read the + * page the write landed on. + */ svc_ledger_diverges++; if (svc_ledger_diverges <= 32) pr_info("sema-ledger: DIVERGE read va=0x%llx wrote=0x%llx read=0x%llx\n", @@ -5184,28 +5306,64 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else if (dst_virt) { u64 pa = 0; u32 aper = 0; + u64 lpa = 0; + u32 lap = 0; - if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) - return -ENOENT; + if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) { + /* + * m180 unification: a per-page resolution miss + * lands on the known page instead of failing + * the whole copy (readers already use it). + */ + if (svc_ledger_lookup(dst_orig + o, &lpa, &lap)) + return -ENOENT; + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (dst_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(dst_orig + o, &lpa, &lap) && + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (lap == SVC_APERTURE_FB) != + (aper == CE_APERTURE_VID))) { + /* + * m180 unification: keep ONE authoritative + * page per VA. A new write goes to the page + * the previous content landed on, and the + * fill/D2H readers (which prefer the table) + * stay consistent with it. + */ + svc_ledger_diverges++; + if (svc_ledger_diverges <= 32) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", + dst_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + chunk > svc_fb_shadow_size(vgpu)) return -EINVAL; memcpy(base + pa, bounce, chunk); - /* - * m171 (AGENT-F R2): a large device-dest write - * records its landing page so the later kernel - * fill / D2H read of the same VA uses it even - * if its own resolution is stale. - */ - if (dst_orig >= SVC_VAS_FLOOR && - total >= (64u << 10)) - svc_ledger_store(dst_orig + o, pa, - SVC_APERTURE_FB); } else if (svc_write(vgpu, pa, bounce, chunk)) { return -EIO; } + /* + * m171/m180: a large device-dest write records its + * landing page - SYS and VID alike, because the + * sysmem-backed UVM buffers are the dominant + * device-visible case and were the half the table + * never saw. The later kernel fill / D2H read of the + * same VA then uses exactly this page even if its own + * resolution is stale. + */ + if (dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) + svc_ledger_store(dst_orig + o, pa, + (aper == CE_APERTURE_VID) ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS); } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); @@ -5307,6 +5465,23 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, svc.memsets_done++; return 0; } + /* + * m180 unification: a page with a known landing (previous H2D/fill or + * engine write-back) is authoritative - write the inline data there + * so the engine fill / D2H readers that follow the table see it. + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + pa = lpa & ~0xfffULL; + ap = (lap == SVC_APERTURE_FB) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + goto write_landing; + } + } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; @@ -5326,6 +5501,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return -ENOENT; } svc_pin_store(dst_va & ~0xfffULL, pa, ap); +write_landing: pa += dst_va & 0xfff; if (gsp_sema_dbg && svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) { @@ -5352,6 +5528,12 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return -EIO; } } + /* + * m180: record the landing so the fill/D2H readers of this VA follow + * the bytes the engine's inline path wrote, not their own resolution. + */ + if (dst_va >= SVC_VAS_FLOOR) + svc_ledger_store(dst_va, pa, (u32)ap); svc.memsets_done++; return 0; } |