diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 145 |
1 files changed, 101 insertions, 44 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f8925df..baac8d4 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3924,11 +3924,11 @@ static struct svc_fwd_cand svc_fwd_cand; /* * m99: guest data identity-map capacity. The forwarded kernel's global data * pages (discovered by scanning the param/const-0 page for device VAs) are - * resolved on the service thread and identity-mapped into the engine tree. - * 16K pages = 64 MB of window coverage, comfortably above the demo buffers - * (2 x 16 MB observed) with headroom. + * resolved on the service thread and relocated into the engine-visible + * region. B-2: 32K records = 2x the grown region's 16384 pages of headroom, + * so the hand-off array is never the binding limit. */ -#define SVC_GR_DATA_MAX 16384 +#define SVC_GR_DATA_MAX 32768 #define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ @@ -12360,32 +12360,36 @@ static bool svc_gr_page_inline_covers(u64 page) * Resolution only happens on the service thread; the work item turns the * guest PAs into HPAs and the backend plants the identity PTEs. */ -#define SVC_GR_DATA_RELOC_BASE 0x110000ULL +/* + * B-2 (large-operand data path): the SVC assigns each resolved operand page + * a reloc VA in the grown boot-planted region (the layout constants live in + * gsp_phys_booter.h so the two sides cannot drift; gsp_phys_gr_data_region + * selects grown vs legacy, default grown). The legacy 0x110000 window below + * the mail page (205 data pages max, 68 per pointer) remains selectable with + * gsp_phys_gr_data_region=0 for A/B trialing. + */ +#define SVC_GR_DATA_RELOC_BASE GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 +#define SVC_GR_DATA_RELOC_BASE_OLD 0x110000ULL /* * Per candidate pointer, map at most this many pages from the pointer's - * page onward (the engine-visible main-PT window has ~220 free 4 KB slots; - * the demo buffers are small on the native path). - * m120: lowered 64 -> 32 so every pointer page fits under the 100-page - * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third - * candidate run was truncated by the cap and its pointer word never - * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched). - * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400 - * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel - * accesses the tail: with 32 pages the tail read the NEXT candidate's - * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. - */ -/* - * m150 (large buffers): the bounce window was grown to 208 pages (mail page - * at slot 206). 68 pages per candidate/pointer covers a 256 KB cudaMalloc'd - * operand whose pointer sits 0x400 into its page (64 full + offset + tail), - * and up to three such operands (3x68 = 204 <= 205) fit below the mail page. - * Larger-than-256 KB operands still hit the cap; they need a chunked data - * path (future work). - */ -#define SVC_GR_DATA_PAGES_PER_PTR 68u -/* Total pages the pre-commit bounce window can carry below the mail page - * (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ -#define SVC_GR_DATA_BOUNCE_MAX 205u + * page onward. With the grown region this is a runaway guard, not a budget: + * a cold page still stops the run (stop-on-miss below), so a real operand's + * span is bounded by its extent. History: 64 -> 32 (m120) -> 33 (m140) under + * the old 100-page cap -> 68 (m150, 256 KB operands under the 205-page + * window); the old value is kept for the legacy path. + */ +#define SVC_GR_DATA_PAGES_PER_PTR 4096u +#define SVC_GR_DATA_PAGES_PER_PTR_OLD 68u +/* + * Total operand pages the engine-visible data region can carry; derived from + * the booter's shared constant (no drift). A cap hit is FAIL-CLOSED: the + * fill marks the launch stalled for the hold/retry path and returns instead + * of silently truncating later pointers - a truncated operand must never + * reach the engine. The legacy value is the pre-B-2 window below the mail + * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). + */ +#define SVC_GR_DATA_BOUNCE_MAX GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES +#define SVC_GR_DATA_BOUNCE_MAX_OLD 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ @@ -12515,10 +12519,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; bool is_arg[16]; /* m150: candidate came from the argument scan */ - u32 n_win = 0, n_cand; + u32 n_cand; u32 off = 0, i, w; u8 kind; /* + * B-2: active data-region geometry. gsp_phys_gr_data_region=1 (the + * default) selects the grown 16384-page region planted by the booter + * at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0; 0 keeps the legacy 205-page + * window below the mail page for A/B trialing. + */ + bool region_on = gsp_phys_gr_data_region_value() != 0; + u64 reloc_base = region_on ? SVC_GR_DATA_RELOC_BASE : + SVC_GR_DATA_RELOC_BASE_OLD; + u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR : + SVC_GR_DATA_PAGES_PER_PTR_OLD; + u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX : + SVC_GR_DATA_BOUNCE_MAX_OLD; + /* * m150: high (user/device-space) argument candidates and how many of * them produced at least one resolved page. None resolving is the * second-client zeros signature: the engine would run the kernel @@ -12623,9 +12640,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } - /* m110: one contiguous run of pages per candidate pointer, relocated - * into the engine-walkable main-PT window (0x110000 onward, fresh - * PTE lines; the commit-time snapshot only preserves that window). */ + /* m110/B-2: one contiguous run of pages per candidate pointer, + * relocated into the active engine-visible data region (grown region + * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the + * legacy 0x110000 window; both are boot-planted, never-rewritten PTE + * ranges the commit-time snapshot preserves). */ for (w = 0; w < n_cand; w++) { u64 base = wins[w]; u32 p; @@ -12640,17 +12659,46 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * operands (high, cudaMalloc-shaped VAs) get the full per-pointer * span. Low-VA candidates (the launch's own QMD/release/reloc * pointers) need a single page at most, and the launch's QMD VA - * is not a kernel operand at all - relocating 68 pages for it - * starved the operands that follow and blew the 205-page cap + * is not a kernel operand at all - relocating a full span for it + * starved the operands that follow and blew the old window cap * (r141: a 256 KB operand resolved 0 pages because the QMD and - * param candidates consumed the window). Excluding them lets - * three 256 KB operands (3x68 = 204) fit. + * param candidates consumed the window). */ if ((base & ~(u64)(SEMA_PAGE_SIZE - 1)) == (svc_fwd_req.gr_qmd_dev_va & ~(u64)(SEMA_PAGE_SIZE - 1))) continue; - span = (base >= 0x40000000000ULL) ? - SVC_GR_DATA_PAGES_PER_PTR : 1u; + /* + * R3 hygiene (large-buffer budget discipline): the launch's own + * param/bank pages (the qmd_word 32..47 gr_pages entries) and + * the param VA are driver bookkeeping, not kernel operands - the + * observed fills resolved a full 68-page span for the param page + * in every launch and starved the real operands under the old + * window. Skip them as data candidates; they are + * shadowed/rewritten through gr_pages/Tier 2, not this path. + */ + if (svc_fwd_req.gr_param_va && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_fwd_req.gr_param_va & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + { + bool launch_internal = false; + + for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { + const struct svc_gr_page *pp = + &svc_fwd_req.gr_pages[i]; + + if (pp->qmd_word >= 32u && + pp->qmd_word <= 47u && + (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) { + launch_internal = true; + break; + } + } + if (launch_internal) + continue; + } + span = (base >= 0x40000000000ULL) ? per_ptr : 1u; if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -12699,16 +12747,26 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) break; } } - if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) { - pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n", + if (svc_fwd_req.gr_n_data >= bounce_max) { + /* + * B-2 fail-closed cap: the active region is + * full and later pointers would be silently + * dropped (the old CAP-HIT return forwarded + * a truncated operand set). Mark the launch + * stalled so svc_gr_request holds/retries it + * via the existing bounded hold instead of + * forwarding incomplete data. + */ + svc_gr_data_stalled = true; + pr_err("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT: region full, launch held fail-closed (no silent truncation)\n", n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + reloc_base); return; } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); - rec->reloc_va = SVC_GR_DATA_RELOC_BASE + + rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; @@ -12773,8 +12831,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) n_cand, svc_fwd_req.gr_n_data, n_arg_hi); else pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", - n_cand, svc_fwd_req.gr_n_data, - (u64)SVC_GR_DATA_RELOC_BASE); + n_cand, svc_fwd_req.gr_n_data, reloc_base); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, |