diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 194 |
1 files changed, 190 insertions, 4 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index add8b29..a7f4b4e 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4076,6 +4076,7 @@ static u64 svc_wait_giveup; static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ +static u32 svc_pitch_log; /* r402: 2D (pitched) op builds */ static u32 svc_fill_raw_log; /* r365 FILL-RAW: raw dropped-fill dumps */ static u32 svc_lowfill_log; /* r365b LOWFILL-WIT: below-floor fills */ static u32 svc_tabcopy_log; /* r367 TABLE-COPY: table-page copies */ @@ -4098,6 +4099,7 @@ struct svc_wait_hold { }; #define SVC_WAIT_HOLD_MAX 16 static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX]; +static u32 svc_dst_div_log; /* r402: copy-dst ledger/probe divergence */ static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -4312,6 +4314,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 +#define CE_M_LINE_COUNT 0x41c +#define CE_M_PITCH_IN 0x410 +#define CE_M_PITCH_OUT 0x414 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -4386,6 +4391,19 @@ struct svc_memset_op { u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; + /* + * r402: 2D (pitched) transfers. rows > 1 selects row-wise mode: + * the op runs `rows` times with the destination advancing by + * dpitch (and the source by spitch) per row, each row carrying + * `bytes` (copy) or elem*count (fill) bytes. cudaMemset2D and + * cudaMemcpy2D issue exactly this shape (LINE_COUNT rows, + * PITCH_OUT/PITCH_IN in bytes); without it the fill/copy ran flat + * and the partial-width rows corrupted the neighbouring columns + * (victory_memset2d bad>12M, memcpy_roundtrip pitched class). + */ + u32 rows; + u32 dpitch; + u32 spitch; }; /* Decoder state carried across the methods of one push. */ @@ -4397,6 +4415,15 @@ struct svc_push_state { u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ + /* + * r402: 2D transfer geometry (CE class): PITCH_IN/PITCH_OUT in + * bytes and LINE_COUNT rows. A pitched fill/copy whose pitch + * exceeds the row length must run row-wise; a flat transfer leaves + * pitch == line_length (or no pitch method at all). + */ + u32 pitch_in, pitch_out; + u32 have_pitch_in, have_pitch_out; + u32 line_count; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -4788,8 +4815,49 @@ static int svc_fill_dst_page_pa(struct nv_vgpu_device *vgpu, u64 va, return -ENOENT; } +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, + u32 elem, u32 count, u32 phys, u32 aperture, + u64 *done_out); +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, + size_t total, u32 dst_phys, u32 dst_aperture, + u32 src_phys, u32 src_aperture, u64 *done_out); + +/* + * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per + * row with the destination advancing by dpitch. Each row keeps the whole + * per-page resolution, ledger-store and guard machinery of the 1D path + * (no new resolution semantics), and a failed row's retry simply re-runs + * the already-written rows with the same pattern - idempotent for fills. + */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, - u32 count, u32 phys, u32 aperture, u64 *done_out) + u32 count, u32 phys, u32 aperture, u64 *done_out, + u32 rows, u32 dpitch) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_fill_1d(vgpu, addr + (u64)r * dpitch, + value, elem, count, phys, aperture, + &row_done); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_fill_1d(vgpu, addr, value, elem, count, phys, aperture, + done_out); +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, + u32 count, u32 phys, u32 aperture, u64 *done_out) { u8 pat[8]; u8 bounce[256]; @@ -5965,9 +6033,41 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, u64 pa, u32 off, u32 bytes, u64 *hpa, u32 *out_off); +/* r402 row-wise wrapper: see svc_fill. The source advances by spitch. */ +/* r402: read-only resolver probe (defined with the GR fill diagnostics). */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out) + u64 *done_out, u32 rows, u32 dpitch, u32 spitch) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, + src + (u64)r * spitch, total, + dst_phys, dst_aperture, src_phys, + src_aperture, &row_done); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture, done_out); +} + +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, + u64 *done_out) { u8 bounce[256]; size_t o; @@ -6008,6 +6108,34 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, dst_shadow = true; } else if (!svc_ledger_lookup(dst, &pa, &aper)) { /* + * r402 diagnostic (bounded): name the case where the + * ledger's landing page for a copy DESTINATION + * disagrees with the live resolver (pin/walk). The + * D2H staging writes failed on the class this + * witness detects: the writer follows the ledger + * while the guest's reader holds a different page, + * so the client reads stale data (streams2 c1 + * zeros / smiley 3-of-4 passes). Read-only probe + * (svc_gr_src_probe stores nothing). + */ + if (gsp_sema_dbg && svc_dst_div_log < 64 && + dst >= SVC_VAS_FLOOR) { + u64 wpa = 0; + u32 wap = 0; + const char *src = + svc_gr_src_probe(vgpu, dst, &wpa, &wap); + + if (wpa && + (((wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + ((wap == SVC_APERTURE_FB) != + (aper == SVC_APERTURE_FB)))) { + svc_dst_div_log++; + pr_err_ratelimited("vgsp-sema: DST-DIVERGE va=0x%llx ledger=0x%llx/%u probe=%s 0x%llx/%u (write follows ledger)\n", + dst, pa, aper, src, wpa, wap); + } + } + /* * m180 unification: a previous write of this VA is * the authority for where the content lives (and so * for where a new write lands); it also spares the @@ -7275,11 +7403,12 @@ svc_wait_done: rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture, - &ce_done); + &ce_done, m->rows, m->dpitch, + m->spitch); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture, - &ce_done); + &ce_done, m->rows, m->dpitch); /* * CEAUDIT (2026-10-18 r220): per-op audit of the CE * data path - the software executor of every guest @@ -9681,6 +9810,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; + /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ + case CE_M_LINE_COUNT: + st.line_count = v0; + break; + case CE_M_PITCH_IN: + st.pitch_in = v0 & 0xffffff; + st.have_pitch_in = 1; + break; + case CE_M_PITCH_OUT: + st.pitch_out = v0 & 0xffffff; + st.have_pitch_out = 1; + break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); @@ -9855,6 +9996,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->count = st.line_length / m->elem; } + /* + * r402: a 2D fill (cudaMemset2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes at + * PITCH_OUT stride. Only a pitch + * WIDER than the row selects row + * mode; a flat fill carries + * pitch == row length (or no + * pitch method) and keeps the + * historical single-run path. + */ + if (st.have_pitch_out && + st.line_count > 1 && + st.pitch_out > st.line_length) { + m->rows = st.line_count; + m->dpitch = st.pitch_out; + } + if (m->rows > 1 && svc_pitch_log < 32) { + svc_pitch_log++; + pr_info("sema-2d: fill rows=%u dpitch=%u line=%u dst=0x%llx h=0x%x push=0x%llx\n", + m->rows, m->dpitch, + st.line_length, m->dst, + ch->h_object, push_va); + } m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; @@ -9883,6 +10048,26 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; + /* + * r402: a 2D copy (cudaMemcpy2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes with + * PITCH_IN/PITCH_OUT strides. + * Row mode only when a pitch is + * wider than the row (a flat 1D + * copy keeps pitch == length). + */ + if (st.line_count > 1 && + st.have_pitch_out && + st.pitch_out > st.line_length) { + m->rows = st.line_count; + m->dpitch = st.pitch_out; + m->spitch = st.have_pitch_in ? + (st.pitch_in ? + st.pitch_in : + st.pitch_out) : + st.pitch_out; + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -10967,6 +11152,7 @@ static void svc_session_flush(void) svc_wait_log = 0; svc_wait_pass_log = 0; svc_remapconst_log = 0; + svc_pitch_log = 0; svc_fill_raw_log = 0; svc_lowfill_log = 0; svc_tabcopy_log = 0; |