diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9b03cc5..db52299 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -347,6 +347,7 @@ struct sema_svc { u64 copy_pat_log; /* sema_copy_pat log budget */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ + u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ @@ -1844,6 +1845,22 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 +/* + * NVC5C0 (compute class) GRCE methods: the compute engine also has a DMA + * path whose source is the push's own inline data. CUDA uses it for small + * HtoD copies (observed: 256-byte cuMemcpyHtoD on the GR channel as + * LINE_LENGTH_IN=256, OFFSET_OUT=<device VA>, LAUNCH_DMA=0x41, then + * LOAD_INLINE_DATA with the 64 payload dwords, then the channel's + * SET_REPORT_SEMAPHORE release). The CE-shaped decoder never runs GR work; + * these methods are decoded so the inline-data write lands. + */ +#define GRCE_M_LINE_LENGTH_IN 0x180 +#define GRCE_M_LINE_COUNT 0x184 +#define GRCE_M_OFFSET_OUT_UPPER 0x188 +#define GRCE_M_OFFSET_OUT 0x18c +#define GRCE_M_LAUNCH_DMA 0x1b0 +#define GRCE_M_LOAD_INLINE_DATA 0x1b4 + /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) @@ -1886,6 +1903,7 @@ struct svc_push_state { u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; + u32 gr_line_count; /* GRCE LINE_COUNT */ u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -2133,6 +2151,58 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, return 0; } +/* + * GRCE inline-data write: the compute engine's DMA path takes its source + * from the method stream itself (LOAD_INLINE_DATA). Resolve the destination + * VA through the trusted sources (pins, discovered map, published page + * directories, targeted rm_mem resolve), pin the page and write the bytes. + */ +static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, + const u8 *data, u32 bytes, u32 ch_h) +{ + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!bytes || bytes > (1u << 20)) + return -EINVAL; + if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n", + dst_va, bytes, ch_h); + } + return -ENOENT; + } + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + pa += dst_va & 0xfff; + if (svc.grce_log < 64) { + u32 n = bytes >= 8 ? 8 : bytes; + + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx -> pa=0x%llx ap=%s bytes=%u data8=%*phN h=0x%x\n", + dst_va, pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram", + bytes, n, data, ch_h); + } + if (ap == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + bytes > vgpu->bars[1].size) + return -EINVAL; + memcpy(base + pa, data, bytes); + } else { + u32 o; + + for (o = 0; o < bytes; o += 4096) { + u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; + + if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) + return -EIO; + } + } + svc.memsets_done++; + return 0; +} + /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h) @@ -2530,6 +2600,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; + /* + * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries + * the DMA source bytes in the push itself. Run the write to + * the destination the preceding methods selected. + */ + if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && + count && off + 4 + count * 4 <= len) { + if ((trusted || allow_ops) && !gsp_sema_skip_ops && + st.have_out && st.line_length) { + u32 bytes = st.line_length * + (st.gr_line_count ? st.gr_line_count : 1); + u32 have = count * 4; + + if (bytes > have) + bytes = have; + svc_grce_inline_write(vgpu, st.offset_out, + buf + off + 4, bytes, + ch_h); + } else if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", + trusted, allow_ops, st.have_out, + st.line_length, ch_h); + } + off += 4 + count * 4; + continue; + } + if (op != 1 || !count || count > 64 || off + 4 + count * 4 > len) { off += 4; /* resync */ @@ -2762,6 +2860,45 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * carry the same method groups (the work-submission pushes that * release the client tracker semaphores). Accept both. */ + /* + * GRCE (compute class) operand methods on subchannel 1. State + * is collected here and consumed by the LOAD_INLINE_DATA run + * above; the channel's SET_REPORT_SEMAPHORE release is handled + * by the shared branch before this point. + */ + if (subch == 1) { + switch (addr) { + case GRCE_M_LINE_LENGTH_IN: + st.line_length = v0; + if (count >= 2) + st.gr_line_count = v1; + break; + case GRCE_M_LINE_COUNT: + st.gr_line_count = v0; + break; + case GRCE_M_OFFSET_OUT_UPPER: + st.offset_out = + (st.offset_out & 0xffffffffULL) | + ((u64)(v0 & 0x1ffff) << 32); + st.have_out = 1; + if (count >= 2) { + st.offset_out = + (st.offset_out & + ~0xffffffffULL) | v1; + } + break; + case GRCE_M_OFFSET_OUT: + st.offset_out = + (st.offset_out & ~0xffffffffULL) | v0; + st.have_out = 1; + break; + default: + break; + } + off += 4 + count * 4; + continue; + } + if (subch != 0 && subch != 4) { off += 4 + count * 4; continue; |