diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 554 |
1 files changed, 495 insertions, 59 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0fd78a4..7c4a946 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1177,6 +1177,17 @@ bool gsp_phys_boot_requested(void) /* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ #define GSP_PHYS_RMCALL_VA_GR_PROBE (GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL) /* + * SESSION 153-CONTINUED-96: secondary CE (COPY0) lane. A COPY0 channel on + * the same VAS translates through the same 2 MiB window PDB, but CE class + * methods are never legal on the GR0 ladder channel, so the ladder's own + * FBMEM readback (gsp_phys_ctx_readback_try) needs its own engine. The + * lane's GPFIFO ring and one-page pushbuffer get their own small-PT slots + * 0x10d/0x10e in the same 128 B PTE line the ladder's pages occupy, planted + * in pdb_build (stage 3, before any engine fetch; there is no invalidate). + */ +#define GSP_PHYS_RMCALL_VA_CE_GPFIFO (GSP_PHYS_RMCALL_VA_BASE + 0xD000ULL) +#define GSP_PHYS_RMCALL_VA_CE_PB (GSP_PHYS_RMCALL_VA_BASE + 0xE000ULL) +/* * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of * the QMD's program/param pages, so the GR engine never has to read through * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2 @@ -1462,6 +1473,13 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u /* + * SESSION 153-CONTINUED-96: the secondary COPY0 lane's TSG/channel handles + * (free in the ladder's client range: 0xcaf000c8/0xcc sit above the mode-35 + * fault-mem handles and below the registered-buffer base 0xcaf01000). + */ +#define GSP_PHYS_RMCALL_HANDLE_CE_TSG 0xcaf000c8u +#define GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL 0xcaf000ccu +/* * Mode 35, graphics-execution plan §F first experiment: the TURING_A * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7 * user-lane channel AFTER its promote - the missing half the mode-35 @@ -1697,6 +1715,19 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GOLDEN_PT0, GSP_PHYS_RMCALL_BUF_GOLDEN_PT1, GSP_PHYS_RMCALL_BUF_GOLDEN_PT2, + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane buffers (one page + * each). The instance/USERD/RAMFC/mthdbuf are the lane's own memdescs + * in the channel alloc (a second channel must never share the GR + * lane's hardware USERD); the GPFIFO ring and pushbuffer page are + * engine-reached through their planted PTE slots. + */ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO, + GSP_PHYS_RMCALL_BUF_CE_USERD, + GSP_PHYS_RMCALL_BUF_CE_INSTANCE, + GSP_PHYS_RMCALL_BUF_CE_RAMFC, + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF, + GSP_PHYS_RMCALL_BUF_CE_PB, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -2437,6 +2468,16 @@ static struct gsp_phys_boot_stage { u32 rmcall_gr_promote_size; u32 rmcall_gr_hvirt; /* mode-6 NV01_MEMORY_VIRTUAL handle */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane (same VAS). Alloc'd + * and scheduled only on a GR0 ladder (CE class is illegal on the GR + * channel); rmcall_ce_lane_valid means the token was obtained and the + * lane can accept CE copy blocks. Zero on a COPY0 ladder. + */ + u32 rmcall_ce_tsg; /* KEPLER_CHANNEL_GROUP_A handle */ + u32 rmcall_ce_channel; /* TURING_CHANNEL_GPFIFO_A handle */ + u32 rmcall_ce_token; /* its GET_WORK_SUBMIT_TOKEN result */ + bool rmcall_ce_lane_valid; /* alloc + BIND/schedules + token done */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ @@ -3275,6 +3316,10 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_promote_size = 0; boot_stage.rmcall_gr_hvirt = 0; boot_stage.rmcall_token = 0; + boot_stage.rmcall_ce_tsg = 0; + boot_stage.rmcall_ce_channel = 0; + boot_stage.rmcall_ce_token = 0; + boot_stage.rmcall_ce_lane_valid = false; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; @@ -10549,6 +10594,26 @@ static void gsp_phys_rmcall_pdb_build(void) (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); /* + * SESSION 153-CONTINUED-96: secondary CE lane GPFIFO ring (slot 0x10d) + * and its one-page pushbuffer (slot 0x10e), planted here in the same + * 128 B PTE line as the ladder's own pages - stage 3, before any + * engine fetch (no invalidate exists). Always planted (the lane's + * page/PTE cost is two pages; a disabled lane simply never rings its + * doorbell), so pdb_build stays deterministic. + */ + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_GPFIFO >> 12) & 0x1ffu) * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_PB], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_PB >> 12) & 0x1ffu) * 8); + /* * Rank-2 qmdwin: plant the dedicated module page for the forwarded * inline-QMD destination in this same small-PT line (slot 0x107, VA * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This @@ -14712,6 +14777,16 @@ static int gsp_phys_rmcall_user_lane_main_promote(void __iomem *bar0) status, rpc_result, GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va); kvfree(p); + /* + * SESSION 153-CONTINUED-96: the F2 mapped-MAIN promote is the server's + * acceptance of the ladder's own context buffer (PA+VA, bInitialize=1), + * i.e. exactly the condition rmcall_ctx_valid records. Modes 0-34 set it + * at the end of their promote step (line ~13675); mode 35 promotes here, + * so without this the state file read valid=0 and the engine-ctx + * readback gate silently skipped. + */ + if (!ret) + boot_stage.rmcall_ctx_valid = true; return ret; } @@ -15778,23 +15853,31 @@ static u32 gsp_phys_rmcall_userd_ctl_off(void) * launch path publish byte-identically; the CE caller's semaphore clear and * method-block build happen before this call. The caller must hold * gsp_phys_rm_lock. - */ -static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, - u32 words, - struct gsp_phys_rmcall_ce_submit *out) + * + * SESSION 153-CONTINUED-96: the lane carries the channel's GPFIFO ring, + * hardware USERD chunk, out-fill chunks and work-submit token, so the + * secondary COPY0 lane publishes through the identical body on its own + * ring/USERD/doorbell token (a second channel must never advance the GR + * lane's GP_PUT or ring the GR channel's doorbell). + */ +struct gsp_phys_rmcall_lane { + struct gsp_phys_boot_chunk *ring; + struct gsp_phys_boot_chunk *userd; + struct gsp_phys_boot_chunk *pb; + struct gsp_phys_boot_chunk *sem; + struct gsp_phys_boot_chunk *src; + struct gsp_phys_boot_chunk *dst; + u32 token; + const char *tag; +}; + +static int gsp_phys_rmcall_publish_lane_locked(void __iomem *bar0, + u64 pb_gpu_va, u32 words, + const struct gsp_phys_rmcall_lane *lane, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *ring = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; - struct gsp_phys_boot_chunk *userd_c = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; - struct gsp_phys_boot_chunk *sem = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; - struct gsp_phys_boot_chunk *src = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; - struct gsp_phys_boot_chunk *dst = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + struct gsp_phys_boot_chunk *ring = lane->ring; + struct gsp_phys_boot_chunk *userd_c = lane->userd; u32 *ring_va, *userd; u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; @@ -15807,8 +15890,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", - gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + pr_err("vgsp-boot: rm-call: %s USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + lane->tag, gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } /* @@ -15832,8 +15915,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, if (gpget != gpput && slotfix_log < 64) { slotfix_log++; - pr_info("vgsp-boot: rm-call: publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", - gpget, gpput, + pr_info("vgsp-boot: rm-call: %s publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", + lane->tag, gpget, gpput, gpput == gpget ? "equal" : (gpput > gpget ? "ahead" : "behind"), (gpput + GSP_PHYS_RMCALL_GPFIFO_ENTRIES - gpget) % @@ -15856,8 +15939,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) - pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", - e0, (u64)(e0 & ~3u), pb_gpu_va); + pr_err("vgsp-boot: rm-call: %s GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + lane->tag, e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); @@ -15878,12 +15961,12 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * write the formula's result (identical unless the token carries stray * high bits, which is logged). */ - runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; - ch_id = boot_stage.rmcall_token & 0xfffu; + runlist_id = (lane->token >> 16) & 0x7fu; + ch_id = lane->token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); - if (doorbell != boot_stage.rmcall_token) - pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", - boot_stage.rmcall_token, doorbell); + if (doorbell != lane->token) + pr_warn("vgsp-boot: rm-call: %s token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + lane->tag, lane->token, doorbell); /* * C3: snapshot the fault epoch immediately before the doorbell, so a * 0x1006 consumed by this submission's timeout drain is accepted only @@ -15903,14 +15986,33 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, out->doorbell = doorbell; out->runlist_id = runlist_id; out->ch_id = ch_id; - out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); - out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); - out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); - out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + out->pb_dma = (u64)gsp_phys_chunk_page_dma(lane->pb, 0); + out->sem_dma = (u64)gsp_phys_chunk_page_dma(lane->sem, 0); + out->src_dma = (u64)gsp_phys_chunk_page_dma(lane->src, 0); + out->dst_dma = (u64)gsp_phys_chunk_page_dma(lane->dst, 0); } return 0; } +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_token, + .tag = "ladder", + }; + + return gsp_phys_rmcall_publish_lane_locked(bar0, pb_gpu_va, words, + &lane, out); +} + /* * One CE copy block on the ladder channel: clear the module semaphore, build * the methods for bytes into the module pushbuffer, publish the GPFIFO entry @@ -16043,6 +16145,66 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * SESSION 153-CONTINUED-96: one CE copy block on the secondary COPY0 lane. + * Byte-for-byte the CE submission body (same method block, same guards, + * same module-semaphore clear) but on the lane's own GPFIFO ring, USERD + * chunk, one-page pushbuffer and work-submit token. Only legal when the + * lane's bringup obtained a token (rmcall_ce_lane_valid). The caller must + * hold gsp_phys_rm_lock and polls the module semaphore separately, exactly + * like the readback's bounded page loop. + */ +static int gsp_phys_rmcall_ce_lane_submit_locked( + void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 sem_payload, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_ce_token, + .tag = "CE lane", + }; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + resource_size_t bar0_len; + u32 *pb_va; + u32 words; + + if (!boot_stage.rmcall_ce_lane_valid) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_ce_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_va = gsp_phys_chunk_page_va(lane.pb, 0); + if (!pb_va || !gsp_phys_chunk_page_va(lane.ring, 0) || + !gsp_phys_chunk_page_va(lane.userd, 0) || + !gsp_phys_chunk_page_va(sem, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, + bytes, sem_payload); + if (words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + + return gsp_phys_rmcall_publish_lane_locked(bar0, + GSP_PHYS_RMCALL_VA_CE_PB, + words, &lane, out); +} + +/* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll; @@ -23861,7 +24023,7 @@ static void gsp_phys_ctx_bind_receipt(void) * is not server-memory and the identity DMA domain holds. Bounded: stops * at the first refused/timed-out page and records how many bytes landed. */ -#define GSP_PHYS_CTX_RB_BYTES (64u * 1024u) +#define GSP_PHYS_CTX_RB_BYTES 0x140000u /* MAIN 0x133000 + PATCH 0x5000 + slack */ static u8 *gsp_phys_ctx_rb_buf; static u32 gsp_phys_ctx_rb_len; @@ -23874,6 +24036,19 @@ static const u8 *gsp_phys_ctx_readback_get(u32 *len) return gsp_phys_ctx_rb_buf; } +/* + * SESSION 153-CONTINUED-96: re-arm the one-shot readback. A write to the + * gsp-phys-ctx-read debugfs file clears the tried flag (and the length), so + * the next forwarded launch re-runs the copy with the current server state. + * Safe from any context: the readback itself only runs in the single-inflight + * launch path and re-checks every gate. + */ +static void gsp_phys_ctx_readback_rearm(void) +{ + WRITE_ONCE(gsp_phys_ctx_rb_len, 0); + WRITE_ONCE(gsp_phys_ctx_rb_tried, false); +} + static void gsp_phys_ctx_readback_try(void) { struct gsp_phys_boot_chunk *sem = @@ -23882,26 +24057,26 @@ static void gsp_phys_ctx_readback_try(void) struct page *pg; u64 hpa; u32 i, bytes = 0; + u32 ctx_pages, patch_pages = 0; if (READ_ONCE(gsp_phys_ctx_rb_tried)) return; - if (!boot_stage.rmcall_ctx_valid || !boot_stage.rmcall_ctx_pa || - !boot_stage.rmcall_channel || - boot_stage.rmcall_channel_server_mem) + if (!boot_stage.rmcall_ctx_valid || !boot_stage.rmcall_ctx_pa) return; /* - * CE submits are legal on this GR0 ladder (the ladder itself copies - * through the CE after the GR channel exists), but - * gsp_phys_backend_ladder_ready() refuses them on a GR0 ladder by - * design (gsp_phys_rmcall_gr_ce_refuse is about forwarding guest CE - * work, not the ladder's own copy). Gate the readback on the same - * pieces the submission needs: claimed, RM init-done, not unloading, - * a live token. + * SESSION 153-CONTINUED-96: the copies run on the secondary COPY0 + * lane (gsp_phys_rmcall_ce_lane_bringup). CE class methods are never + * legal on the GR0 ladder channel, so the lane's readiness (alloc + + * BIND/schedules + its own token) is the hard requirement here; the + * GR channel pieces are not used by the copy. Gate on the pieces the + * lane submission needs: claimed, RM init-done, not unloading. */ + if (!boot_stage.rmcall_ce_lane_valid || !boot_stage.rmcall_ce_token) + return; if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) return; if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || - boot_stage.rm_unload_submitted || !boot_stage.rmcall_token) + boot_stage.rm_unload_submitted) return; if (gsp_phys_dma_domain_identity() != 1) return; @@ -23913,33 +24088,60 @@ static void gsp_phys_ctx_readback_try(void) if (!gsp_phys_ctx_rb_buf) return; } + /* + * SESSION 153-CONTINUED-96: copy the WHOLE MAIN context (the golden + * register image the engine restores) and the PATCH buffer (the + * context patch area the DS's SPH region may live in), in that order, + * after the 64 KB r1323/r1324 first run only proved the head. The + * dump file serves MAIN first, then PATCH; lengths are logged. + */ + ctx_pages = DIV_ROUND_UP(boot_stage.rmcall_ctx_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages > (GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE)) + ctx_pages = GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE; + if (boot_stage.rmcall_golden_fb_aux && boot_stage.rmcall_patch_size) { + patch_pages = DIV_ROUND_UP(boot_stage.rmcall_patch_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages + patch_pages > + GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE) + patch_pages = GSP_PHYS_CTX_RB_BYTES / + GSP_PHYS_PAGE_SIZE - ctx_pages; + } pg = alloc_page(GFP_KERNEL); if (!pg) return; hpa = (u64)page_to_phys(pg); - for (i = 0; i < GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE; i++) { - u64 src_va = 0, dst_va = 0, sem_va = 0; + for (i = 0; i < ctx_pages + patch_pages; i++) { + u64 src_base = (i < ctx_pages) ? + GSP_PHYS_RMCALL_VA_GR_CTX : + GSP_PHYS_RMCALL_VA_GR_PATCH; + u64 page_off = (i < ctx_pages) ? + (u64)i * GSP_PHYS_PAGE_SIZE : + (u64)(i - ctx_pages) * GSP_PHYS_PAGE_SIZE; + u64 copy_src_va = src_base + page_off; + u64 map_src_va = 0, dst_va = 0, sem_va = 0; u32 poll = 0; unsigned long deadline; int ret; mutex_lock(&gsp_phys_rm_lock); - ret = gsp_phys_rmcall_guest_map_locked(hpa, hpa, 0, &src_va, + ret = gsp_phys_rmcall_guest_map_locked(hpa, hpa, 0, &map_src_va, &dst_va, &sem_va); if (ret) { mutex_unlock(&gsp_phys_rm_lock); break; } memset((u8 *)__va(hpa), 0xab, GSP_PHYS_PAGE_SIZE); - ret = gsp_phys_rmcall_ce_submit_locked(bar0, - GSP_PHYS_RMCALL_VA_GR_CTX + - (u64)i * GSP_PHYS_PAGE_SIZE, - dst_va, GSP_PHYS_RMCALL_VA_SEM, + ret = gsp_phys_rmcall_ce_lane_submit_locked(bar0, + copy_src_va, dst_va, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_PAGE_SIZE, - GSP_PHYS_RMCALL_SEM_PAYLOAD, false, NULL); + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); mutex_unlock(&gsp_phys_rm_lock); - if (ret) + if (ret) { + pr_warn("vgsp-gr: CTX-READBACK page %u submit refused (%d) after %u bytes; stopping\n", + i, ret, bytes); break; + } deadline = jiffies + msecs_to_jiffies(250); for (;;) { poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); @@ -23951,8 +24153,11 @@ static void gsp_phys_ctx_readback_try(void) } gsp_phys_rmcall_wait_quantum(); } - if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) + if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) { + pr_warn("vgsp-gr: CTX-READBACK page %u (src VA 0x%llx) no CE release within 250 ms after %u bytes; stopping\n", + i, copy_src_va, bytes); break; + } memcpy(gsp_phys_ctx_rb_buf + (size_t)i * GSP_PHYS_PAGE_SIZE, (const u8 *)__va(hpa), GSP_PHYS_PAGE_SIZE); bytes += GSP_PHYS_PAGE_SIZE; @@ -23960,9 +24165,8 @@ static void gsp_phys_ctx_readback_try(void) __free_page(pg); WRITE_ONCE(gsp_phys_ctx_rb_len, bytes); WRITE_ONCE(gsp_phys_ctx_rb_tried, true); - pr_info("vgsp-gr: CTX-READBACK engine ctx pa=0x%llx copied=%u/%u bytes (CE copies from VA 0x%llx; server/own buffer, not the client promote; WPR2 untouched)\n", - boot_stage.rmcall_ctx_pa, bytes, GSP_PHYS_CTX_RB_BYTES, - GSP_PHYS_RMCALL_VA_GR_CTX); + pr_info("vgsp-gr: CTX-READBACK engine ctx pa=0x%llx copied=%u bytes on the CE lane (MAIN %u pages then PATCH %u pages; server/own buffer, not the client promote; WPR2 untouched)\n", + boot_stage.rmcall_ctx_pa, bytes, ctx_pages, patch_pages); } int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, @@ -28379,6 +28583,181 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) ramfc_valid); } +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane enable. On a GR0 ladder + * the ladder channel speaks GR and CE class methods are never legal on it, + * so the ladder's own FBMEM reads (gsp_phys_ctx_readback_try) need a second + * engine. Default 1 builds the lane (best-effort: a failed build only + * disables the readback, the GR ladder is unchanged); 0 skips it entirely. + */ +static bool gsp_phys_rmcall_ce_lane = 1; +module_param_named(gsp_phys_rmcall_ce_lane, gsp_phys_rmcall_ce_lane, bool, + 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_ce_lane, + "Session 153-CONTINUED-96: build and schedule the secondary COPY0 lane (COPY0 TSG + channel on the ladder VAS, engineType 9) on a GR0 ladder so the ladder's own CE copy blocks (engine FBMEM readback) have a legal engine (default 1; 0 disables the lane, the GR ladder is unchanged)"); + +static bool gsp_phys_rmcall_ce_lane_value(void) +{ + return gsp_phys_rmcall_ce_lane; +} + +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane bringup (GR0 ladders only). + * + * The ladder's own FBMEM readback (and any later diagnostic copy of engine + * state the host cannot aperture) needs a CE engine, but CE class methods + * are never legal on the GR0 ladder channel. This builds the smallest + * second lane on the SAME VAS: its own COPY0 TSG (parent device, same + * ladder VAS) and its own V5-shape channel (engineType 9, TSG parent, + * lane memdescs, GPFIFO at VA_CE_GPFIFO), then runs the historical + * COPY0 stage-4 sequence - BIND(COPY0) -> GPFIFO_SCHEDULE -> + * TSG_GPFIFO_SCHEDULE -> notif index (advisory) -> GET_WORK_SUBMIT_TOKEN - + * and stores the lane's own work-submit token. Best-effort by design: a + * failure is logged and the GR ladder continues unchanged (no stage-4 + * abort, no retry); rmcall_ce_lane_valid stays false and the readback is + * skipped. The PDB slots for the lane's GPFIFO/PB pages were planted in + * pdb_build (stage 3, before any engine fetch). + */ +static int gsp_phys_rmcall_ce_lane_bringup(void __iomem *bar0) +{ + struct gsp_phys_rmcall_tsg_params tsg; + struct gsp_phys_rmcall_channel_params ch; + struct gsp_phys_rmcall_channel_variant v; + u32 token = 0, rpc_result = 0, status = 0; + u8 sched_params[2] = { 1, 0 }; + u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; + u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; + int ret; + + if (!gsp_phys_rmcall_engine_is_gr()) + return 0; /* COPY0 ladder: the lane already is it */ + if (!gsp_phys_rmcall_ce_lane_value()) + return 0; + if (boot_stage.rmcall_ce_lane_valid) + return 0; + + /* (1) TSG on the ladder VAS bound to the COPY0 engine. */ + memset(&tsg, 0, sizeof(tsg)); + tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_CE_TSG, + GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), + false, NULL, "KEPLER_CHANNEL_GROUP_A (CE lane)", + true, NULL, NULL); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: COPY0 TSG alloc failed (%d); the lane is not built and the GR ladder continues unchanged\n", + ret); + return ret; + } + boot_stage.rmcall_ce_tsg = GSP_PHYS_RMCALL_HANDLE_CE_TSG; + + /* (2) The lane channel: V5 shape, COPY0 engine, lane memdescs. */ + memset(&v, 0, sizeof(v)); + v.name = "ce-lane-tsg-vas-pdb-skip-scrubber"; + v.id = 5; + v.tsg_parent = true; + v.flags = GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER; + v.internal_flags = GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL; + v.server_mem = false; + v.handle = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + gsp_phys_rmcall_channel_params_init(&ch, &v); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch.h_context_share = 0; /* a CE channel never joins a GR share */ + ch.gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_CE_GPFIFO); + ch.instance_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_INSTANCE); + ch.userd_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_USERD); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_RAMFC); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL, + GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), + false, NULL, "CE lane channel", true, NULL, + &status); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: channel alloc failed (%d, status=0x%x); the lane is not built and the GR ladder continues unchanged\n", + ret, status); + return ret; + } + boot_stage.rmcall_ce_channel = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + + /* (3) BIND to the COPY0 engine (the historical stage-4 sequence). */ + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, + &bind_params, sizeof(bind_params), 16, NULL, 0, + &rpc_result, &status, + "CE lane BIND_CHANNEL COPY0"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: BIND_CHANNEL (engineType=0x%x) failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + bind_params, rpc_result, status); + return ret; + } + + /* (4) GPFIFO_SCHEDULE + TSG_GPFIFO_SCHEDULE. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane GPFIFO_SCHEDULE"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: GPFIFO_SCHEDULE failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + rpc_result, status); + return ret; + } + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("vgsp-boot: rm-call: CE lane: TSG_GPFIFO_SCHEDULE advisory failure (rpc_result=0x%x status=0x%x); continuing\n", + rpc_result, status); + + /* (5) notif index (advisory) + the lane's own work-submit token. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, + ¬if_index, sizeof(notif_index), 16, NULL, 0, + &rpc_result, &status, + "CE lane SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); + if (ret) + pr_info("vgsp-boot: rm-call: CE lane: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX advisory failure (rpc_result=0x%x status=0x%x); continuing to the token\n", + rpc_result, status); + token = 0; + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, sizeof(token), + &rpc_result, &status, "CE lane GET_WORK_SUBMIT_TOKEN"); + if (ret && token != 0) { + pr_warn("vgsp-boot: rm-call: CE lane: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; proceeding\n", + rpc_result, status, token); + ret = 0; + } + if (ret || !token) { + pr_warn("vgsp-boot: rm-call: CE lane: no work-submit token (ret=%d rpc_result=0x%x status=0x%x token=0x%x); lane unusable, GR ladder continues\n", + ret, rpc_result, status, token); + return ret ? ret : -EIO; + } + boot_stage.rmcall_ce_token = token; + boot_stage.rmcall_ce_lane_valid = true; + pr_info("vgsp-boot: rm-call: CE lane READY (secondary COPY0 on the ladder VAS): hTsg=0x%x hChannel=0x%x token=0x%x (runlistId=%u chId=0x%x) gpFifoVa=0x%llx pbVa=0x%llx; CE copies on it read FBMEM the host cannot aperture\n", + boot_stage.rmcall_ce_tsg, boot_stage.rmcall_ce_channel, token, + (token >> 16) & 0x7fu, token & 0xfffu, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, GSP_PHYS_RMCALL_VA_CE_PB); + return 0; +} + static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -28560,6 +28939,22 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) boot_stage.rmcall_token = token; /* + * SESSION 153-CONTINUED-96: on a GR0 ladder, build the secondary + * COPY0 lane (own TSG/channel on the same VAS, own token) so the + * ladder's own CE copy blocks - the engine-context FBMEM readback - + * have a legal engine. Best-effort: a failure disables the readback + * only; the GR ladder below is byte-identical. On a COPY0 ladder the + * bringup is a no-op (the ladder channel already is the CE lane). + */ + { + int ce_ret = gsp_phys_rmcall_ce_lane_bringup(bar0); + + if (ce_ret) + pr_warn("vgsp-boot: rm-call: CE lane bringup failed (%d); engine FBMEM readback unavailable this boot (GR ladder unchanged)\n", + ce_ret); + } + + /* * H1: a GR0 ladder releases a semaphore on the compute engine instead * of running a CE copy. The dedicated helper performs the same guarded * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM @@ -29252,6 +29647,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_gr_promote_step, boot_stage.rmcall_gr_promote_size, boot_stage.rmcall_gr_hvirt); + /* SESSION 153-CONTINUED-96: secondary COPY0 lane (GR0 ladders only). */ + if (boot_stage.rmcall_ce_tsg || boot_stage.rmcall_ce_lane_valid) + seq_printf(m, + "rm_call_ce_lane: valid=%u hTsg=0x%x hChannel=0x%x token=0x%x gpFifoVa=0x%llx pbVa=0x%llx ceLaneParam=%u\n", + boot_stage.rmcall_ce_lane_valid, + boot_stage.rmcall_ce_tsg, + boot_stage.rmcall_ce_channel, + boot_stage.rmcall_ce_token, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, + GSP_PHYS_RMCALL_VA_CE_PB, + gsp_phys_rmcall_ce_lane_value() ? 1u : 0u); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -30885,6 +31291,26 @@ static const struct file_operations gsp_phys_ctx_dump_fops = { .llseek = default_llseek, }; +/* + * SESSION 153-CONTINUED-96: gsp-phys-ctx-read - write-only re-arm for the + * one-shot engine-context readback. `echo 1 > .../gsp-phys-ctx-read` clears + * the tried flag; the next forwarded launch re-copies MAIN+PATCH with the + * current server state. The copy itself still runs only in the launch path. + */ +static ssize_t gsp_phys_ctx_read_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + gsp_phys_ctx_readback_rearm(); + pr_info("vgsp-gr: CTX-READBACK re-armed; the next forwarded launch re-copies the engine ctx (MAIN+PATCH)\n"); + return count; +} + +static const struct file_operations gsp_phys_ctx_read_fops = { + .owner = THIS_MODULE, + .write = gsp_phys_ctx_read_write, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -31087,7 +31513,17 @@ int gsp_phys_boot_debugfs_init(void) &gsp_phys_ctx_dump_fops); if (IS_ERR_OR_NULL(file)) pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-dump creation failed (engine ctx readback unavailable)\n"); - pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-ctx-dump read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); + /* + * SESSION 153-CONTINUED-96: write-only re-arm for the readback (see + * gsp_phys_ctx_read_write). `echo 1 >` it to re-copy MAIN+PATCH at + * the next forwarded launch without a module reload. + */ + file = debugfs_create_file("gsp-phys-ctx-read", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_read_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-read creation failed (readback re-arm unavailable)\n"); + pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-ctx-dump read-only, gsp-phys-ctx-read write-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } @@ -31441,8 +31877,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, - "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 87, + "the rm-call ladder registers exactly eighty-seven DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels + 6 secondary CE lane buffers)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, |