diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 485 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 10 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 26 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 10 |
4 files changed, 509 insertions, 22 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index a033022..4409a29 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -236,6 +236,25 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u /* + * H1 ladder engine selection (gsp_phys_rmcall_engine). GR0 is engine type 1 + * (NV2080_ENGINE_TYPE_GRAPHICS / ENG_GR(0)); the guest compute-object path + * allocates TURING_COMPUTE_A 0xc5c0 on the channel + * (generated/g_gpu_class_list.c: { TURING_COMPUTE_A, ENG_GR(0) }) and its + * methods live on the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1, + * class/cla06fsubch.h:29). The local handle 0xcaf0002c is free in the + * ladder's client handle range (0xcaf00000..0xcaf7ffff). The compute + * class's alloc params are NV_GR_ALLOCATION_PARAMETERS + * (common/sdk/nvidia/inc/nvos.h:2322-2328): version 2, flags 0, + * size sizeof (0x10), caps is an output. + */ +#define GSP_PHYS_RMCALL_ENGINE_GR0 0x1u +#define GSP_PHYS_RMCALL_CLASS_COMPUTE 0xc5c0u +#define GSP_PHYS_RMCALL_HANDLE_GR_OBJ 0xcaf0002cu +#define GSP_PHYS_RMCALL_GR_SUBCHANNEL 1u +#define GSP_PHYS_RMCALL_GR_PARAM_VERSION 0x2u +#define GSP_PHYS_RMCALL_GR_PARAM_SIZE 0x10u + +/* * Work-submit token notifier slot. The default index is * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1 * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex @@ -532,6 +551,22 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN 0x418u #define GSP_PHYS_RMCALL_CE_LINE_COUNT 0x41cu +/* + * H1 GR (TURING_COMPUTE_A 0xc5c0) release methods on the compute subchannel: + * SET_OBJECT 0x0000 (class/clc5c0.h:34) and SET_REPORT_SEMAPHORE_A..D at + * 0x1b00/0x1b04/0x1b08/0x1b0c (class/clc5c0.h:622-631). A/B are the + * semaphore GPU VA upper/lower, C the payload and D the operation word: + * OPERATION=RELEASE (bits 1:0 = 0), FLUSH_DISABLE=0, REDUCTION_ENABLE=0, + * AWAKEN_ENABLE=1 (bit 20) and STRUCTURE_SIZE=ONE_WORD (bit 28), i.e. + * 0x10100000 (class/clc5c0.h:632-644). + */ +#define GSP_PHYS_RMCALL_GR_SET_OBJECT 0x000u +#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A 0x1b00u +#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B 0x1b04u +#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C 0x1b08u +#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D 0x1b0cu +#define GSP_PHYS_RMCALL_GR_SEM_RELEASE_D 0x10100000u + /* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3= * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE VIRTUAL * bits12/13=0 (clc5b5.h:69-108); exactly the UVM user-channel pitch copy @@ -1010,6 +1045,17 @@ static struct gsp_phys_boot_stage { u32 rmcall_channel_variant; /* winning stage-3 variant (0=none; see * the variant table for its number) */ bool rmcall_channel_server_mem; /* winning variant had server-allocated mem */ + /* + * H1 engine snapshot: gsp_phys_rmcall_engine (0=COPY0, 1=GR0) is + * snapshotted when stage 3 starts and every later step uses the + * snapshot, so a runtime sysfs change can never mix engines. A later + * change is logged once and ignored. + */ + u32 rmcall_engine; /* COPY0 or GR0 (stage-3 snapshot) */ + bool rmcall_engine_valid; /* the stage-3 snapshot was taken */ + bool rmcall_engine_change_logged; /* mid-session change noted once */ + bool rmcall_gr_ce_refused_logged; /* CE-on-GR refusal noted once */ + u32 rmcall_gr_object; /* TURING_COMPUTE_A handle (0=none) */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ @@ -1586,6 +1632,11 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_channel = 0; boot_stage.rmcall_channel_variant = 0; boot_stage.rmcall_channel_server_mem = false; + boot_stage.rmcall_engine = 0; + boot_stage.rmcall_engine_valid = false; + boot_stage.rmcall_engine_change_logged = false; + boot_stage.rmcall_gr_ce_refused_logged = false; + boot_stage.rmcall_gr_object = 0; boot_stage.rmcall_token = 0; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; @@ -5413,6 +5464,22 @@ struct gsp_phys_rmcall_channel_params { __le32 tpc_config_id; }; +/* + * NV_GR_ALLOCATION_PARAMETERS for the H1 TURING_COMPUTE_A engine object + * (common/sdk/nvidia/inc/nvos.h:2322-2328): version must be 2, flags has no + * defined bits, size is sizeof (16) and caps is an output. The server's + * class-constructor path for 0xc5c0 consumes exactly this 16-byte shape + * (the guest driver's own wire request carries paramsSize=16 for the class). + * If a live server rejects {2,0,0x10,0}, the bisection order recorded in the + * H1 notes is: {2,0,0x10,0} -> {1,0,0x10,0} -> {2,0,0x0,0}. + */ +struct gsp_phys_rmcall_gr_alloc_params { + __le32 version; + __le32 flags; + __le32 size; + __le32 caps; +}; + /* Bounded little-endian read of a reply payload word: a read past the * received payload is never performed (off+4 <= payload_len). */ static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec, @@ -6024,13 +6091,87 @@ gsp_phys_rmcall_channel_variants[] = { GSP_PHYS_RMCALL_HANDLE_CHANNEL }, }; +/* + * H1 engine snapshot. gsp_phys_rmcall_engine is a load-time module + * parameter (0=COPY0, 1=GR0); the ladder snapshots it when stage 3 starts + * and every later step uses the snapshot, so a runtime sysfs change can + * never mix a GR0 TSG/channel with a COPY0 BIND or the CE method block. A + * later change is logged once and ignored. With the default COPY0 value the + * snapshot emits no log and every request is byte-identical to before. + */ +static void gsp_phys_rmcall_engine_snapshot(void) +{ + int v; + + if (boot_stage.rmcall_engine_valid) + return; + v = gsp_phys_rmcall_engine_value(); + if (v == 1) { + boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_GR0; + } else { + if (v != 0) + pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine=%d is neither 0 (COPY0) nor 1 (GR0); using COPY0\n", + v); + boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_COPY0; + } + boot_stage.rmcall_engine_valid = true; + if (boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0) + pr_info("gsp-phys-boot: rm-call: engine snapshot: gsp_phys_rmcall_engine=1 -> GR0 (TSG/channel engineType=0x1, TURING_COMPUTE_A 0xc5c0, BIND 0x1, GR semaphore release)\n"); +} + +static bool gsp_phys_rmcall_engine_is_gr(void) +{ + return boot_stage.rmcall_engine_valid && + boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0; +} + +static const char *gsp_phys_rmcall_engine_name(void) +{ + return gsp_phys_rmcall_engine_is_gr() ? "GR0" : "COPY0"; +} + +static void gsp_phys_rmcall_engine_check_change(void) +{ + int v = gsp_phys_rmcall_engine_value(); + u32 live = (v == 1) ? GSP_PHYS_RMCALL_ENGINE_GR0 : + GSP_PHYS_RMCALL_ENGINE_COPY0; + + if (!boot_stage.rmcall_engine_valid || + live == boot_stage.rmcall_engine || + boot_stage.rmcall_engine_change_logged) + return; + boot_stage.rmcall_engine_change_logged = true; + pr_warn("gsp-phys-boot: rm-call: gsp_phys_rmcall_engine changed to %d after the stage-3 snapshot; the ladder keeps engine=%s for the rest of this staging (mid-session changes are ignored)\n", + v, gsp_phys_rmcall_engine_name()); +} + +/* + * H1 CE-refusal gate: true when the current staging's ladder channel speaks + * GR, i.e. when a CE method block must never be submitted on it. Logs once + * per staging (the callers are the lock-free readiness gates, so only the + * first attempt is noisy). + */ +static bool gsp_phys_rmcall_gr_ce_refuse(const char *what) +{ + if (!gsp_phys_rmcall_engine_is_gr()) + return false; + if (!boot_stage.rmcall_gr_ce_refused_logged) { + boot_stage.rmcall_gr_ce_refused_logged = true; + pr_info("gsp-phys-work: %s refused: the ladder channel is a %s engine channel (gsp_phys_rmcall_engine snapshot); CE methods are never submitted on a GR channel (logged once per staging)\n", + what, gsp_phys_rmcall_engine_name()); + } + return true; +} + static void gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, const struct gsp_phys_rmcall_channel_variant *v) { memset(ch, 0, sizeof(*ch)); ch->flags = cpu_to_le32(v->flags); - ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch->engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ? + GSP_PHYS_RMCALL_ENGINE_GR0 : + GSP_PHYS_RMCALL_ENGINE_COPY0); ch->cid = cpu_to_le32(1); /* client hint; server assigns its own */ ch->sub_device_id = cpu_to_le32(0); ch->internal_flags = cpu_to_le32(v->internal_flags); @@ -6078,17 +6219,25 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, * state. Failed variants log one compact line each and no payload dump; an * all-failed walk ends in one summary line with every variant's * rpc_result/status. Kept out of stage 3 so the inline stack stays small. + * + * H1: on a GR0 ladder only the first variant (V5 tsg-vas-pdb-skip-scrubber) + * is tried. A single rpc_result != 0 aborts the walk with no retry and no + * fallback variants: mixing a failed GR channel attempt with the historical + * COPY0-shaped fallbacks could leave an unknown server state, so the whole + * ladder stops at the first failure. */ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) { struct gsp_phys_rmcall_channel_params ch; u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; + u32 attempts = gsp_phys_rmcall_engine_is_gr() ? 1u : + (u32)ARRAY_SIZE(gsp_phys_rmcall_channel_variants); u32 i; int ret; ret = -EIO; - for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) { + for (i = 0; i < attempts; i++) { const struct gsp_phys_rmcall_channel_variant *v = &gsp_phys_rmcall_channel_variants[i]; u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : @@ -6118,6 +6267,14 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) } } + if (attempts == 1) { + pr_err("gsp-phys-boot: rm-call: stage 3: the single GR0 channel attempt (V%u %s) failed (ret=%d rpc_result=0x%x status=0x%x); the GR ladder aborts with no fallback variants and no retry\n", + gsp_phys_rmcall_channel_variants[0].id, + gsp_phys_rmcall_channel_variants[0].name, ret, + v_rpc[0], v_status[0]); + return ret; + } + { char summary[256]; u32 pos = 0; @@ -6136,6 +6293,48 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) } /* + * H1 stage-3 step (GR0 snapshot only): allocate the TURING_COMPUTE_A + * (0xc5c0) engine object the GR method block's SET_OBJECT binds to. The + * client chooses the local handle 0xcaf0002c (free in the ladder's resource + * range; the server validates it with clientAssignResourceHandle) and the + * parent is the ladder channel: the GR class constructor routes to the + * engine from the parent channel handle (kgrobjConstruct_IMPL: + * kgrmgrCtrlSetChannelHandle(pParams->hParent)). The params are + * NV_GR_ALLOCATION_PARAMETERS {version=2, flags=0, size=0x10, caps=0}; if + * the server rejects them the alloc helper dumps the failed reply and the + * ladder aborts with no retry (the H1 notes carry the bisection order). + */ +static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0) +{ + struct gsp_phys_rmcall_gr_alloc_params gr; + u32 status = 0; + int ret; + + gr.version = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_VERSION); + gr.flags = cpu_to_le32(0); + gr.size = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_SIZE); + gr.caps = cpu_to_le32(0); + + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_HANDLE_GR_OBJ, + GSP_PHYS_RMCALL_CLASS_COMPUTE, + &gr, sizeof(gr), false, NULL, + "TURING_COMPUTE_A", true, NULL, &status); + pr_info("gsp-phys-boot: rm-call: TURING_COMPUTE_A alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n", + GSP_PHYS_RMCALL_CLASS_COMPUTE, status, + GSP_PHYS_RMCALL_HANDLE_GR_OBJ, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_GR_PARAM_VERSION, 0u, + GSP_PHYS_RMCALL_GR_PARAM_SIZE, 0u, + ret ? " (rejected)" : ""); + if (ret) + return ret; + + boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_GR_OBJ; + return 0; +} + +/* * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE @@ -6384,6 +6583,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) int ret; /* + * H1: freeze the ladder engine for the rest of this staging before the + * first engine-dependent request (TSG/channel alloc params below). + */ + gsp_phys_rmcall_engine_snapshot(); + + /* * The TSG below carries this externally-owned FERMI_VASPACE_A, whose * root page directory is committed from the zeroed VASPDB chunk before * any channel alloc (the instance-block setup programs that base and @@ -6439,7 +6644,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) memset(&tsg, 0, sizeof(tsg)); tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); - tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + tsg.engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ? + GSP_PHYS_RMCALL_ENGINE_GR0 : + GSP_PHYS_RMCALL_ENGINE_COPY0); /* * The TSG's hParent must be the DEVICE, not the subdevice: the server * resolves it with deviceGetByHandle (kernel_channel_group_api.c:150) @@ -6474,11 +6681,28 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) if (ret) return ret; - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", + /* + * H1: bind the compute engine object to the just-allocated GR channel + * before stage 3 returns. A rejection aborts the ladder (the GR + * method block's SET_OBJECT would fail too) and is never retried. On a + * COPY0 ladder this step is skipped entirely: no RPC, no log. + */ + if (gsp_phys_rmcall_engine_is_gr()) { + ret = gsp_phys_rmcall_alloc_gr_object(bar0); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n", + ret); + return ret; + } + } + + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", - boot_stage.rmcall_notifier_base); + boot_stage.rmcall_notifier_base, + gsp_phys_rmcall_engine_name(), + gsp_phys_rmcall_engine_is_gr() ? " hGr=0xcaf0002c" : ""); return 0; } @@ -6538,6 +6762,45 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, return (u32)(p - pb); } +/* H1 GR method header: the same immediate-pair shape as the CE helper but on + * the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1). */ +static u32 *gsp_phys_rmcall_push_gr(u32 *p, u32 method, u32 data) +{ + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) | + (method >> 2); + *p++ = data; + return p; +} + +/* + * H1 GR method block: five immediate pairs (10 dwords) that bind the compute + * class to the channel's compute subchannel and release one word into the + * semaphore GPU VA. This is not a copy block: no src/dst VAs, offsets, + * pitches or LAUNCH_DMA. The semaphore VA is the same channel-PDB VA the CE + * ladder armed, so the release lands in the module semaphore page. Used by + * stage 4's GR verify and by the debugfs GR re-run trigger; only called when + * the stage-3 snapshot selected GR0. + */ +static u32 gsp_phys_rmcall_build_gr_methods(u32 *pb, u64 sem_va, + u32 sem_payload) +{ + u32 *p = pb; + + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT, + GSP_PHYS_RMCALL_CLASS_COMPUTE); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + return (u32)(p - pb); +} + /* * Result of one ladder CE submit, valid only when the submit helper returned 0. * Kept for the caller's log line and the GP_GET/GPPut context. @@ -6572,10 +6835,16 @@ struct gsp_phys_rmcall_ce_submit { * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is * still cleared even when the caller polls a guest page: the clear only * guards the module-sem poll path and is a harmless non-guest write. + * + * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release + * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and + * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call + * site passes false, so the default COPY0 ladder emits the same bytes as + * before. */ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, + u32 sem_payload, bool gr, struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = @@ -6623,8 +6892,13 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); wmb(); - words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, - bytes, sem_payload); + if (gr) + words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va, + sem_payload); + else + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, + dst_va, bytes, + sem_payload); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -6704,6 +6978,80 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, + * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller + * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll; + * the trigger takes the same lock), and this helper polls the module + * semaphore while the lock is held, exactly like the CE stage-4 tail. The + * submission goes through the shared guarded helper with gr=true, so the + * server-memory/BAR0/token/GP_PUT/GPFIFO-GET guards apply unchanged. There is + * no src/dst copy to compare: the GR method block only releases the + * semaphore. On timeout the existing status-queue drain runs and the helper + * returns -ETIMEDOUT; it never retries. + */ +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *userd_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_rmcall_ce_submit sub; + u32 *userd; + u32 poll, gpget, gpput; + unsigned long deadline; + int ret; + + ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, true, + &sub); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GR submission refused before the doorbell (%d)\n", + ret); + return ret; + } + userd = gsp_phys_chunk_page_va(userd_c, 0); + + pr_info("gsp-phys-boot: rm-call: GR submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSem=0x%llx vaSrc=0x%llx vaDst=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_DST, sub.words, sub.slot, sub.gpget, + sub.gpput, sub.e0, sub.e1, boot_stage.rmcall_token, + sub.runlist_id, sub.ch_id, sub.doorbell, + GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma, + GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + gpget = get_unaligned_le32((u8 *)userd + + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpput = get_unaligned_le32((u8 *)userd + + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + pr_err("gsp-phys-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block\n", + GSP_PHYS_RMCALL_TIMEOUT_MS, poll, + GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput, + gsp_phys_rmcall_engine_name()); + gsp_phys_status_drain(NULL); + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + pr_info("gsp-phys-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n", + poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, + get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + sub.words, boot_stage.rmcall_token); + return 0; +} + +/* * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock * (the same lock discipline as the P5 shadow API, so a guest copy can never @@ -6728,11 +7076,15 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, u32 poll; int ret; + /* H1: never submit CE methods on a GR0 ladder channel. */ + if (gsp_phys_rmcall_gr_ce_refuse("guest CE copy (trigger 8 selftest)")) + return -EOPNOTSUPP; + mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, bytes, GSP_PHYS_RMCALL_SEM_PAYLOAD, - &sub); + false, &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) return ret; @@ -6773,6 +7125,10 @@ static u64 gsp_phys_work_refused; * P5/P6b forwarding APIs additionally require gsp_phys_work_forward (see * gsp_phys_backend_work_ready below); triggers 6/7 must run with forwarding * off, so their guest-copy variant uses this gate directly. + * + * H1: a GR0 ladder channel has no CE engine bound, so every CE forwarding + * path refuses here. The refusal is logged once per staging by + * gsp_phys_rmcall_gr_ce_refuse(); the software release path keeps working. */ static bool gsp_phys_backend_ladder_ready(void) { @@ -6781,6 +7137,8 @@ static bool gsp_phys_backend_ladder_ready(void) if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) return false; + if (gsp_phys_rmcall_gr_ce_refuse("CE forwarding")) + return false; if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) return false; @@ -6868,7 +7226,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, bytes, GSP_PHYS_RMCALL_SEM_PAYLOAD, - &sub); + false, &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) { gsp_phys_work_refused++; @@ -7331,7 +7689,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, bytes, relay ? sem_payload : GSP_PHYS_RMCALL_SEM_PAYLOAD, - NULL); + false, NULL); mutex_unlock(&gsp_phys_rm_lock); if (ret) { if (relay) @@ -7526,7 +7884,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; struct gsp_phys_rmcall_ce_submit sub; - u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; /* NVA06F engineType */ + u32 bind_params = gsp_phys_rmcall_engine_is_gr() ? + GSP_PHYS_RMCALL_ENGINE_GR0 : /* NVA06F engineType */ + GSP_PHYS_RMCALL_ENGINE_COPY0; u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; /* NVC36F index */ u8 sched_params[2] = { 1, 0 }; /* NVA06F bEnable=1, bSkipSubmit=0 */ resource_size_t bar0_len; @@ -7536,6 +7896,9 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) unsigned long deadline; int ret; + /* H1: note (once) a mid-session engine-parameter change, keep snapshot. */ + gsp_phys_rmcall_engine_check_change(); + /* * The winning variant requested server-allocated memdescs: there is no * client GPFIFO, USERD or pushbuffer to fill, so no work item can be @@ -7663,6 +8026,21 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) } boot_stage.rmcall_token = token; + /* + * H1: a GR0 ladder releases a semaphore on the compute engine instead + * of running a CE copy. The dedicated helper performs the same guarded + * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM + * OK / REAL GR SEM FAILED and never compares src/dst; the CE tail + * below is untouched and byte-identical for the default COPY0 ladder. + */ + if (gsp_phys_rmcall_engine_is_gr()) { + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + if (ret) + pr_err("gsp-phys-boot: rm-call: stage 4 GR path failed (%d); no retry\n", + ret); + return ret; + } + userd = gsp_phys_chunk_page_va(userd_c, 0); /* @@ -7677,7 +8055,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, - &sub); + false, &sub); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", ret); @@ -7736,6 +8114,39 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) } /* + * H1 debugfs trigger 9: re-run one GR semaphore release on the already-built + * GR ladder without a full stage cycle. Valid only when the stage-3 snapshot + * selected GR0, the GR engine object was allocated and stage 4 already + * produced a work-submit token (a failed GR poll still leaves the token set, + * so the trigger is useful for bisection). Takes gsp_phys_rm_lock like + * gsp_phys_rmcall_run() and performs no RPC: only the method-block submit, + * GPFIFO/USERD publish, doorbell and bounded semaphore poll. Never retries; + * every gate miss is a logged no-op. + */ +static int gsp_phys_rmcall_gr_sem_rerun(void __iomem *bar0) +{ + int ret; + + mutex_lock(&gsp_phys_rm_lock); + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted || + !gsp_phys_rmcall_engine_is_gr() || + !boot_stage.rmcall_channel || !boot_stage.rmcall_gr_object || + !boot_stage.rmcall_token || boot_stage.rmcall_stage < 3) { + mutex_unlock(&gsp_phys_rm_lock); + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 no-op: requires the GR ladder through stage 3 (gr=%u claimed=%u init_done=%u unload=%u hChannel=0x%x hGr=0x%x token=0x%x stage=%u)\n", + gsp_phys_rmcall_engine_is_gr(), gsp_phys_gpu_claimed(), + boot_stage.rm_init_done, boot_stage.rm_unload_submitted, + boot_stage.rmcall_channel, boot_stage.rmcall_gr_object, + boot_stage.rmcall_token, boot_stage.rmcall_stage); + return -EINVAL; + } + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + return ret; +} + +/* * Stage 5: adopt the server's internal client/device/subdevice handles from * this session's decoded GspStaticConfigInfo instead of allocating our own * NV01_ROOT/DEVICE/SUBDEVICE. If the rm-probe cycle has not produced them @@ -8236,7 +8647,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); seq_printf(m, - "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x notifier=%s base=0x%llx\n", + "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u engine=%s hGr=0x%x token=0x%x notifier=%s base=0x%llx\n", boot_stage.rmcall_ran, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret, boot_stage.rmcall_rpcs, boot_stage.rmcall_client, @@ -8244,6 +8655,8 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, + gsp_phys_rmcall_engine_name(), + boot_stage.rmcall_gr_object, boot_stage.rmcall_token, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base); @@ -8867,11 +9280,15 @@ static const struct file_operations gsp_phys_rm_probe_fops = { * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the * fresh-line PTE self-test (no GPA needed); all three require - * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every - * other value or state (parameter off, GPU unclaimed, stage 3d not run, - * init-done not observed, unload already submitted, BAR0 unmapped) is a - * logged no-op. The whole ladder is serialised with the unload/probe triggers - * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. + * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Writing 9 + * re-runs one GR semaphore release on an already-built GR ladder + * (gsp_phys_rmcall_engine=1 snapshot through stage 3 with the GR object and + * work-submit token) without a full stage cycle; it needs no guest-memory + * parameter. Every other value or state (parameter off, GPU unclaimed, + * stage 3d not run, init-done not observed, unload already submitted, BAR0 + * unmapped) is a logged no-op. The whole ladder is serialised with the + * unload/probe triggers by gsp_phys_rm_lock; at most one run per write; WPR2 + * is never touched. */ static ssize_t gsp_phys_rm_call_write(struct file *file, const char __user *ubuf, size_t len, @@ -8890,8 +9307,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; - if (value < 1 || value > 8) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n"); + if (value < 1 || value > 9) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, 8 for the guest PTE self-test, or 9 to re-run one GR semaphore release on a GR ladder; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { @@ -8932,6 +9349,21 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, return len; } + /* + * H1 trigger 9: re-run one GR semaphore release without a full ladder + * cycle. Not a stage and not cumulative: it issues no RPC, only the + * method-block submit, GPFIFO/USERD publish, doorbell and bounded + * module-sem poll on the already-built GR channel. All gates live in + * the helper (under gsp_phys_rm_lock); 6-8 below keep their existing + * gsp_phys_guest_mem gating. + */ + if (value == 9) { + ret = gsp_phys_rmcall_gr_sem_rerun(bar0); + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger 9 returned %d (GR semaphore re-run; WPR2 untouched)\n", + ret); + return len; + } + if (value >= 6) { /* * P6a guest-memory directions and self-test: strictly opt-in, @@ -9505,6 +9937,17 @@ static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32, "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes"); static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20, "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes"); +static_assert(sizeof(struct gsp_phys_rmcall_gr_alloc_params) == + GSP_PHYS_RMCALL_GR_PARAM_SIZE, + "NV_GR_ALLOCATION_PARAMETERS must be 16 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, version) == 0, + "NV_GR_ALLOCATION_PARAMETERS.version must be at offset 0"); +static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, flags) == 4, + "NV_GR_ALLOCATION_PARAMETERS.flags must be at offset 4"); +static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, size) == 8, + "NV_GR_ALLOCATION_PARAMETERS.size must be at offset 8"); +static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, caps) == 12, + "NV_GR_ALLOCATION_PARAMETERS.caps must be at offset 12"); static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368, "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8, diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 30e30db..52d9c43 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -407,7 +407,15 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); * verified with a direct-map memcmp. Both directions are gated on a live * READY claim, ladder stage >= 3 and the pinned GPA; every refusal is a * logged no-op and no guest byte is touched unless G2H already proved - * the translation. + * the translation. With gsp_phys_rmcall_engine=1 the stage-3 snapshot + * selects the GR0 engine for the same ladder: the TSG/channel carry + * engineType 1, the channel walk is a single no-retry V5 attempt, a + * TURING_COMPUTE_A (0xc5c0) engine object is allocated on the channel, + * BIND uses engineType 1 and stage 4 releases a GR semaphore instead of + * copying (logged as REAL GR SEM OK / REAL GR SEM FAILED); all CE + * forwarding paths refuse once per staging. Writing 9 re-runs one GR + * semaphore release on an already-built GR ladder without a full stage + * cycle (no RPC; requires the GR object and work-submit token). * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM * GPA ("0x<hex>", 4 KB page-aligned) that the rm-call guest directions * may touch. One GPA per boot (a second different GPA is refused: diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index cfa3d4e..227246b 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -316,6 +316,32 @@ bool gsp_phys_fault_inject_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_fault_inject_enabled); /* + * H1 rm-call ladder engine selector (default 0). 0 keeps the existing + * COPY0/CE ladder byte-identical: TSG/channel engineType COPY0, the V5 + * variant walk, BIND engineType 0x9 and the CE copy method block. 1 selects + * GR0 (NV2080_ENGINE_TYPE_GRAPHICS): the stage-3 snapshot builds the + * TSG/channel with engineType 1, makes one (no-retry) V5 channel attempt, + * allocates the TURING_COMPUTE_A (0xc5c0) engine object on the ladder + * channel, BIND uses engineType 1 and stage 4 pushes a GR + * SET_REPORT_SEMAPHORE release instead of a CE copy. The value is snapshotted + * when stage 3 starts; a later sysfs change is logged once and ignored, so a + * session can never mix a GR0 channel with a COPY0 BIND. Every CE forwarding + * path (P5/P6a selftest/P6b/P6c) refuses cleanly on a GR ladder. One-line + * rollback: gsp_phys_rmcall_engine=0. + */ +static int gsp_phys_rmcall_engine_param; +module_param_named(gsp_phys_rmcall_engine, gsp_phys_rmcall_engine_param, int, + 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_engine, + "RM-client ladder engine: 0 = COPY0 (default, existing CE copy ladder), 1 = GR0 (TURING_COMPUTE_A engine object and a GR semaphore release)"); + +int gsp_phys_rmcall_engine_value(void) +{ + return gsp_phys_rmcall_engine_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_engine_value); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 214c62d..f34d7a2 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -166,6 +166,16 @@ bool gsp_phys_fault_relay_enabled(void); bool gsp_phys_fault_inject_enabled(void); /* + * RM-client ladder engine selector (gsp_phys_rmcall_engine; default 0): + * returns the raw module-parameter value, where 0 = COPY0 (the existing CE + * ladder) and 1 = GR0 (the TURING_COMPUTE_A engine object plus a GR + * semaphore release). The booter snapshots this when the rm-call ladder's + * stage 3 starts and ignores later changes, so this accessor is for the + * snapshot and diagnostics only. + */ +int gsp_phys_rmcall_engine_value(void); + +/* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): * its VFIO type-1 container keeps the guest DMA mappings privately, so |