diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 195 |
1 files changed, 167 insertions, 28 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6cb7034..6d72a62 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -36034,6 +36034,14 @@ struct svc_fwd_req { * path rewinds the pack so the next client window re-emits from word 0. */ bool gr_synthetic; + /* + * Saved-window release carrier (sema_gl_mme_pack_release): a synthetic + * request that carries no capture of its own; the work item fills the + * submit request from the booter's saved record and submits it with + * the held CALL runs released. gr_synthetic is also set so the + * outcome/software-release paths never touch it. + */ + bool gr_release; bool gr_bridge_done; bool gr_page_overflow; /* @@ -38674,6 +38682,69 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); +#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ +#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ + +/* + * Saved-window release carrier work item (sema_gl_mme_pack_release). + * + * The booter saved the held-call window + its capture at the MME-CALL-HOLD + * point; gsp_phys_gr_mme_release_fill() copies that snapshot into the submit + * request and the normal backend fill/build path replays it with the CALL + * runs released. No guest release or QMD word is authored (gr_synthetic + * suppresses the stall-completion path); the booter's release_complete / + * release_failed own the receipts and the one-fire budget. + */ +static void svc_fwd_gr_release_work(void) +{ + struct gsp_phys_gr_submit gs; + struct gsp_phys_backend_copy_result cp = { 0 }; + struct task_struct *task; + int hw_rc; + + memset(&gs, 0, sizeof(gs)); + if (!gsp_phys_gr_mme_release_fill(&gs)) { + /* + * The record was cancelled after the claim (context wipe, + * re-register, gen rewind): never submit a stale snapshot. + */ + gsp_phys_gr_mme_release_cancel("save-lost"); + svc_fwd_req.hw_rc = -EINVAL; + svc_fwd_req.fault = cp.fault; + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + task = READ_ONCE(svc.task); + if (task) + wake_up_process(task); + return; + } + /* + * The launch receipts key on the synthetic carrier id; the saved push + * is only the record's provenance. + */ + gs.push_va = SVC_GR_CARRIER_PUSH_VA; + hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + if (hw_rc) { + svc.fwd_gr_fallback++; + if (hw_rc == -ETIMEDOUT) { + svc.fwd_gr_timeout++; + if (cp.fault.valid) + svc.fwd_gr_fault++; + } + gsp_phys_gr_mme_release_failed(hw_rc); + } else { + svc.fwd_gr_ok++; + gsp_phys_gr_mme_release_complete(); + } + svc_fwd_req.fault = cp.fault; + svc_fwd_req.hw_rc = hw_rc; + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + task = READ_ONCE(svc.task); + if (task) + wake_up_process(task); +} + /* * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The * request already resolved every launch page's guest VA->PA on the service @@ -38689,6 +38760,15 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); static void svc_fwd_gr_work(void) { /* + * Saved-window release carrier: the slot carries no capture of its + * own; the booter's saved snapshot is the launch. Takes its own + * completion shape (no guest release/QMD word). + */ + if (svc_fwd_req.gr_release) { + svc_fwd_gr_release_work(); + return; + } + /* * Static: the ordered gsp-phys-work queue runs one item at a time and * the GR page array is ~0.4 KB; keeping it off the stack keeps the * frame under the kernel's warning bound (same reason as the ops array @@ -41951,6 +42031,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_synthetic = false; + svc_fwd_req.gr_release = false; svc_fwd_req.gr_bridge_done = false; /* STALLFREE: fresh request - re-arm the software-completion latch and * stamp the fill time for the GR-STALL waited_ms accounting. */ @@ -43488,8 +43569,6 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * or written. The booter re-validates the live cursor in carry_take(); on any * mismatch it cancels and this function is a no-op. */ -#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ -#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ static bool svc_gr_cand_unoffered_for(u32 h_object) { @@ -43505,10 +43584,10 @@ static bool svc_gr_cand_unoffered_for(u32 h_object) } static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch) + const struct svc_channel *ch, bool release) { u32 h_object = ch->h_object; - u32 gen = 0, off = 0, maxa = 0; + u32 gen = 0, off = 0, maxa = 0, held = 0, win = 0; if (!gsp_phys_gr_forward_enabled() || !gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) @@ -43516,11 +43595,19 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, /* One launch owner per channel: never compete with a real candidate. */ if (svc_gr_cand_unoffered_for(h_object) || svc_fwd_hold.active) return false; - if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) - return false; - pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", - h_object, gen, off, maxa, - (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + if (release) { + if (!gsp_phys_gr_mme_release_take(h_object, &held, &win)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE mode=release held=%u win=%u h=0x%x push=0x%llx\n", + held, win, h_object, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + } else { + if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, off, maxa, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { @@ -43530,7 +43617,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, * without rewinding the pack (the next client window continues * from the current cursor). */ - gsp_phys_gr_mme_carry_cancel("request-busy"); + if (release) + gsp_phys_gr_mme_release_cancel("request-busy"); + else + gsp_phys_gr_mme_carry_cancel("request-busy"); return false; } memset(&svc_fwd_req, 0, sizeof(svc_fwd_req)); @@ -43542,21 +43632,33 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_h_object = h_object; svc_fwd_req.gr_push_va = SVC_GR_CARRIER_PUSH_VA; svc_fwd_req.gr_fill_jiffies = jiffies; - /* - * The backend's 3D branch is entered by a non-empty stream; the - * two-word stub here is only the request shape (the booter replaces it - * with its own WAIT_FOR_IDLE stub and forces LOAD-ONLY), so this body - * is never executed. No pages, inline records, data records, windows, - * program binds, sem fields or release fields are filled - the memset - * above zeroed them and qmd[] is the all-zero 64-word source the - * submit requires. - */ - svc_fwd_req.gr_stream_3d = true; - svc_fwd_req.gr_stream_off = 0; - svc_fwd_req.gr_stream_n = 2; - svc_fwd_req.gr_stream_rel = 0; - svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; - svc_fwd_req.gr_raw[1] = 0; + if (release) { + /* + * Saved-window release: the slot carries no capture of its + * own. The work item fills the submit request from the + * booter's saved record (gr_release) - window, pages, inline + * and data all come from that snapshot - and submits it with + * the held CALL runs released. + */ + svc_fwd_req.gr_release = true; + } else { + /* + * The backend's 3D branch is entered by a non-empty stream; + * the two-word stub here is only the request shape (the + * booter replaces it with its own WAIT_FOR_IDLE stub and + * forces LOAD-ONLY), so this body is never executed. No + * pages, inline records, data records, windows, program + * binds, sem fields or release fields are filled - the + * memset above zeroed them and qmd[] is the all-zero 64-word + * source the submit requires. + */ + svc_fwd_req.gr_stream_3d = true; + svc_fwd_req.gr_stream_off = 0; + svc_fwd_req.gr_stream_n = 2; + svc_fwd_req.gr_stream_rel = 0; + svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; + svc_fwd_req.gr_raw[1] = 0; + } svc_fwd_requests++; if (!svc_fwd_wq) { svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); @@ -43564,7 +43666,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); - gsp_phys_gr_mme_carry_cancel("queue-alloc"); + if (release) + gsp_phys_gr_mme_release_cancel("queue-alloc"); + else + gsp_phys_gr_mme_carry_cancel("queue-alloc"); return false; } } @@ -43572,7 +43677,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); - gsp_phys_gr_mme_carry_cancel("queue-work"); + if (release) + gsp_phys_gr_mme_release_cancel("queue-work"); + else + gsp_phys_gr_mme_carry_cancel("queue-work"); return false; } mutex_unlock(&svc_fwd_lock); @@ -43649,6 +43757,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, */ svc_fwd_req.gr = false; svc_fwd_req.gr_synthetic = false; + svc_fwd_req.gr_release = false; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); @@ -49921,6 +50030,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * Saved-window release carrier (sema_gl_mme_pack_release). After the + * channel loop (a client window decoded in this pass wins and cancels + * the pending release), fire the saved held-call window once the pack + * completed and the settle grace expired. Same gates as the + * continuation carrier, except the FE-busy defer (this launch carries + * a real window, not a LOAD-ONLY carrier). + */ + { + u32 rel_h = 0; + + if (gsp_phys_gr_mme_release_peek(&rel_h)) { + int ci; + + for (ci = 0; ci < n; ci++) { + struct svc_channel *c = &local[ci]; + + if (c->h_object != rel_h || c->engine_type) + continue; + if (svc_gr_cand_unoffered_for(rel_h)) + break; + if (svc_fwd_hold.active || + test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + break; + (void)svc_gr_carrier_request(vgpu, c, true); + break; + } + } + } + + /* * c727 host-initiated MME pack continuation. After the channel loop (so * a client window decoded in this pass can complete the pack first), * emit the next golden pack part for the armed beef channel. The gates @@ -49949,7 +50088,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2)) break; - (void)svc_gr_carrier_request(vgpu, c); + (void)svc_gr_carrier_request(vgpu, c, false); break; } } |