diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 180 |
1 files changed, 173 insertions, 7 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e97c2ff..25a3cdb 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -41,10 +41,13 @@ #include <linux/mutex.h> #include <linux/sort.h> #include <linux/bitmap.h> +#include <linux/workqueue.h> #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" +#include "gsp_phys_booter.h" +#include "gsp_phys_gpu.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 @@ -5016,6 +5019,145 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) test_bit(idx, ch->slot_done_epoch); } +/* ------------------------------------------------------------------ */ +/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1) */ +/* ------------------------------------------------------------------ */ + +/* + * The pump never waits on the backend: for an eligible CE push it only marks + * one shadow request in flight and queues a dedicated ordered work item. A + * request that arrives while one is in flight (or before the backend/ladder + * is ready) is dropped and counted. The work item runs the ladder's + * module-owned copy and observes the real semaphore; the software release + * path is unchanged and stays authoritative. No guest VA or guest PA ever + * reaches the engine in this wave. + */ +#define SVC_FWD_INFLIGHT 0 +#define SVC_FWD_LOG_MASK 63 + +static struct workqueue_struct *svc_fwd_wq; +static unsigned long svc_fwd_state; +static u64 svc_fwd_requests; +static u64 svc_fwd_dropped; +/* + * Pause gate shared by the pump's request path and the backend release path: + * quiesce takes this lock, sets paused and waits for the in-flight work item; + * a request either queues before quiesce (and is waited for) or observes + * paused and drops. Without it a request could be queued after quiesce + * returned and run while the ladder buffers are being freed. + */ +static DEFINE_MUTEX(svc_fwd_lock); +static bool svc_fwd_paused; + +static void svc_fwd_work_fn(struct work_struct *work) +{ + u64 src_va = 0, dst_va = 0, sem_va = 0; + u32 bytes = 0; + unsigned long start = jiffies; + int ret; + + /* + * Re-resolve the ladder operands on the work item: the backend may + * have gone away between the pump's gate and this run. A false return + * means no module buffer is usable; drop without touching anything. + */ + if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va, + &bytes)) { + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n", + svc_fwd_dropped); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + return; + } + + ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes, + GSP_PHYS_BACKEND_WORK_POLL_MS); + if (!ret) + pr_info("gsp-phys-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n", + bytes, jiffies_to_msecs(jiffies - start), + svc_fwd_requests, svc_fwd_dropped); + /* Timeout/refusal counters and their rate-limited logs are in the + * submit; the software release has already run either way. */ + + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); +} + +static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); + +/* Called only from the service thread (the pump). */ +static void svc_fwd_request(const struct svc_channel *ch) +{ + if (!ch->engine_type) + return; + if (!gsp_phys_work_forward_enabled()) + return; + if (!gsp_phys_backend_work_ready()) { + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (backend not ready) dropped=%llu\n", + svc_fwd_dropped); + return; + } + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused) { + mutex_unlock(&svc_fwd_lock); + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (forwarding paused for release) dropped=%llu\n", + svc_fwd_dropped); + return; + } + if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + svc_fwd_dropped++; + if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) + pr_info("gsp-phys-work: shadow request dropped (one in flight) dropped=%llu\n", + svc_fwd_dropped); + return; + } + svc_fwd_requests++; + if (!svc_fwd_wq) { + /* + * Lazy, on the service thread only: no workqueue exists at all + * while the parameter is off. + */ + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + return; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + } + mutex_unlock(&svc_fwd_lock); +} + +/* + * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new + * requests under svc_fwd_lock and waits for the in-flight one; the work item + * never takes that lock, so the wait cannot deadlock and is bounded by the + * submit's 200 ms poll cap. + */ +void gsp_sema_service_work_quiesce(void) +{ + mutex_lock(&svc_fwd_lock); + svc_fwd_paused = true; + cancel_work_sync(&svc_fwd_work); + mutex_unlock(&svc_fwd_lock); +} + +void gsp_sema_service_work_resume(void) +{ + mutex_lock(&svc_fwd_lock); + svc_fwd_paused = false; + mutex_unlock(&svc_fwd_lock); +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ @@ -5262,6 +5404,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); /* + * P5 wave 1: shadow this + * CE push on the ladder + * channel. The software + * release above is already + * applied and stays + * authoritative; the work + * item only observes the + * real semaphore. + */ + svc_fwd_request(ch); + /* * A CE push that should carry this * channel's next tracking release but * decoded none read from a @@ -6397,6 +6550,11 @@ int gsp_sema_service_start(void) init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); spin_lock_init(&svc.vas_lock); + /* + * The P5 shadow workqueue is created lazily by svc_fwd_request() only + * while gsp_phys_work_forward is enabled; the software path needs no + * workqueue at all. + */ svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); @@ -6411,14 +6569,22 @@ int gsp_sema_service_start(void) * Stop the service thread without kthread_stop(): the thread can also end on * its own if it ever faults, and joining a task that is already gone faults in * the caller (which wedges the whole module). The completion tells us the - * thread function has returned, whatever ended it. + * thread function has returned, whatever ended it. The shadow work item is + * cancelled after the thread stops (so no new request can be queued) and + * before the workqueue is destroyed; the submit's own poll is bounded to + * 200 ms, so this wait is bounded. */ void gsp_sema_service_stop(void) { - if (!svc.task) - return; - svc.stop = true; - if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) - pr_err("sema-svc: thread did not stop within 5s\n"); - svc.task = NULL; + if (svc.task) { + svc.stop = true; + if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) + pr_err("sema-svc: thread did not stop within 5s\n"); + svc.task = NULL; + } + cancel_work_sync(&svc_fwd_work); + if (svc_fwd_wq) { + destroy_workqueue(svc_fwd_wq); + svc_fwd_wq = NULL; + } } |