diff options
Diffstat (limited to 'kernel/gsp_cuda_bridge.c')
| -rw-r--r-- | kernel/gsp_cuda_bridge.c | 92 |
1 files changed, 88 insertions, 4 deletions
diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c index 2fac2b2..7adc8e3 100644 --- a/kernel/gsp_cuda_bridge.c +++ b/kernel/gsp_cuda_bridge.c @@ -141,6 +141,9 @@ MODULE_PARM_DESC(cuda_bridge, /* ------------------------------------------------------------------ */ struct cubb_req { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * queue is FIFO, so only the owner's + * service pass may marshal this job */ u32 h_object; u64 qmd_dev_va; u64 push_va; @@ -178,6 +181,9 @@ static u64 cubb_dedup_hits; struct cubb_pend { bool valid; + struct nv_vgpu_device *vgpu; /* owning mdev: the completion write-back + * must run through this guest's + * address space, never another's */ u64 seq; u32 h_object; u32 opcode; @@ -576,12 +582,15 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, if (!gsp_cuda_bridge_enabled()) return; - (void)vgpu; for (i = 0; i < 64; i++) { hash ^= qmd[i]; hash *= 1099511628211ULL; } + /* The owner is part of the identity: two guests' identical launches + * must never dedup against each other. */ + hash ^= (u64)(unsigned long)vgpu; + hash *= 1099511628211ULL; hash ^= (u64)h_object << 32; hash *= 1099511628211ULL; hash ^= qmd_dev_va; @@ -612,6 +621,7 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, struct cubb_req *r = &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; + r->vgpu = vgpu; r->h_object = h_object; r->qmd_dev_va = qmd_dev_va; r->push_va = push_va; @@ -972,6 +982,7 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu) } job.seq = ++cubb_seq; cubb_pend[slot].valid = true; + cubb_pend[slot].vgpu = r.vgpu; cubb_pend[slot].seq = job.seq; cubb_pend[slot].h_object = r.h_object; cubb_pend[slot].opcode = opcode; @@ -1303,12 +1314,48 @@ static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p, } /* + * Is this mdev still on the driver list with an open VFIO device? Guards the + * bridge's per-request/per-pend owner pointers (dual-guest goal 1.0): a + * removed mdev must never be dereferenced and its queued work can never be + * observed by a live guest. Pointer comparisons only - safe to call with a + * possibly-stale pointer. + */ +static bool cubb_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + +/* * Service-thread hook: marshal at most one queued launch into a daemon-visible * job, stage any returned done records, then drive staged completions. Called * every service pass; a no-op when no launch has been seen. + * + * Dual-guest (goal 1.0): request and pend records carry their owning mdev + * (the bridge falls back here when forwarding was refused under the + * single-inflight slot). Only the owning guest's pass marshals its job (the + * marshal resolves VAs and reads guest memory through that guest) and only + * the owning pass writes a completion back; a head/record whose owner is + * gone is dropped instead of ever being dereferenced. */ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) { + struct nv_vgpu_device *head_owner = NULL; + unsigned long flags; + u32 head_h = 0; + bool have_head = false; struct cubb2_done d; int guard = 0; int i; @@ -1316,7 +1363,29 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) if (!gsp_cuda_bridge_enabled()) return; - (void)cubb_marshal_one(vgpu); + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_head != cubb_req_tail) { + struct cubb_req *head = + &cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; + + head_owner = head->vgpu; + head_h = head->h_object; + have_head = true; + } + spin_unlock_irqrestore(&cubb_req_lock, flags); + if (have_head) { + if (head_owner == vgpu) { + (void)cubb_marshal_one(vgpu); + } else if (!cubb_vgpu_open(head_owner)) { + /* The owner's guest is gone: nobody can observe it. */ + cubb_req_drop_head(); + cubb_dropped++; + pr_info_ratelimited("cuda-bridge: dropped a queued launch whose mdev is gone (h=0x%x)\n", + head_h); + } + /* else: another live guest's head; its own pass marshals it. */ + } + while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { if (d.magic != CUBB2_DONE_MAGIC || d.version != CUBB2_VERSION) { @@ -1330,8 +1399,23 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) mutex_lock(&cubb_lock); for (i = 0; i < CUBB2_PEND_MAX; i++) { - if (cubb_pend[i].valid && cubb_pend[i].have_done && - cubb_finish_try(vgpu, &cubb_pend[i], i)) + struct cubb_pend *p = &cubb_pend[i]; + + if (!p->valid || !p->have_done) + continue; + if (p->vgpu != vgpu) { + /* + * Another guest's staged completion: its own pass + * drives the write-back. A removed owner's record is + * released here so the slot frees. + */ + if (!cubb_vgpu_open(p->vgpu)) { + cubb_pend_release(p, i); + i = -1; /* compaction shifted the entries */ + } + continue; + } + if (cubb_finish_try(vgpu, p, i)) i = -1; /* compaction shifted the entries; rescan */ } mutex_unlock(&cubb_lock); |