From 81c11ca09adf85bd314f7406a8e0d7d637726a80 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 15 Oct 2026 17:53:52 -0400 Subject: vgsp-dual: stage B+C minimal per-mdev separation (G1-G3, G5, G6, G7, G8a) Goal 1.0: two mdevs + two QEMU guests running CUDA simultaneously. Single-mdev behaviour is kept identical: the legacy guest-RAM record still serves the first/only open mdev, the channel table degenerates to the historical key, and the session flush fires at the same last-channel-closed transition. - G6: guest-RAM backing {file, path, ranges} per mdev (legacy record + up to 4 vgpu-bound records); debugfs gsp-phys-guest-ram accepts an mdev UUID as an extra first token and keeps the 4-token form; records released on mdev cleanup and physical release. - G7: gsp_phys_gpa_to_hpa_vgpu(vgpu, gpa, *hpa); NULL keeps the historical first-open resolution. Forwarding operands pass the request's vgpu; the single-guest boot-ladder pin passes NULL. - G1: the sema-service thread services every open mdev each pass. - G2: channel table keyed by (vgpu, h_object); per-guest ranks; counter/scrub/opdef/pump snapshot+writeback matches per vgpu; scrubber records per guest. - G3: per-guest session records drive the flush; the shared learned tables (staged G4) are only flushed when no guest has live channels. - G5: hold/abort keyed by (vgpu, handle); dead-owner guards on the hold resolution paths; single-inflight serialization kept. - G8a: guest-GPA pin refused while more than one mdev is open. Build: make -j8 clean success (vgsp.ko). No module load. See DUAL-REPORT.md for the per-item status and live-validation plan. --- kernel/gsp_phys_booter.c | 115 +++++++-- kernel/gsp_phys_gpu.c | 432 ++++++++++++++++++++++++++----- kernel/gsp_phys_gpu.h | 35 ++- kernel/gsp_rm_emulation.c | 3 +- kernel/gsp_sema_service.c | 644 +++++++++++++++++++++++++++++++++++----------- kernel/gsp_sema_service.h | 20 +- kernel/main.c | 10 + 7 files changed, 1010 insertions(+), 249 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9eb1179..9a4aedd 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -9728,8 +9728,16 @@ static void gsp_phys_rmcall_pdb_build(void) u64 guest_pte; int gpa_ret; - gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, - &guest_hpa); + /* + * The pinned GPA is a single-guest test aid (the pin write is + * refused while more than one mdev is open; see + * gsp_phys_guest_gpa_write): resolve through the first open + * mdev exactly as before the per-mdev records, via the NULL + * selector (a per-mdev pin is not modelled). + */ + gpa_ret = gsp_phys_gpa_to_hpa_vgpu(NULL, + boot_stage.rmcall_guest_gpa, + &guest_hpa); if (!gpa_ret) { guest_pte = gsp_phys_rmcall_pte(guest_hpa); put_unaligned_le64(guest_pte, @@ -19110,7 +19118,11 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -19193,7 +19205,11 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -20404,6 +20420,18 @@ static ssize_t gsp_phys_guest_gpa_write(struct file *file, gpa); return -EINVAL; } + /* + * Dual-guest guard (goal 1.0): the pin is a single-guest test aid. The + * triggers translate it through the first open mdev (see + * gsp_phys_gpa_to_hpa / boot_stage.rmcall_guest_gpa), so with more than + * one open mdev the pin cannot name its owner and is refused instead of + * silently resolving through whichever guest opened first. + */ + if (gsp_phys_open_mdev_count() > 1) { + pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: %d mdevs are open; the guest-GPA pin is a single-guest test aid (translates through the first open mdev). Close all but one mdev, or use the per-mdev guest/GPA paths\n", + gsp_phys_open_mdev_count()); + return -EBUSY; + } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", @@ -20426,18 +20454,26 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* - * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM - * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing + * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is * * " " * * (path without spaces; all three numbers hex with 0x, e.g. - * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the - * file and fixes the path for this module cycle; later writes may add ranges - * (up to four) or replace a range with the same gpa_base. The ranges are - * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev - * session exists. A single "-" clears the configuration and closes the file. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). This historical form + * configures the legacy record and serves the FIRST/ONLY open mdev exactly + * as before the per-mdev records existed. With two live guests each writes + * its own record by naming its mdev UUID first: + * + * " " + * + * Paths and UUIDs contain no spaces, so the token count (4 vs 5) decides. + * The first write for a record opens the file and fixes the path for that + * record; later writes may add ranges (up to four) or replace a range with + * the same gpa_base. The ranges are only consulted by the translators, and + * only while a live mdev session exists. A single "-" clears ALL records + * (legacy and per-mdev) and closes the files. */ static char *gsp_phys_guest_ram_token(char **cursor) { @@ -20452,9 +20488,11 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { - char buf[384]; - char *p, *tok, *path; + char buf[448]; + char *t[5]; + char *path; u64 gpa_base, file_offset, size; + int nt = 0, off; int ret; if (len == 0 || len >= sizeof(buf)) @@ -20463,30 +20501,63 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, return -EFAULT; buf[len] = '\0'; - p = strim(buf); - if (!strcmp(p, "-")) { + if (!strcmp(strim(buf), "-")) { gsp_phys_guest_ram_clear(); pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } - path = gsp_phys_guest_ram_token(&p); - tok = gsp_phys_guest_ram_token(&p); - if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + { + char *p = buf; + + while (nt < 5 && (t[nt] = gsp_phys_guest_ram_token(&p))) + nt++; + } + if (nt != 4 && nt != 5) { + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \" \", \" \" or \"-\"\n"); + return -EINVAL; + } + off = nt - 4; /* 0 = legacy form, 1 = per-mdev form */ + path = t[off]; + if (!path || kstrtoull(t[off + 1], 0, &gpa_base)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \" \" or \"-\"\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &file_offset)) { + if (kstrtoull(t[off + 2], 0, &file_offset)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &size)) { + if (kstrtoull(t[off + 3], 0, &size)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } + if (off) { + /* + * Per-mdev form: the selector must name an OPEN mdev (the mdev + * UUID is its device name). Refusing an unknown selector keeps + * a typo from silently configuring the legacy record. + */ + struct nv_vgpu_device *mdev = + gsp_phys_open_mdev_by_name(t[0]); + + if (!mdev) { + pr_err("vgsp-boot: gsp-phys-guest-ram: no open mdev named %s; write refused\n", + t[0]); + return -ENODEV; + } + ret = gsp_phys_guest_ram_configure_vgpu(mdev, path, gpa_base, + file_offset, size); + if (ret) { + pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + t[0], path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("vgsp-boot: gsp-phys-guest-ram: configured mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + t[0], path, gpa_base, file_offset, size); + return len; + } + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9b7235b..642a743 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1256,16 +1256,28 @@ EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The - * ranges map guest GPA windows to file offsets; up to four windows of one - * file per module cycle. + * ranges map guest GPA windows to file offsets; up to four windows per file. + * + * Per-mdev records (dual-guest goal 1.0): a guest RAM backing file belongs + * to ONE guest session, so the record moves with the mdev. Slot 0 is the + * legacy record (vgpu == NULL): it is what the historical top-level + * gsp-phys-guest-ram debugfs write configures and it serves the first/only + * open mdev exactly as the pre-dual-guest code did (one guest: byte-identical + * behaviour). A per-mdev record is configured with the mdev UUID as an extra + * first token on the same debugfs file and serves only that mdev; the + * record is released when the mdev is removed (gsp_phys_guest_ram_vgpu_clear) + * or when the physical cycle ends (gsp_phys_guest_ram_clear). * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. - * One file per module cycle: a second path is refused -EBUSY because a fresh - * cycle reloads the module; more ranges of the same file may be added. + * Within one record the historical rule is unchanged: the first configure + * opens the file, a different path is refused -EBUSY, more ranges of the + * same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 +/* Per-mdev guest-RAM records (a handful of guests; 4 is far beyond 1.0). */ +#define GSP_PHYS_GUEST_RAM_MDEVS 4 struct gsp_phys_guest_ram_range { u64 gpa_base; @@ -1273,35 +1285,121 @@ struct gsp_phys_guest_ram_range { u64 size; }; -static struct file *guest_ram_file; -static char guest_ram_path[128]; -static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; -static unsigned int guest_ram_ranges_n; +struct gsp_phys_guest_ram_rec { + struct nv_vgpu_device *vgpu; /* NULL = legacy (first/only open mdev) */ + struct file *file; + char path[128]; + struct gsp_phys_guest_ram_range ranges[GSP_PHYS_GUEST_RAM_RANGES]; + unsigned int ranges_n; +}; + +static struct gsp_phys_guest_ram_rec guest_ram_legacy; +static struct gsp_phys_guest_ram_rec guest_ram_mdevs[GSP_PHYS_GUEST_RAM_MDEVS]; static DEFINE_MUTEX(guest_ram_lock); +/* Caller holds guest_ram_lock. NULL when this vgpu has no bound record. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_find_locked(const struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return NULL; + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].vgpu == vgpu) + return &guest_ram_mdevs[i]; + return NULL; +} + +/* Caller holds guest_ram_lock. NULL when every per-mdev slot is taken. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_alloc_locked(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (!guest_ram_mdevs[i].vgpu) { + guest_ram_mdevs[i].vgpu = vgpu; + return &guest_ram_mdevs[i]; + } + return NULL; +} + /* - * Configure the guest RAM backing file and add (or replace, keyed by - * gpa_base) one GPA window. The first call opens the file read-only and - * fixes the path for this module cycle; later calls may only add ranges of - * that same file. The file must be a regular file and every range must fit - * inside its current size, so a translation can never index a page beyond - * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges - * in use, or the file/argument error. + * Number of mdevs whose VFIO device is currently open. Shared by the + * per-mdev debugfs selector (which mdev does this UUID name?) and the + * single-guest guard on the legacy test-only paths. */ -int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, - u64 file_offset, u64 size) +int gsp_phys_open_mdev_count(void) +{ + struct nv_vgpu_device *d; + int n = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) + n++; + } + mutex_unlock(&nv_vgpu_driver.lock); + return n; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_count); + +/* + * Resolve an OPEN mdev by its device name (the mdev UUID, e.g. + * "b6acdc5e-..."). Returns the device or NULL. The caller must use the + * result only while that mdev's VFIO fd is known to be open (the same + * lifetime rule as the first-open resolution below): the driver list is + * walked under nv_vgpu_driver.lock, but the returned pointer is not + * reference-counted. + */ +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name) +{ + struct nv_vgpu_device *d, *found = NULL; + + if (!name || !*name) + return NULL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0 && d->mdev && + !strcmp(dev_name(&d->mdev->dev), name)) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return found; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_by_name); + +/* + * Shared configure body for one record: open the record's file (first call) + * and add (or replace, keyed by gpa_base) one GPA window. The first call + * opens the file read-only and fixes the path for the record's lifetime; + * later calls may only add ranges of that same file. The file must be a + * regular file and every range must fit inside its current size, so a + * translation can never index a page beyond EOF. Returns 0, -EBUSY for a + * different path, -ENOSPC with all four ranges in use, or the + * file/argument error. `who` is "" for the legacy record and + * " for mdev " for a per-mdev record; the legacy strings stay + * byte-identical. + */ +static int gsp_phys_guest_ram_rec_configure(struct gsp_phys_guest_ram_rec *rec, + const char *path, u64 gpa_base, + u64 file_offset, u64 size, + const char *who) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; - if (!path || !*path) + if (!rec || !path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { + if (!rec->file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond @@ -1331,35 +1429,38 @@ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, ret = -ERANGE; goto out_fput; } - guest_ram_file = f; - strscpy(guest_ram_path, path, sizeof(guest_ram_path)); - } else if (strcmp(guest_ram_path, path)) { - pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", - guest_ram_path, path); + rec->file = f; + strscpy(rec->path, path, sizeof(rec->path)); + } else if (strcmp(rec->path, path)) { + if (who[0]) + pr_err("vgsp: guest RAM already configured from %s; one file per guest session (reload or mdev remove to switch); refusing %s\n", + rec->path, path); + else + pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + rec->path, path); ret = -EBUSY; goto out; } - for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { - if (guest_ram_ranges[i].gpa_base == gpa_base) { - range = &guest_ram_ranges[i]; + for (i = 0, range = NULL; i < rec->ranges_n; i++) { + if (rec->ranges[i].gpa_base == gpa_base) { + range = &rec->ranges[i]; break; } } if (!range) { - if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + if (rec->ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } - range = &guest_ram_ranges[guest_ram_ranges_n++]; + range = &rec->ranges[rec->ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; - pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", - guest_ram_path, gpa_base, file_offset, size, - guest_ram_ranges_n); + pr_info("vgsp: guest RAM configured%s: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + who, rec->path, gpa_base, file_offset, size, rec->ranges_n); out: mutex_unlock(&guest_ram_lock); @@ -1369,35 +1470,148 @@ out_fput: fput(f); goto out; } + +/* + * Configure the legacy guest RAM record (the historical top-level + * gsp-phys-guest-ram write). Serves the first/only open mdev exactly as + * before the per-mdev records existed. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + return gsp_phys_guest_ram_rec_configure(&guest_ram_legacy, path, + gpa_base, file_offset, size, + ""); +} EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* - * Close the configured guest RAM file (if any) and drop every range. Called - * from the physical-GPU release path so module unload and a backend release - * end the file reference; a fresh cycle reconfigures from debugfs. + * Configure the guest RAM record bound to one mdev (per-mdev selector on the + * debugfs write). A record for this vgpu is created on first use and closed + * by gsp_phys_guest_ram_vgpu_clear() when the mdev is removed. A failed + * first configure leaves no bound record behind. + */ +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_rec *rec; + char who[80]; + const char *name = (vgpu && vgpu->mdev) ? + dev_name(&vgpu->mdev->dev) : "?"; + bool fresh = false; + int ret; + + if (!vgpu) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec) { + rec = gsp_phys_guest_ram_rec_alloc_locked(vgpu); + if (!rec) { + mutex_unlock(&guest_ram_lock); + pr_err("vgsp: guest RAM: no free per-mdev record slot (max %u); refusing mdev %s\n", + GSP_PHYS_GUEST_RAM_MDEVS, name); + return -ENOSPC; + } + fresh = true; + } + mutex_unlock(&guest_ram_lock); + + scnprintf(who, sizeof(who), " for mdev %s", name); + ret = gsp_phys_guest_ram_rec_configure(rec, path, gpa_base, + file_offset, size, who); + + /* + * A fresh record whose first configure failed must not hold the slot + * (the vgpu pointer would otherwise be reserved with no file and the + * next attempt could not create a clean record). + */ + if (ret && fresh) { + mutex_lock(&guest_ram_lock); + if (rec->file) + ret = 0; /* raced with another configure; keep it */ + else + rec->vgpu = NULL; + mutex_unlock(&guest_ram_lock); + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure_vgpu); + +/* Lock held: close one record's file and drop its ranges. */ +static void gsp_phys_guest_ram_rec_close(struct gsp_phys_guest_ram_rec *rec, + const char *release_log) +{ + if (rec->file) { + pr_info("%s: path=%s ranges=%u\n", release_log, + rec->path, rec->ranges_n); + fput(rec->file); + rec->file = NULL; + } + rec->path[0] = '\0'; + memset(rec->ranges, 0, sizeof(rec->ranges)); + rec->ranges_n = 0; +} + +/* + * Close every configured guest RAM record (legacy and per-mdev) and drop + * every range. Called from the physical-GPU release path so module unload + * and a backend release end the file references; a fresh cycle reconfigures + * from debugfs. The legacy release log line is unchanged. */ void gsp_phys_guest_ram_clear(void) { + unsigned int i; + mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", - guest_ram_path, guest_ram_ranges_n); - fput(guest_ram_file); - guest_ram_file = NULL; - } - guest_ram_path[0] = '\0'; - memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); - guest_ram_ranges_n = 0; + gsp_phys_guest_ram_rec_close(&guest_ram_legacy, + "vgsp: guest RAM released"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); +/* + * Release the guest RAM record bound to one mdev. Called from the mdev + * cleanup path so the file reference and the vgpu pointer never outlive the + * session (a removed mdev's translations must stop, and the record slot must + * free up for a restarted guest). + */ +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return; + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + if (guest_ram_mdevs[i].vgpu == vgpu) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } + } + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_vgpu_clear); + bool gsp_phys_guest_ram_active(void) { - bool active; + bool active = false; + unsigned int i; mutex_lock(&guest_ram_lock); - active = guest_ram_file != NULL; + if (guest_ram_legacy.file) + active = true; + for (i = 0; !active && i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) + active = true; mutex_unlock(&guest_ram_lock); return active; } @@ -1405,33 +1619,78 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, - * path, range count and every range (gpa/offset/size). The caller supplies - * the buffer; nothing is written when the file is not configured beyond the - * active=0 marker. + * path, range count and every range (gpa/offset/size). With only the legacy + * record configured (the one-mdev campaign) the output is byte-identical to + * the pre-dual-guest format; a per-mdev record adds its mdev UUID so a + * dual-guest operator can tell the records apart. */ +static void gsp_phys_guest_ram_rec_state(char *buf, size_t len, + unsigned int *n, + const struct gsp_phys_guest_ram_rec *rec, + const char *label) +{ + unsigned int i; + + if (*n >= len) + return; + *n += scnprintf(buf + *n, len - *n, "%s path=%s ranges=%u", + label, rec->path, rec->ranges_n); + for (i = 0; i < rec->ranges_n && *n < len; i++) + *n += scnprintf(buf + *n, len - *n, + " gpa=0x%llx off=0x%llx size=0x%llx", + rec->ranges[i].gpa_base, + rec->ranges[i].file_offset, + rec->ranges[i].size); +} + void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; + bool any_mdev = false; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { - scnprintf(buf, len, "active=0"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) { + any_mdev = true; + break; + } + + if (!any_mdev) { + /* + * Legacy-only (or nothing): keep the historical shape - the + * campaign's parser (if any) sees the same text as before. + */ + if (!guest_ram_legacy.file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + n += scnprintf(buf + n, len - n, "active=1"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_legacy, ""); mutex_unlock(&guest_ram_lock); return; } - n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", - guest_ram_path, guest_ram_ranges_n); - for (i = 0; i < guest_ram_ranges_n && n < len; i++) - n += scnprintf(buf + n, len - n, - " gpa=0x%llx off=0x%llx size=0x%llx", - guest_ram_ranges[i].gpa_base, - guest_ram_ranges[i].file_offset, - guest_ram_ranges[i].size); + n += scnprintf(buf + n, len - n, "active=1"); + if (guest_ram_legacy.file) + gsp_phys_guest_ram_rec_state(buf, len, &n, &guest_ram_legacy, + " legacy"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS && n < len; i++) { + char label[64]; + + if (!guest_ram_mdevs[i].file) + continue; + scnprintf(label, sizeof(label), " mdev=%s", + (guest_ram_mdevs[i].vgpu && guest_ram_mdevs[i].vgpu->mdev) ? + dev_name(&guest_ram_mdevs[i].vgpu->mdev->dev) : "?"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_mdevs[i], label); + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); @@ -1452,12 +1711,20 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * - * Guards kept from the original path: the active-mdev walk + * Guards kept from the original path: the mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - - * the engine path must only run with a live guest session; no such session - * returns -ENODEV. The IOMMU-domain query remains as a documented fallback - * for a setup whose mdev has a real domain (it is inert with the - * emulated-IOMMU mdev used here). + * the engine path must only run with a live guest session; a vgpu given by + * the caller must itself be open, and no open session at all returns + * -ENODEV. The IOMMU-domain query remains as a documented fallback for a + * setup whose mdev has a real domain (it is inert with the emulated-IOMMU + * mdev used here). + * + * Per-mdev resolution (dual-guest goal 1.0): the caller names the owning + * vgpu (a captured forwarding request, a held release, ...). The vgpu's own + * record resolves the translation; the legacy record (configured by the + * historical top-level debugfs write) still serves the FIRST open mdev so + * single-guest scripts are unchanged. vgpu == NULL keeps the historical + * resolution: the first open mdev. * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file @@ -1467,13 +1734,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ -int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, u64 *hpa_out) { - struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; + struct nv_vgpu_device *first = NULL; + struct gsp_phys_guest_ram_rec *rec = NULL; struct iommu_domain *dom; phys_addr_t p; unsigned int i; + bool vgpu_gone = false; if (!hpa_out) return -EINVAL; @@ -1481,19 +1750,28 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; + first = d; break; } } + if (vgpu && atomic_read(&vgpu->vfio_dev_open_count) <= 0) + vgpu_gone = true; /* named session is gone: never retarget */ mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu_gone) + return -ENODEV; + if (!vgpu) + vgpu = first; /* historical resolution: first open mdev */ if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - for (i = 0; i < guest_ram_ranges_n; i++) { + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec && vgpu == first) + rec = &guest_ram_legacy; /* legacy file: first/only mdev */ + if (rec && rec->file) { + for (i = 0; i < rec->ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = - &guest_ram_ranges[i]; + &rec->ranges[i]; struct page *page; u64 file_off; @@ -1502,7 +1780,7 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) continue; file_off = r->file_offset + (gpa - r->gpa_base); - page = read_cache_page(guest_ram_file->f_mapping, + page = read_cache_page(rec->file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); @@ -1538,6 +1816,18 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) *hpa_out = (u64)p; return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa_vgpu); + +/* + * Historical entry point: resolve through the first open mdev (and that + * mdev's own record, else the legacy record). Used by the single-guest + * boot-ladder guest-GPA pin (test-only; see gsp_phys_guest_gpa_write) and + * kept as the compatibility wrapper for callers without a request context. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + return gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, hpa_out); +} EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 73b0822..e2be7a8 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -3,6 +3,8 @@ #include +struct nv_vgpu_device; + /* * Physical GPU ownership (real-GSP execution backend, bring-up rung 1). * Claim is disabled unless the gsp_phys_gpu module parameter is set. @@ -428,21 +430,50 @@ bool gsp_phys_rmcall_vas_shared_value(void); */ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out); +/* + * Per-mdev variant: translate the GPA for the named owning vgpu (a captured + * forwarding request, a held release). The vgpu's own guest-RAM record is + * authoritative; the legacy record still serves the FIRST open mdev so + * single-guest callers are unchanged. NULL keeps the historical resolution + * (first open mdev). A named vgpu whose session is closed returns -ENODEV + * rather than being retargeted at another guest. + */ +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, + u64 *hpa_out); + /* * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One - * shared file per module cycle, up to four GPA->file-offset ranges keyed by + * record per guest session, up to four GPA->file-offset ranges keyed by * gpa_base. The first configure opens the file; the same path adds/replaces * ranges, a different path is refused -EBUSY, and the ranges are dropped and - * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The + * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release (or + * gsp_phys_guest_ram_vgpu_clear() when one mdev is removed). The + * gsp_phys_guest_ram_configure() entry point configures the legacy record + * (the historical top-level debugfs write: serves the first/only open mdev); + * gsp_phys_guest_ram_configure_vgpu() binds the record to one mdev. The * state accessor is lock-safe and writes "active=0" or the active path, * range count and per-range gpa/off/size into the caller's buffer. */ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, u64 file_offset, u64 size); +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size); void gsp_phys_guest_ram_clear(void); +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu); bool gsp_phys_guest_ram_active(void); void gsp_phys_guest_ram_state(char *buf, size_t len); +/* + * Open-mdev helpers for the per-mdev debugfs selector and the single-guest + * guards: count the mdevs whose VFIO device is open, or resolve one open + * mdev by its device name (the mdev UUID). The returned device is not + * reference-counted; the caller must only use it while that mdev's fd is + * known to be open. + */ +int gsp_phys_open_mdev_count(void); +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name); + /* * Domain shape of the physical (claimed) device, as the P6a engine path * needs it: the module programs raw host PAs into the CE page tables, so the diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 8318f56..2ffc8ac 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2357,6 +2357,7 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, return; memset(&desc, 0, sizeof(desc)); + desc.vgpu = ctx->vgpu; desc.h_object = h_object; memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); @@ -3244,7 +3245,7 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, memcpy(&hObjOld, params + 8, 4); memset(params + 12, 0, 4); /* status = NV_OK */ - gsp_sema_service_channel_del(hObjOld); + gsp_sema_service_channel_del(ctx->vgpu, hObjOld); /* * Retire event records that refer to the object being freed. Posting an diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f8925df..c1eaf80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -162,6 +162,10 @@ struct sema_map_entry { struct svc_channel { bool valid; + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * table is keyed by (vgpu, h_object) + * because two guests' stock allocators + * choose identical handles */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ @@ -449,12 +453,65 @@ static u32 svc_ring_n; * client spins). One clean boot always works, so make the next session look * like one: when the channel set empties, request a flush and let the next * rebuild start from the same empty state as a fresh boot. + * + * Dual-guest staging (goal 1.0): the discovery tables are still shared + * between guests (per-guest learned tables are the staged G4 remainder), so + * the flush is requested only when the channel set is globally empty. The + * active decision is made per guest (svc_guests below): one guest closing + * its last channel never flushes while another guest still has live + * channels - otherwise the departing guest would wipe the live guest's + * pins/maps and stall it. Single-guest behaviour is unchanged: the one + * guest's last-channel close is also the global empty transition. */ static u8 svc_sess_flush; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ +/* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ +#define SVC_MAX_GUESTS 8 +struct svc_guest_sess { + struct nv_vgpu_device *vgpu; /* NULL = free slot */ + bool active; /* a channel of this guest is + * registered until this guest's + * last channel closes */ + u64 last_build; /* per-guest svc_rebuild freshness */ +}; +static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; +static DEFINE_SPINLOCK(svc_guest_lock); + +/* + * Find (or, with create, allocate) the session record for one mdev. The + * records are pointer-keyed; a removed mdev's record is freed by + * gsp_sema_service_vgpu_cleanup() so the pointer can never match a newly + * allocated device. + */ +static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu, + bool create) +{ + struct svc_guest_sess *sess = NULL; + unsigned long flags; + int i; + + if (!vgpu) + return NULL; + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_guests[i].vgpu == vgpu) { + sess = &svc_guests[i]; + break; + } + if (create && !sess && !svc_guests[i].vgpu) + sess = &svc_guests[i]; + } + if (sess && create && !sess->vgpu) { + memset(sess, 0, sizeof(*sess)); + sess->vgpu = vgpu; + } + spin_unlock_irqrestore(&svc_guest_lock, flags); + return sess; +} + /* * CeUtils scrubber channel (the guest PMA's internal CE channel created at * driver init). When the PMA must hand out scrubbed pages it waits on the @@ -476,6 +533,7 @@ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session st * block count (so the scrub wait completes). Idempotent, forward-only. */ struct svc_scrub { + struct nv_vgpu_device *vgpu; /* owning mdev (NULL = free slot) */ bool userd_ok; /* a 4096-entry channel was registered */ u32 h_object; /* handle of the registered scrubber channel */ u64 userd_pa; @@ -492,7 +550,34 @@ struct svc_scrub { u32 pa_gen; /* scanner registration generation (logged) */ u32 log_budget; }; -static struct svc_scrub svc_scrub; +/* + * One scrubber record per guest: every guest's PMA has its own 4096-entry + * scrubber channel, and the authored words must be written through the + * owning guest's address space (writing guest A's PA through guest B's VFIO + * device would corrupt B). Keyed by vgpu pointer, freed by + * gsp_sema_service_vgpu_cleanup(). + */ +static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS]; + +static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create) +{ + struct svc_scrub *s = NULL; + int i; + + if (!vgpu) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_scrubs[i].vgpu == vgpu) + return &svc_scrubs[i]; + if (create && !s && !svc_scrubs[i].vgpu) + s = &svc_scrubs[i]; + } + if (s) { + memset(s, 0, sizeof(*s)); + s->vgpu = vgpu; + } + return s; +} static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -1066,7 +1151,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu && + svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } @@ -1079,7 +1165,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) u64 keep_release_pa = 0; u8 keep_gr_index = 0, keep_ce_index = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); - bool same = ch->valid && ch->h_object == desc->h_object; + bool same = ch->valid && ch->vgpu == desc->vgpu && + ch->h_object == desc->h_object; /* * A re-registration of the same channel must not lose the @@ -1102,18 +1189,22 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); /* - * Rank among the live CE channels. The RM hands out object - * handles in creation order and the guest's UVM allocates one - * push buffer, ring and tracking semaphore per channel in that - * same order, so the rank is what ties a channel to its ring - * offset and its push-buffer base (see svc_ring_autopin / - * svc_pb_pair). + * Rank among the live CE channels OF THIS GUEST. The RM hands + * out object handles in creation order and the guest's UVM + * allocates one push buffer, ring and tracking semaphore per + * channel in that same order, so the rank is what ties a + * channel to its ring offset and its push-buffer base (see + * svc_ring_autopin / svc_pb_pair). The rank must not count the + * other guest's channels: their handles are independent and + * their allocators interleave. */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; @@ -1121,13 +1212,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && !svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->gr_index = rank; } ch->valid = true; svc_sess_active = 1; + ch->vgpu = desc->vgpu; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; @@ -1145,6 +1239,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } + /* Per-guest session flag: this guest has a live channel. */ + { + struct svc_guest_sess *sess = + svc_guest_get(desc->vgpu, true); + + if (sess) + sess->active = true; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -1169,25 +1271,30 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && desc->userd_pa) { + struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); + + if (!s) + goto scrub_done; /* table full: max guests */ /* Re-seed the counting on a fresh scrubber object. */ - if (!svc_scrub.userd_ok || - svc_scrub.h_object != desc->h_object || - svc_scrub.gpfifo_va != desc->gpfifo_va) { - svc_scrub.seeded = false; - svc_scrub.submitted = 0; + if (!s->userd_ok || + s->h_object != desc->h_object || + s->gpfifo_va != desc->gpfifo_va) { + s->seeded = false; + s->submitted = 0; } - svc_scrub.userd_ok = true; - svc_scrub.h_object = desc->h_object; - svc_scrub.userd_pa = desc->userd_pa; - svc_scrub.userd_as = desc->userd_as ? desc->userd_as - : SVC_APERTURE_SYS; - svc_scrub.gpfifo_va = desc->gpfifo_va; - svc_scrub.entries = desc->gpfifo_entries; + s->userd_ok = true; + s->h_object = desc->h_object; + s->userd_pa = desc->userd_pa; + s->userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + s->gpfifo_va = desc->gpfifo_va; + s->entries = desc->gpfifo_entries; pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, - svc_scrub.userd_as); + s->userd_as); } +scrub_done: pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -1199,7 +1306,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } /* P6b wave-1.5 hold state is defined with the forwarding section below. */ -static void svc_fwd_hold_abort(u32 h_object); +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object); /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. @@ -1218,7 +1325,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 entry_idx); static void svc_opdef_reset_all(void); -void gsp_sema_service_channel_del(u32 h_object) +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { unsigned long flags; int i; @@ -1228,7 +1335,8 @@ void gsp_sema_service_channel_del(u32 h_object) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); @@ -1238,19 +1346,27 @@ void gsp_sema_service_channel_del(u32 h_object) /* * ISSUE-0002: when the last channel closes, the next client is a new * session in a new UVM VA space; its discovery state must not inherit - * this session's map/pins/page-directory latch. Request the flush - * here; it executes at the next rebuild (the only writer of that - * state) just like a fresh boot would start empty. Only the transition - * from "a session existed" to "empty" may request it: redundant - * deletes for handles that never registered would otherwise re-flush - * (and wipe) a session that started after the real one ended. + * this session's map/pins/page-directory latch. The active decision is + * per guest (svc_guests) and the flush request is raised only when NO + * guest has a live channel left: the discovery tables are still shared + * between guests (staged G4), so flushing while another guest is live + * would wipe that guest's pins and stall it. One guest: the last-channel + * close IS the global empty transition, so behaviour is unchanged. */ if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false, own = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) - break; + if (!svc.chan[i].valid) + continue; + any = true; + if (svc.chan[i].vgpu == vgpu) + own = true; } - if (i == SVC_MAX_CHANNELS) { + if (sess && !own) + sess->active = false; + if (!any) { svc_sess_active = 0; svc_sess_flush = 1; } @@ -1265,9 +1381,10 @@ void gsp_sema_service_channel_del(u32 h_object) * P6b wave 1.5: a deleted channel's held releases must not wait out * the timeout; mark the hold aborted so the pump applies them (or * drops them if the snapshot already lost the channel) on its next - * pass. + * pass. The hold is matched per (vgpu, handle): the other guest's + * identical handle must not abort this guest's hold. */ - svc_fwd_hold_abort(h_object); + svc_fwd_hold_abort(vgpu, h_object); } /* @@ -5078,7 +5195,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { /* * The release targets the slot the push itself named * (release_pa learned from the release VA); the @@ -5150,6 +5268,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && + svc.chan[i].vgpu == vgpu && svc.chan[i].h_object == h) { if (payload > svc.chan[i].pending_rel_pay) { @@ -5229,7 +5348,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * harmless for an absolute release (its guard drops it), but an increment * would advance the slot twice. */ -static void svc_counter_pending_clear(u32 h) +static void svc_counter_pending_clear(struct nv_vgpu_device *vgpu, u32 h) { unsigned long flags; int i; @@ -5238,7 +5357,8 @@ static void svc_counter_pending_clear(u32 h) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { svc.chan[i].pending_rel_pay = 0; svc.chan[i].pending_rel_inc = 0; break; @@ -5272,13 +5392,13 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, if (payload > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); return; } } else if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (cur > ch->counter_last) ch->counter_last = cur; return; @@ -5289,7 +5409,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, return; ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (nv > ch->counter_last) ch->counter_last = nv; svc.releases_applied++; @@ -5428,6 +5548,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = ipa & ~0x3ULL; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = ch->counter_pa; @@ -5476,6 +5597,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, spin_lock_irqsave(&svc.chan_lock, flags); for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object != ch->h_object && svc.chan[j].counter_pa == pa) { collision = true; @@ -5501,6 +5623,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = pa; break; @@ -5560,7 +5683,8 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { if (svc.chan[i].release_pa != (pa & ~0x3ULL)) { svc.chan[i].release_pa = pa & ~0x3ULL; svc.chan[i].relearn_n++; @@ -7694,7 +7818,8 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) u32 slot, k; bool have = false; - if (!svc.chan[i].valid || !svc.chan[i].engine_type) + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + !svc.chan[i].engine_type) continue; ce = true; if (svc.chan[i].ce_index >= 32) @@ -8975,7 +9100,8 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) int pap = 0; u32 r = c->gr_index; - if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS) + if (!c->valid || c->vgpu != vgpu || c->engine_type || + r >= SVC_GR_CONTENT_RANKS) continue; if (test_bit(r, &svc_gr_ring_done)) continue; @@ -9316,7 +9442,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, */ spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && + svc.chan[k].engine_type && svc.chan[k].ce_index + 1 > max_ce) max_ce = svc.chan[k].ce_index + 1; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -10792,7 +10919,7 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, } if (kind != SVC_APERTURE_SYS) return -EINVAL; - ret = gsp_phys_gpa_to_hpa(page, hpa); + ret = gsp_phys_gpa_to_hpa_vgpu(vgpu, page, hpa); if (ret) return ret; *out_off = page_off; @@ -10961,9 +11088,10 @@ struct svc_fwd_hold { static struct svc_fwd_hold svc_fwd_hold; -static void svc_fwd_hold_abort(u32 h_object) +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object) { if (READ_ONCE(svc_fwd_hold.active) && + svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == h_object) WRITE_ONCE(svc_fwd_hold.aborted, true); } @@ -11334,18 +11462,22 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, } } -/* A slot whose channel is gone from this pass's snapshot: drop it. */ -static void svc_opdef_drop_gone(const struct svc_channel *local, int n) +/* A slot of THIS pump's guest whose channel is gone from the pass snapshot: + * drop it. Slots of other guests are never touched (their channels are not in + * this snapshot by construction). */ +static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, + const struct svc_channel *local, int n) { int i, k; for (i = 0; i < SVC_OPDEF_SLOTS; i++) { struct svc_opdef_ent *e = &svc_opdef[i]; - if (!e->active) + if (!e->active || e->vgpu != vgpu) continue; for (k = 0; k < n; k++) - if (local[k].h_object == e->h_object) + if (local[k].vgpu == e->vgpu && + local[k].h_object == e->h_object) break; if (k == n) svc_opdef_clear(e); @@ -11786,6 +11918,31 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, return true; } +/* + * Is this mdev still on the driver list with an open VFIO device? Decides + * whether a hold that names it can still be resolved through it: a removed + * device must never be dereferenced (its struct can be freed before the next + * pump pass would see the hold). Pointer comparisons only - safe to call + * with a possibly-stale pointer. + */ +static bool svc_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + /* * Channel vanished while a hold was active. Any resolved hold (completed, * aborted, failed or expired) applies the recorded releases through a shim @@ -11806,6 +11963,18 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + /* + * The owning mdev was removed mid-hold (the mdev cleanup path + * already released its session state): its guest can never + * observe the releases and dereferencing the device would be a + * use-after-free. Drop the hold; the request slot frees for the + * next guest. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } if (!done && !aborted && !svc_fwd_hold_expired()) return; /* @@ -11858,6 +12027,15 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) if (!svc_fwd_hold.active) return; + /* + * A hold whose owning mdev is already gone must never be dereferenced: + * drop it (the guest cannot observe the releases anyway). + */ + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } done = smp_load_acquire(&svc_fwd_req.done); /* * P6b wave 2: the backend is going away; a skipped candidate whose @@ -13471,7 +13649,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -13507,7 +13685,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * pass's snapshot belongs to a vanished channel; its waiter is gone, * so drop the set instead of resolving it later. */ - svc_opdef_drop_gone(local, n); + svc_opdef_drop_gone(vgpu, local, n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; @@ -15020,7 +15198,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { - if (svc.chan[k].valid && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; @@ -15334,7 +15512,7 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -15725,9 +15903,18 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish * payload the guest's scrub wait polls; hsem_pa is the channel's host * semaphore (channelWaitForFreeEntry polls it). Plain state update. + * + * Dual-guest: the words are written through a specific guest's address space + * (see svc_scrub_pass), so the registration binds to the FIRST open mdev - + * the same single-guest convention the scanner tooling follows (it tracks + * one client). Refused with no open mdev rather than parked globally. */ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) { + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct svc_scrub *s; + if (!svc.task) return -ENODEV; /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ @@ -15736,12 +15923,28 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) return -EINVAL; - svc_scrub.fin_pa = fin_pa; - svc_scrub.hsem_pa = hsem_pa; - svc_scrub.pa_ok = true; - svc_scrub.pa_gen++; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) { + pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n"); + return -ENODEV; + } + s = svc_scrub_get(vgpu, true); + if (!s) + return -ENOSPC; + + s->fin_pa = fin_pa; + s->hsem_pa = hsem_pa; + s->pa_ok = true; + s->pa_gen++; pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", - fin_pa, hsem_pa, svc_scrub.pa_gen); + fin_pa, hsem_pa, s->pa_gen); return 0; } @@ -15769,11 +15972,11 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) */ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) { - struct svc_scrub *s = &svc_scrub; + struct svc_scrub *s = svc_scrub_get(vgpu, false); u32 put = 0; u64 delta; - if (!s->userd_ok || !s->pa_ok || !s->entries) + if (!s || !s->userd_ok || !s->pa_ok || !s->entries) return; if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { @@ -15830,7 +16033,9 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) static int svc_thread(void *arg) { + struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; struct nv_vgpu_device *vgpu; + int n_vgpus, g; pr_info("sema-svc: thread started\n"); @@ -15850,130 +16055,174 @@ static int svc_thread(void *arg) complete_all(&svc_sess_flush_done); } - vgpu = NULL; + /* + * Service EVERY open mdev each pass, in list order. With one + * mdev this is exactly the historical single-vgpu pass; with + * two, both guests get a pump turn per iteration (hardware + * forwarding stays serialized by the single-inflight slot; see + * the forwarding section). The list walk is the historical + * bounded scan under nv_vgpu_driver.lock. + */ + n_vgpus = 0; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { - if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; - break; - } + if (atomic_read(&d->vfio_dev_open_count) > 0 && + n_vgpus < SVC_MAX_GUESTS) + vgpus[n_vgpus++] = d; } } mutex_unlock(&nv_vgpu_driver.lock); - if (!vgpu) { + if (!n_vgpus) { msleep(500); continue; } - svc_handle_probe(vgpu); - svc_handle_walk_probe(vgpu); + /* Host-injection queue drain: once per pass, not per guest. */ svc_handle_map_add(); - svc_handle_chan_dump(vgpu); - svc_scrub_pass(vgpu); - if (svc_sess_flush || !svc.direct_base || - time_after(jiffies, - (unsigned long)(svc.last_build + 10 * HZ))) - svc_rebuild(vgpu); + for (g = 0; g < n_vgpus; g++) { + struct svc_guest_sess *sess; + u64 done_before; - /* - * Channel-driven execution: notice submissions as they are made - * and run the pushed work immediately, so the guest's UVM - * progress waits complete in milliseconds rather than in the - * time it takes the fallback scan to find the push. A doorbell - * token means a channel was just scheduled; run everything now. - */ - { - u64 done_before = svc.gpfifo_entries_done; + vgpu = vgpus[g]; + svc_handle_probe(vgpu); + svc_handle_walk_probe(vgpu); + svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); + + /* + * Per-guest rebuild freshness: the discovery map is + * still shared between guests (staged G4), but each + * guest's RAM must be scanned on its own clock, or the + * first guest's 10 s tick would starve the second + * guest of rebuilds. The freshness stamp is the one + * svc_rebuild() publishes (svc.last_build). + */ + sess = svc_guest_get(vgpu, true); + if (svc_sess_flush || !svc.direct_base || + !sess || + time_after(jiffies, + (unsigned long)(sess->last_build + + 10 * HZ))) { + svc_rebuild(vgpu); + if (sess) + sess->last_build = svc.last_build; + } + /* + * Channel-driven execution: notice submissions as they + * are made and run the pushed work immediately, so the + * guest's UVM progress waits complete in milliseconds + * rather than in the time it takes the fallback scan + * to find the push. A doorbell token means a channel + * was just scheduled; run everything now. + */ + done_before = svc.gpfifo_entries_done; svc.doorbell_pending = false; svc_pump_channels(vgpu); - pumped = (svc.gpfifo_entries_done != done_before); - } + if (svc.gpfifo_entries_done != done_before) + pumped = true; - /* - * Host CUDA bridge: marshal one queued launch into a daemon - * job and resolve completions. Inert until a launch is seen. - */ - gsp_cuda_bridge_service(vgpu); + /* + * Host CUDA bridge: marshal one queued launch into a + * daemon job and resolve completions. Inert until a + * launch is seen. + */ + gsp_cuda_bridge_service(vgpu); - /* - * Releases whose target VA was not translatable when the scan - * saw them: retry every pass, so a pin or a learned pair applies - * them immediately instead of on the next full sweep. - */ - svc_apply_pending(vgpu); + /* + * Releases whose target VA was not translatable when + * the scan saw them: retry every pass, so a pin or a + * learned pair applies them immediately instead of on + * the next full sweep. + */ + svc_apply_pending(vgpu); - /* - * A fresh VA->PA injection changes which releases are - * applicable: run the fast sliced sweep while the forced window - * is open (the pump's stall path also uses it). - */ - if (time_before(jiffies, (unsigned long)svc.fast_force_until)) - svc_apply_sweep_slice(vgpu); + /* + * A fresh VA->PA injection changes which releases are + * applicable: run the fast sliced sweep while the + * forced window is open (the pump's stall path also + * uses it). + */ + if (time_before(jiffies, + (unsigned long)svc.fast_force_until)) + svc_apply_sweep_slice(vgpu); + } /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip - * the discovery/sweep passes, which otherwise dominate the pass - * period (the pump cadence is what the client's context init - * deadline sees). Scans resume the first pass without progress. + * the discovery/sweep passes, which otherwise dominate the + * pass period (the pump cadence is what the client's context + * init deadline sees). Scans resume the first pass without + * progress. (With two guests, any guest's progress keeps the + * loop tight; the next iteration pumps both.) */ if (pumped) { msleep(1); continue; } - /* - * Push-buffer base discovery: a chunked walk of guest RAM - * finds the pages whose first push names a channel, so a - * channel that appears with work already waiting is pinned and - * pumped without waiting for the rolling sweep. - */ - svc_pb_discover(vgpu); + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; - /* - * Completion hint: re-post the events for a few seconds after a - * completion, so a client that starts waiting after the - * edge-triggered wake-up still observes it. - */ - if (time_before(jiffies, (unsigned long)svc.post_hint_until) && - time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { - int rc; + /* + * Push-buffer base discovery: a chunked walk of guest + * RAM finds the pages whose first push names a + * channel, so a channel that appears with work + * already waiting is pinned and pumped without + * waiting for the rolling sweep. + */ + svc_pb_discover(vgpu); - svc.last_hint = jiffies; - rc = gsp_rm_post_all_events(vgpu); - if (gsp_sema_dbg) - pr_info_ratelimited("sema-svc: completion hint rc=%d\n", - rc); - } + /* + * Completion hint: re-post the events for a few + * seconds after a completion, so a client that starts + * waiting after the edge-triggered wake-up still + * observes it. + */ + if (time_before(jiffies, + (unsigned long)svc.post_hint_until) && + time_after(jiffies, + (unsigned long)svc.last_hint + HZ / 2)) { + int rc; + + svc.last_hint = jiffies; + rc = gsp_rm_post_all_events(vgpu); + if (gsp_sema_dbg) + pr_info_ratelimited("sema-svc: completion hint rc=%d\n", + rc); + } - /* - * Fast path for the pushes themselves: every mapped page is a - * candidate pushbuffer page, and there are far fewer of those - * than there are pages of guest RAM. - */ - if (svc.direct_base && svc.map_n) - svc_apply_mapped(vgpu); + /* + * Fast path for the pushes themselves: every mapped + * page is a candidate pushbuffer page, and there are + * far fewer of those than there are pages of guest + * RAM. + */ + if (svc.direct_base && svc.map_n) + svc_apply_mapped(vgpu); - /* - * Rolling RAM scan: the pushbuffers of some clients are not - * covered by the VA map, so sweep guest RAM in bounded windows - * in parallel with the targeted pass. Work per pass is fixed, - * which keeps the emulation responsive while a full sweep - * completes every few seconds. - */ - svc_apply_pushes(vgpu); + /* + * Rolling RAM scan: the pushbuffers of some clients + * are not covered by the VA map, so sweep guest RAM in + * bounded windows in parallel with the targeted pass. + * Work per pass is fixed, which keeps the emulation + * responsive while a full sweep completes every few + * seconds. + */ + svc_apply_pushes(vgpu); - /* - * Deferred readability probes for faked CE pushes (observation - * only; see svc_retry_poll). - */ - svc_retry_poll(vgpu); + /* + * Deferred readability probes for faked CE pushes + * (observation only; see svc_retry_poll). + */ + svc_retry_poll(vgpu); + } msleep(10); } @@ -16096,6 +16345,97 @@ void gsp_sema_service_stop(void) svc_vas_free_all(); } +/* + * Release every piece of service state owned by one mdev (dual-guest goal + * 1.0). Called from the mdev cleanup path after that mdev's emulation thread + * stopped and before its struct can be freed: + * - quiesce forwarding so no in-flight work item can still be reading the + * request's vgpu, and abort a release hold that names it; + * - drop any hold that survives (a request that never queued leaves no work + * item to clear it) BEFORE the device can go away - resolution would + * dereference it; + * - drop the guest's channels; request the global session flush only when no + * guest has live channels left (the discovery tables are still shared; + * staged G4); + * - drop the guest's held-operand release slots (its waiters are gone); + * - free the per-guest session and scrubber records so the vgpu pointer can + * never match a newly allocated device. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + if (!vgpu) + return; + + /* + * The forward work item runs on its own kthread and may be using the + * request's vgpu; quiesce waits for it (bounded by the submit's + * 200 ms poll cap) and aborts an active hold. Resume afterwards so the + * other guests keep forwarding. + */ + gsp_sema_service_work_quiesce(); + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) + WRITE_ONCE(svc_fwd_hold.aborted, true); + gsp_sema_service_work_resume(); + + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) { + /* + * Pure state clear (never a resolution through the vanishing + * device): the releases belong to a guest that is gone. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + } + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + + if (!ch->valid || ch->vgpu != vgpu) + continue; + pr_info("sema-svc: channel 0x%x dropped (mdev cleanup; submissions=%llu)\n", + ch->h_object, ch->submissions); + memset(ch, 0, sizeof(*ch)); + } + if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid) { + any = true; + break; + } + if (sess) + sess->active = false; + if (!any) { + svc_sess_active = 0; + svc_sess_flush = 1; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + /* Held-operand release slots of this guest cannot be delivered. */ + for (i = 0; i < SVC_OPDEF_SLOTS; i++) + if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu) + svc_opdef_clear(&svc_opdef[i]); + + /* Free the per-guest session and scrubber records. */ + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_guests[i].vgpu == vgpu) + memset(&svc_guests[i], 0, sizeof(svc_guests[i])); + spin_unlock_irqrestore(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_scrubs[i].vgpu == vgpu) + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + + if (svc_sess_flush && svc.task) + wake_up_process(svc.task); +} + /* ------------------------------------------------------------------ */ /* host CUDA bridge exports (gsp_cuda_bridge.c) */ /* ------------------------------------------------------------------ */ diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 6822bd5..c1dd763 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -18,8 +18,16 @@ * * All addresses are guest-physical except gpfifo_va, which is the GPU VA of * the channel's GPFIFO ring (translated through the discovered VA->PA map). + * + * vgpu names the owning mdev (guest session). Two guests' stock allocators + * choose the same object handles, so the channel table is keyed by + * (vgpu, h_object): a registration can never overwrite another guest's slot, + * and ranks/session bookkeeping are computed per guest. */ +struct nv_vgpu_device; + struct sema_channel_desc { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session) */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (the doorbell token vector) */ @@ -33,7 +41,17 @@ struct sema_channel_desc { }; void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); -void gsp_sema_service_channel_del(u32 h_object); +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); + +/* + * Release every piece of service state that belongs to one mdev before its + * guest session disappears: drop the mdev's channels (per-vgpu session end), + * clear a release hold that names it (never dereference a removed device), + * and free its per-guest session record. Called from the mdev cleanup path + * after the device's emulation thread stopped and before the mdev struct can + * be freed. No-op for a vgpu that never registered a channel. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu); /* * Register the guest PMA scrubber's channel-buffer words (the CeUtils diff --git a/kernel/main.c b/kernel/main.c index d0308c4..257638a 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -1739,6 +1739,16 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work) nv_vgpu_migration_cleanup(vgpu); gsp_emu_cleanup(vgpu); + /* + * Dual-guest (goal 1.0): release everything the sema service still holds + * for this mdev (channel entries, session/scrubber records, a release + * hold that names it - quiescing any in-flight forwarding work) and drop + * its per-mdev guest-RAM record, BEFORE the vgpu struct can be freed: a + * stale pointer in the service's tables must never match a later device. + */ + gsp_sema_service_vgpu_cleanup(vgpu); + gsp_phys_guest_ram_vgpu_clear(vgpu); + /* * Last-reference backend teardown: gsp_emu_cleanup() above has stopped * this mdev's emulation thread, so no forward can be in flight when the -- cgit v1.2.3 From 5769d5c43f97434fa4b82b4dab2f83ebee7ed73c Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 15 Oct 2026 18:02:49 -0400 Subject: vgsp-dual: cuda bridge queued work carries its owning mdev (G5 follow-on) The host CUDA bridge is the GR fallback path when a forward request is refused under the single-inflight slot (the expected steady state with two guests). Its FIFO request and pending records dropped the owning vgpu, so guest B's fallback job could be marshalled (VAs resolved, guest memory read/written) through guest A's address space. - cubb_req/cubb_pend gain the owning mdev; the launch stores it and it is part of the dedup identity. - Only the owning guest's service pass marshals its job or drives its staged completion; a head/record whose mdev is gone is dropped instead of ever being dereferenced. - One guest: head_owner is always the pumped vgpu, so behaviour is unchanged. --- kernel/gsp_cuda_bridge.c | 92 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_cuda_bridge.c b/kernel/gsp_cuda_bridge.c index 2fac2b2..7adc8e3 100644 --- a/kernel/gsp_cuda_bridge.c +++ b/kernel/gsp_cuda_bridge.c @@ -141,6 +141,9 @@ MODULE_PARM_DESC(cuda_bridge, /* ------------------------------------------------------------------ */ struct cubb_req { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * queue is FIFO, so only the owner's + * service pass may marshal this job */ u32 h_object; u64 qmd_dev_va; u64 push_va; @@ -178,6 +181,9 @@ static u64 cubb_dedup_hits; struct cubb_pend { bool valid; + struct nv_vgpu_device *vgpu; /* owning mdev: the completion write-back + * must run through this guest's + * address space, never another's */ u64 seq; u32 h_object; u32 opcode; @@ -576,12 +582,15 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, if (!gsp_cuda_bridge_enabled()) return; - (void)vgpu; for (i = 0; i < 64; i++) { hash ^= qmd[i]; hash *= 1099511628211ULL; } + /* The owner is part of the identity: two guests' identical launches + * must never dedup against each other. */ + hash ^= (u64)(unsigned long)vgpu; + hash *= 1099511628211ULL; hash ^= (u64)h_object << 32; hash *= 1099511628211ULL; hash ^= qmd_dev_va; @@ -612,6 +621,7 @@ void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, struct cubb_req *r = &cubb_req[cubb_req_head % CUBB2_REQ_MAX]; + r->vgpu = vgpu; r->h_object = h_object; r->qmd_dev_va = qmd_dev_va; r->push_va = push_va; @@ -972,6 +982,7 @@ static int cubb_marshal_one(struct nv_vgpu_device *vgpu) } job.seq = ++cubb_seq; cubb_pend[slot].valid = true; + cubb_pend[slot].vgpu = r.vgpu; cubb_pend[slot].seq = job.seq; cubb_pend[slot].h_object = r.h_object; cubb_pend[slot].opcode = opcode; @@ -1302,13 +1313,49 @@ static bool cubb_finish_try(struct nv_vgpu_device *vgpu, struct cubb_pend *p, return true; } +/* + * Is this mdev still on the driver list with an open VFIO device? Guards the + * bridge's per-request/per-pend owner pointers (dual-guest goal 1.0): a + * removed mdev must never be dereferenced and its queued work can never be + * observed by a live guest. Pointer comparisons only - safe to call with a + * possibly-stale pointer. + */ +static bool cubb_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + /* * Service-thread hook: marshal at most one queued launch into a daemon-visible * job, stage any returned done records, then drive staged completions. Called * every service pass; a no-op when no launch has been seen. + * + * Dual-guest (goal 1.0): request and pend records carry their owning mdev + * (the bridge falls back here when forwarding was refused under the + * single-inflight slot). Only the owning guest's pass marshals its job (the + * marshal resolves VAs and reads guest memory through that guest) and only + * the owning pass writes a completion back; a head/record whose owner is + * gone is dropped instead of ever being dereferenced. */ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) { + struct nv_vgpu_device *head_owner = NULL; + unsigned long flags; + u32 head_h = 0; + bool have_head = false; struct cubb2_done d; int guard = 0; int i; @@ -1316,7 +1363,29 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) if (!gsp_cuda_bridge_enabled()) return; - (void)cubb_marshal_one(vgpu); + spin_lock_irqsave(&cubb_req_lock, flags); + if (cubb_req_head != cubb_req_tail) { + struct cubb_req *head = + &cubb_req[cubb_req_tail % CUBB2_REQ_MAX]; + + head_owner = head->vgpu; + head_h = head->h_object; + have_head = true; + } + spin_unlock_irqrestore(&cubb_req_lock, flags); + if (have_head) { + if (head_owner == vgpu) { + (void)cubb_marshal_one(vgpu); + } else if (!cubb_vgpu_open(head_owner)) { + /* The owner's guest is gone: nobody can observe it. */ + cubb_req_drop_head(); + cubb_dropped++; + pr_info_ratelimited("cuda-bridge: dropped a queued launch whose mdev is gone (h=0x%x)\n", + head_h); + } + /* else: another live guest's head; its own pass marshals it. */ + } + while (guard++ < CUBB2_DONE_RING && cubb_done_pop(&d)) { if (d.magic != CUBB2_DONE_MAGIC || d.version != CUBB2_VERSION) { @@ -1330,8 +1399,23 @@ void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu) mutex_lock(&cubb_lock); for (i = 0; i < CUBB2_PEND_MAX; i++) { - if (cubb_pend[i].valid && cubb_pend[i].have_done && - cubb_finish_try(vgpu, &cubb_pend[i], i)) + struct cubb_pend *p = &cubb_pend[i]; + + if (!p->valid || !p->have_done) + continue; + if (p->vgpu != vgpu) { + /* + * Another guest's staged completion: its own pass + * drives the write-back. A removed owner's record is + * released here so the slot frees. + */ + if (!cubb_vgpu_open(p->vgpu)) { + cubb_pend_release(p, i); + i = -1; /* compaction shifted the entries */ + } + continue; + } + if (cubb_finish_try(vgpu, p, i)) i = -1; /* compaction shifted the entries; rescan */ } mutex_unlock(&cubb_lock); -- cgit v1.2.3