diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 115 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 432 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 35 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 3 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 644 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 20 | ||||
| -rw-r--r-- | kernel/main.c | 10 |
7 files changed, 1010 insertions, 249 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 9eb1179..9a4aedd 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -9728,8 +9728,16 @@ static void gsp_phys_rmcall_pdb_build(void) u64 guest_pte; int gpa_ret; - gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, - &guest_hpa); + /* + * The pinned GPA is a single-guest test aid (the pin write is + * refused while more than one mdev is open; see + * gsp_phys_guest_gpa_write): resolve through the first open + * mdev exactly as before the per-mdev records, via the NULL + * selector (a per-mdev pin is not modelled). + */ + gpa_ret = gsp_phys_gpa_to_hpa_vgpu(NULL, + boot_stage.rmcall_guest_gpa, + &guest_hpa); if (!gpa_ret) { guest_pte = gsp_phys_rmcall_pte(guest_hpa); put_unaligned_le64(guest_pte, @@ -19110,7 +19118,11 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -19193,7 +19205,11 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -20404,6 +20420,18 @@ static ssize_t gsp_phys_guest_gpa_write(struct file *file, gpa); return -EINVAL; } + /* + * Dual-guest guard (goal 1.0): the pin is a single-guest test aid. The + * triggers translate it through the first open mdev (see + * gsp_phys_gpa_to_hpa / boot_stage.rmcall_guest_gpa), so with more than + * one open mdev the pin cannot name its owner and is refused instead of + * silently resolving through whichever guest opened first. + */ + if (gsp_phys_open_mdev_count() > 1) { + pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: %d mdevs are open; the guest-GPA pin is a single-guest test aid (translates through the first open mdev). Close all but one mdev, or use the per-mdev guest/GPA paths\n", + gsp_phys_open_mdev_count()); + return -EBUSY; + } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", @@ -20426,18 +20454,26 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* - * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM - * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing + * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is * * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" * * (path without spaces; all three numbers hex with 0x, e.g. - * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the - * file and fixes the path for this module cycle; later writes may add ranges - * (up to four) or replace a range with the same gpa_base. The ranges are - * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev - * session exists. A single "-" clears the configuration and closes the file. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). This historical form + * configures the legacy record and serves the FIRST/ONLY open mdev exactly + * as before the per-mdev records existed. With two live guests each writes + * its own record by naming its mdev UUID first: + * + * "<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * Paths and UUIDs contain no spaces, so the token count (4 vs 5) decides. + * The first write for a record opens the file and fixes the path for that + * record; later writes may add ranges (up to four) or replace a range with + * the same gpa_base. The ranges are only consulted by the translators, and + * only while a live mdev session exists. A single "-" clears ALL records + * (legacy and per-mdev) and closes the files. */ static char *gsp_phys_guest_ram_token(char **cursor) { @@ -20452,9 +20488,11 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { - char buf[384]; - char *p, *tok, *path; + char buf[448]; + char *t[5]; + char *path; u64 gpa_base, file_offset, size; + int nt = 0, off; int ret; if (len == 0 || len >= sizeof(buf)) @@ -20463,30 +20501,63 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, return -EFAULT; buf[len] = '\0'; - p = strim(buf); - if (!strcmp(p, "-")) { + if (!strcmp(strim(buf), "-")) { gsp_phys_guest_ram_clear(); pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } - path = gsp_phys_guest_ram_token(&p); - tok = gsp_phys_guest_ram_token(&p); - if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + { + char *p = buf; + + while (nt < 5 && (t[nt] = gsp_phys_guest_ram_token(&p))) + nt++; + } + if (nt != 4 && nt != 5) { + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\", \"<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + off = nt - 4; /* 0 = legacy form, 1 = per-mdev form */ + path = t[off]; + if (!path || kstrtoull(t[off + 1], 0, &gpa_base)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &file_offset)) { + if (kstrtoull(t[off + 2], 0, &file_offset)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &size)) { + if (kstrtoull(t[off + 3], 0, &size)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } + if (off) { + /* + * Per-mdev form: the selector must name an OPEN mdev (the mdev + * UUID is its device name). Refusing an unknown selector keeps + * a typo from silently configuring the legacy record. + */ + struct nv_vgpu_device *mdev = + gsp_phys_open_mdev_by_name(t[0]); + + if (!mdev) { + pr_err("vgsp-boot: gsp-phys-guest-ram: no open mdev named %s; write refused\n", + t[0]); + return -ENODEV; + } + ret = gsp_phys_guest_ram_configure_vgpu(mdev, path, gpa_base, + file_offset, size); + if (ret) { + pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + t[0], path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("vgsp-boot: gsp-phys-guest-ram: configured mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + t[0], path, gpa_base, file_offset, size); + return len; + } + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9b7235b..642a743 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1256,16 +1256,28 @@ EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The - * ranges map guest GPA windows to file offsets; up to four windows of one - * file per module cycle. + * ranges map guest GPA windows to file offsets; up to four windows per file. + * + * Per-mdev records (dual-guest goal 1.0): a guest RAM backing file belongs + * to ONE guest session, so the record moves with the mdev. Slot 0 is the + * legacy record (vgpu == NULL): it is what the historical top-level + * gsp-phys-guest-ram debugfs write configures and it serves the first/only + * open mdev exactly as the pre-dual-guest code did (one guest: byte-identical + * behaviour). A per-mdev record is configured with the mdev UUID as an extra + * first token on the same debugfs file and serves only that mdev; the + * record is released when the mdev is removed (gsp_phys_guest_ram_vgpu_clear) + * or when the physical cycle ends (gsp_phys_guest_ram_clear). * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. - * One file per module cycle: a second path is refused -EBUSY because a fresh - * cycle reloads the module; more ranges of the same file may be added. + * Within one record the historical rule is unchanged: the first configure + * opens the file, a different path is refused -EBUSY, more ranges of the + * same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 +/* Per-mdev guest-RAM records (a handful of guests; 4 is far beyond 1.0). */ +#define GSP_PHYS_GUEST_RAM_MDEVS 4 struct gsp_phys_guest_ram_range { u64 gpa_base; @@ -1273,35 +1285,121 @@ struct gsp_phys_guest_ram_range { u64 size; }; -static struct file *guest_ram_file; -static char guest_ram_path[128]; -static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; -static unsigned int guest_ram_ranges_n; +struct gsp_phys_guest_ram_rec { + struct nv_vgpu_device *vgpu; /* NULL = legacy (first/only open mdev) */ + struct file *file; + char path[128]; + struct gsp_phys_guest_ram_range ranges[GSP_PHYS_GUEST_RAM_RANGES]; + unsigned int ranges_n; +}; + +static struct gsp_phys_guest_ram_rec guest_ram_legacy; +static struct gsp_phys_guest_ram_rec guest_ram_mdevs[GSP_PHYS_GUEST_RAM_MDEVS]; static DEFINE_MUTEX(guest_ram_lock); +/* Caller holds guest_ram_lock. NULL when this vgpu has no bound record. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_find_locked(const struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return NULL; + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].vgpu == vgpu) + return &guest_ram_mdevs[i]; + return NULL; +} + +/* Caller holds guest_ram_lock. NULL when every per-mdev slot is taken. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_alloc_locked(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (!guest_ram_mdevs[i].vgpu) { + guest_ram_mdevs[i].vgpu = vgpu; + return &guest_ram_mdevs[i]; + } + return NULL; +} + /* - * Configure the guest RAM backing file and add (or replace, keyed by - * gpa_base) one GPA window. The first call opens the file read-only and - * fixes the path for this module cycle; later calls may only add ranges of - * that same file. The file must be a regular file and every range must fit - * inside its current size, so a translation can never index a page beyond - * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges - * in use, or the file/argument error. + * Number of mdevs whose VFIO device is currently open. Shared by the + * per-mdev debugfs selector (which mdev does this UUID name?) and the + * single-guest guard on the legacy test-only paths. */ -int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, - u64 file_offset, u64 size) +int gsp_phys_open_mdev_count(void) +{ + struct nv_vgpu_device *d; + int n = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) + n++; + } + mutex_unlock(&nv_vgpu_driver.lock); + return n; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_count); + +/* + * Resolve an OPEN mdev by its device name (the mdev UUID, e.g. + * "b6acdc5e-..."). Returns the device or NULL. The caller must use the + * result only while that mdev's VFIO fd is known to be open (the same + * lifetime rule as the first-open resolution below): the driver list is + * walked under nv_vgpu_driver.lock, but the returned pointer is not + * reference-counted. + */ +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name) +{ + struct nv_vgpu_device *d, *found = NULL; + + if (!name || !*name) + return NULL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0 && d->mdev && + !strcmp(dev_name(&d->mdev->dev), name)) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return found; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_by_name); + +/* + * Shared configure body for one record: open the record's file (first call) + * and add (or replace, keyed by gpa_base) one GPA window. The first call + * opens the file read-only and fixes the path for the record's lifetime; + * later calls may only add ranges of that same file. The file must be a + * regular file and every range must fit inside its current size, so a + * translation can never index a page beyond EOF. Returns 0, -EBUSY for a + * different path, -ENOSPC with all four ranges in use, or the + * file/argument error. `who` is "" for the legacy record and + * " for mdev <uuid>" for a per-mdev record; the legacy strings stay + * byte-identical. + */ +static int gsp_phys_guest_ram_rec_configure(struct gsp_phys_guest_ram_rec *rec, + const char *path, u64 gpa_base, + u64 file_offset, u64 size, + const char *who) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; - if (!path || !*path) + if (!rec || !path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { + if (!rec->file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond @@ -1331,35 +1429,38 @@ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, ret = -ERANGE; goto out_fput; } - guest_ram_file = f; - strscpy(guest_ram_path, path, sizeof(guest_ram_path)); - } else if (strcmp(guest_ram_path, path)) { - pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", - guest_ram_path, path); + rec->file = f; + strscpy(rec->path, path, sizeof(rec->path)); + } else if (strcmp(rec->path, path)) { + if (who[0]) + pr_err("vgsp: guest RAM already configured from %s; one file per guest session (reload or mdev remove to switch); refusing %s\n", + rec->path, path); + else + pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + rec->path, path); ret = -EBUSY; goto out; } - for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { - if (guest_ram_ranges[i].gpa_base == gpa_base) { - range = &guest_ram_ranges[i]; + for (i = 0, range = NULL; i < rec->ranges_n; i++) { + if (rec->ranges[i].gpa_base == gpa_base) { + range = &rec->ranges[i]; break; } } if (!range) { - if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + if (rec->ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } - range = &guest_ram_ranges[guest_ram_ranges_n++]; + range = &rec->ranges[rec->ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; - pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", - guest_ram_path, gpa_base, file_offset, size, - guest_ram_ranges_n); + pr_info("vgsp: guest RAM configured%s: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + who, rec->path, gpa_base, file_offset, size, rec->ranges_n); out: mutex_unlock(&guest_ram_lock); @@ -1369,35 +1470,148 @@ out_fput: fput(f); goto out; } + +/* + * Configure the legacy guest RAM record (the historical top-level + * gsp-phys-guest-ram write). Serves the first/only open mdev exactly as + * before the per-mdev records existed. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + return gsp_phys_guest_ram_rec_configure(&guest_ram_legacy, path, + gpa_base, file_offset, size, + ""); +} EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* - * Close the configured guest RAM file (if any) and drop every range. Called - * from the physical-GPU release path so module unload and a backend release - * end the file reference; a fresh cycle reconfigures from debugfs. + * Configure the guest RAM record bound to one mdev (per-mdev selector on the + * debugfs write). A record for this vgpu is created on first use and closed + * by gsp_phys_guest_ram_vgpu_clear() when the mdev is removed. A failed + * first configure leaves no bound record behind. + */ +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_rec *rec; + char who[80]; + const char *name = (vgpu && vgpu->mdev) ? + dev_name(&vgpu->mdev->dev) : "?"; + bool fresh = false; + int ret; + + if (!vgpu) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec) { + rec = gsp_phys_guest_ram_rec_alloc_locked(vgpu); + if (!rec) { + mutex_unlock(&guest_ram_lock); + pr_err("vgsp: guest RAM: no free per-mdev record slot (max %u); refusing mdev %s\n", + GSP_PHYS_GUEST_RAM_MDEVS, name); + return -ENOSPC; + } + fresh = true; + } + mutex_unlock(&guest_ram_lock); + + scnprintf(who, sizeof(who), " for mdev %s", name); + ret = gsp_phys_guest_ram_rec_configure(rec, path, gpa_base, + file_offset, size, who); + + /* + * A fresh record whose first configure failed must not hold the slot + * (the vgpu pointer would otherwise be reserved with no file and the + * next attempt could not create a clean record). + */ + if (ret && fresh) { + mutex_lock(&guest_ram_lock); + if (rec->file) + ret = 0; /* raced with another configure; keep it */ + else + rec->vgpu = NULL; + mutex_unlock(&guest_ram_lock); + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure_vgpu); + +/* Lock held: close one record's file and drop its ranges. */ +static void gsp_phys_guest_ram_rec_close(struct gsp_phys_guest_ram_rec *rec, + const char *release_log) +{ + if (rec->file) { + pr_info("%s: path=%s ranges=%u\n", release_log, + rec->path, rec->ranges_n); + fput(rec->file); + rec->file = NULL; + } + rec->path[0] = '\0'; + memset(rec->ranges, 0, sizeof(rec->ranges)); + rec->ranges_n = 0; +} + +/* + * Close every configured guest RAM record (legacy and per-mdev) and drop + * every range. Called from the physical-GPU release path so module unload + * and a backend release end the file references; a fresh cycle reconfigures + * from debugfs. The legacy release log line is unchanged. */ void gsp_phys_guest_ram_clear(void) { + unsigned int i; + mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", - guest_ram_path, guest_ram_ranges_n); - fput(guest_ram_file); - guest_ram_file = NULL; - } - guest_ram_path[0] = '\0'; - memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); - guest_ram_ranges_n = 0; + gsp_phys_guest_ram_rec_close(&guest_ram_legacy, + "vgsp: guest RAM released"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); +/* + * Release the guest RAM record bound to one mdev. Called from the mdev + * cleanup path so the file reference and the vgpu pointer never outlive the + * session (a removed mdev's translations must stop, and the record slot must + * free up for a restarted guest). + */ +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return; + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + if (guest_ram_mdevs[i].vgpu == vgpu) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } + } + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_vgpu_clear); + bool gsp_phys_guest_ram_active(void) { - bool active; + bool active = false; + unsigned int i; mutex_lock(&guest_ram_lock); - active = guest_ram_file != NULL; + if (guest_ram_legacy.file) + active = true; + for (i = 0; !active && i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) + active = true; mutex_unlock(&guest_ram_lock); return active; } @@ -1405,33 +1619,78 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, - * path, range count and every range (gpa/offset/size). The caller supplies - * the buffer; nothing is written when the file is not configured beyond the - * active=0 marker. + * path, range count and every range (gpa/offset/size). With only the legacy + * record configured (the one-mdev campaign) the output is byte-identical to + * the pre-dual-guest format; a per-mdev record adds its mdev UUID so a + * dual-guest operator can tell the records apart. */ +static void gsp_phys_guest_ram_rec_state(char *buf, size_t len, + unsigned int *n, + const struct gsp_phys_guest_ram_rec *rec, + const char *label) +{ + unsigned int i; + + if (*n >= len) + return; + *n += scnprintf(buf + *n, len - *n, "%s path=%s ranges=%u", + label, rec->path, rec->ranges_n); + for (i = 0; i < rec->ranges_n && *n < len; i++) + *n += scnprintf(buf + *n, len - *n, + " gpa=0x%llx off=0x%llx size=0x%llx", + rec->ranges[i].gpa_base, + rec->ranges[i].file_offset, + rec->ranges[i].size); +} + void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; + bool any_mdev = false; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { - scnprintf(buf, len, "active=0"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) { + any_mdev = true; + break; + } + + if (!any_mdev) { + /* + * Legacy-only (or nothing): keep the historical shape - the + * campaign's parser (if any) sees the same text as before. + */ + if (!guest_ram_legacy.file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + n += scnprintf(buf + n, len - n, "active=1"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_legacy, ""); mutex_unlock(&guest_ram_lock); return; } - n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", - guest_ram_path, guest_ram_ranges_n); - for (i = 0; i < guest_ram_ranges_n && n < len; i++) - n += scnprintf(buf + n, len - n, - " gpa=0x%llx off=0x%llx size=0x%llx", - guest_ram_ranges[i].gpa_base, - guest_ram_ranges[i].file_offset, - guest_ram_ranges[i].size); + n += scnprintf(buf + n, len - n, "active=1"); + if (guest_ram_legacy.file) + gsp_phys_guest_ram_rec_state(buf, len, &n, &guest_ram_legacy, + " legacy"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS && n < len; i++) { + char label[64]; + + if (!guest_ram_mdevs[i].file) + continue; + scnprintf(label, sizeof(label), " mdev=%s", + (guest_ram_mdevs[i].vgpu && guest_ram_mdevs[i].vgpu->mdev) ? + dev_name(&guest_ram_mdevs[i].vgpu->mdev->dev) : "?"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_mdevs[i], label); + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); @@ -1452,12 +1711,20 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * - * Guards kept from the original path: the active-mdev walk + * Guards kept from the original path: the mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - - * the engine path must only run with a live guest session; no such session - * returns -ENODEV. The IOMMU-domain query remains as a documented fallback - * for a setup whose mdev has a real domain (it is inert with the - * emulated-IOMMU mdev used here). + * the engine path must only run with a live guest session; a vgpu given by + * the caller must itself be open, and no open session at all returns + * -ENODEV. The IOMMU-domain query remains as a documented fallback for a + * setup whose mdev has a real domain (it is inert with the emulated-IOMMU + * mdev used here). + * + * Per-mdev resolution (dual-guest goal 1.0): the caller names the owning + * vgpu (a captured forwarding request, a held release, ...). The vgpu's own + * record resolves the translation; the legacy record (configured by the + * historical top-level debugfs write) still serves the FIRST open mdev so + * single-guest scripts are unchanged. vgpu == NULL keeps the historical + * resolution: the first open mdev. * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file @@ -1467,13 +1734,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ -int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, u64 *hpa_out) { - struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; + struct nv_vgpu_device *first = NULL; + struct gsp_phys_guest_ram_rec *rec = NULL; struct iommu_domain *dom; phys_addr_t p; unsigned int i; + bool vgpu_gone = false; if (!hpa_out) return -EINVAL; @@ -1481,19 +1750,28 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; + first = d; break; } } + if (vgpu && atomic_read(&vgpu->vfio_dev_open_count) <= 0) + vgpu_gone = true; /* named session is gone: never retarget */ mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu_gone) + return -ENODEV; + if (!vgpu) + vgpu = first; /* historical resolution: first open mdev */ if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - for (i = 0; i < guest_ram_ranges_n; i++) { + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec && vgpu == first) + rec = &guest_ram_legacy; /* legacy file: first/only mdev */ + if (rec && rec->file) { + for (i = 0; i < rec->ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = - &guest_ram_ranges[i]; + &rec->ranges[i]; struct page *page; u64 file_off; @@ -1502,7 +1780,7 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) continue; file_off = r->file_offset + (gpa - r->gpa_base); - page = read_cache_page(guest_ram_file->f_mapping, + page = read_cache_page(rec->file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); @@ -1538,6 +1816,18 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) *hpa_out = (u64)p; return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa_vgpu); + +/* + * Historical entry point: resolve through the first open mdev (and that + * mdev's own record, else the legacy record). Used by the single-guest + * boot-ladder guest-GPA pin (test-only; see gsp_phys_guest_gpa_write) and + * kept as the compatibility wrapper for callers without a request context. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + return gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, hpa_out); +} EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 73b0822..e2be7a8 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -3,6 +3,8 @@ #include <linux/types.h> +struct nv_vgpu_device; + /* * Physical GPU ownership (real-GSP execution backend, bring-up rung 1). * Claim is disabled unless the gsp_phys_gpu module parameter is set. @@ -429,21 +431,50 @@ bool gsp_phys_rmcall_vas_shared_value(void); int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out); /* + * Per-mdev variant: translate the GPA for the named owning vgpu (a captured + * forwarding request, a held release). The vgpu's own guest-RAM record is + * authoritative; the legacy record still serves the FIRST open mdev so + * single-guest callers are unchanged. NULL keeps the historical resolution + * (first open mdev). A named vgpu whose session is closed returns -ENODEV + * rather than being retargeted at another guest. + */ +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, + u64 *hpa_out); + +/* * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One - * shared file per module cycle, up to four GPA->file-offset ranges keyed by + * record per guest session, up to four GPA->file-offset ranges keyed by * gpa_base. The first configure opens the file; the same path adds/replaces * ranges, a different path is refused -EBUSY, and the ranges are dropped and - * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The + * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release (or + * gsp_phys_guest_ram_vgpu_clear() when one mdev is removed). The + * gsp_phys_guest_ram_configure() entry point configures the legacy record + * (the historical top-level debugfs write: serves the first/only open mdev); + * gsp_phys_guest_ram_configure_vgpu() binds the record to one mdev. The * state accessor is lock-safe and writes "active=0" or the active path, * range count and per-range gpa/off/size into the caller's buffer. */ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, u64 file_offset, u64 size); +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size); void gsp_phys_guest_ram_clear(void); +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu); bool gsp_phys_guest_ram_active(void); void gsp_phys_guest_ram_state(char *buf, size_t len); /* + * Open-mdev helpers for the per-mdev debugfs selector and the single-guest + * guards: count the mdevs whose VFIO device is open, or resolve one open + * mdev by its device name (the mdev UUID). The returned device is not + * reference-counted; the caller must only use it while that mdev's fd is + * known to be open. + */ +int gsp_phys_open_mdev_count(void); +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name); + +/* * Domain shape of the physical (claimed) device, as the P6a engine path * needs it: the module programs raw host PAs into the CE page tables, so the * engine may only run when the physical device's DMA addresses ARE host PAs. diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 8318f56..2ffc8ac 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -2357,6 +2357,7 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, return; memset(&desc, 0, sizeof(desc)); + desc.vgpu = ctx->vgpu; desc.h_object = h_object; memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); @@ -3244,7 +3245,7 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, memcpy(&hObjOld, params + 8, 4); memset(params + 12, 0, 4); /* status = NV_OK */ - gsp_sema_service_channel_del(hObjOld); + gsp_sema_service_channel_del(ctx->vgpu, hObjOld); /* * Retire event records that refer to the object being freed. Posting an diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f8925df..c1eaf80 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -162,6 +162,10 @@ struct sema_map_entry { struct svc_channel { bool valid; + struct nv_vgpu_device *vgpu; /* owning mdev (guest session); the + * table is keyed by (vgpu, h_object) + * because two guests' stock allocators + * choose identical handles */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ @@ -449,12 +453,65 @@ static u32 svc_ring_n; * client spins). One clean boot always works, so make the next session look * like one: when the channel set empties, request a flush and let the next * rebuild start from the same empty state as a fresh boot. + * + * Dual-guest staging (goal 1.0): the discovery tables are still shared + * between guests (per-guest learned tables are the staged G4 remainder), so + * the flush is requested only when the channel set is globally empty. The + * active decision is made per guest (svc_guests below): one guest closing + * its last channel never flushes while another guest still has live + * channels - otherwise the departing guest would wipe the live guest's + * pins/maps and stall it. Single-guest behaviour is unchanged: the one + * guest's last-channel close is also the global empty transition. */ static u8 svc_sess_flush; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ +/* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ +#define SVC_MAX_GUESTS 8 +struct svc_guest_sess { + struct nv_vgpu_device *vgpu; /* NULL = free slot */ + bool active; /* a channel of this guest is + * registered until this guest's + * last channel closes */ + u64 last_build; /* per-guest svc_rebuild freshness */ +}; +static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; +static DEFINE_SPINLOCK(svc_guest_lock); + +/* + * Find (or, with create, allocate) the session record for one mdev. The + * records are pointer-keyed; a removed mdev's record is freed by + * gsp_sema_service_vgpu_cleanup() so the pointer can never match a newly + * allocated device. + */ +static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu, + bool create) +{ + struct svc_guest_sess *sess = NULL; + unsigned long flags; + int i; + + if (!vgpu) + return NULL; + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_guests[i].vgpu == vgpu) { + sess = &svc_guests[i]; + break; + } + if (create && !sess && !svc_guests[i].vgpu) + sess = &svc_guests[i]; + } + if (sess && create && !sess->vgpu) { + memset(sess, 0, sizeof(*sess)); + sess->vgpu = vgpu; + } + spin_unlock_irqrestore(&svc_guest_lock, flags); + return sess; +} + /* * CeUtils scrubber channel (the guest PMA's internal CE channel created at * driver init). When the PMA must hand out scrubbed pages it waits on the @@ -476,6 +533,7 @@ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session st * block count (so the scrub wait completes). Idempotent, forward-only. */ struct svc_scrub { + struct nv_vgpu_device *vgpu; /* owning mdev (NULL = free slot) */ bool userd_ok; /* a 4096-entry channel was registered */ u32 h_object; /* handle of the registered scrubber channel */ u64 userd_pa; @@ -492,7 +550,34 @@ struct svc_scrub { u32 pa_gen; /* scanner registration generation (logged) */ u32 log_budget; }; -static struct svc_scrub svc_scrub; +/* + * One scrubber record per guest: every guest's PMA has its own 4096-entry + * scrubber channel, and the authored words must be written through the + * owning guest's address space (writing guest A's PA through guest B's VFIO + * device would corrupt B). Keyed by vgpu pointer, freed by + * gsp_sema_service_vgpu_cleanup(). + */ +static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS]; + +static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create) +{ + struct svc_scrub *s = NULL; + int i; + + if (!vgpu) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS; i++) { + if (svc_scrubs[i].vgpu == vgpu) + return &svc_scrubs[i]; + if (create && !s && !svc_scrubs[i].vgpu) + s = &svc_scrubs[i]; + } + if (s) { + memset(s, 0, sizeof(*s)); + s->vgpu = vgpu; + } + return s; +} static u8 svc_pb_passes; static u64 svc_pb_last; /* @@ -1066,7 +1151,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu && + svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } @@ -1079,7 +1165,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) u64 keep_release_pa = 0; u8 keep_gr_index = 0, keep_ce_index = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); - bool same = ch->valid && ch->h_object == desc->h_object; + bool same = ch->valid && ch->vgpu == desc->vgpu && + ch->h_object == desc->h_object; /* * A re-registration of the same channel must not lose the @@ -1102,18 +1189,22 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); /* - * Rank among the live CE channels. The RM hands out object - * handles in creation order and the guest's UVM allocates one - * push buffer, ring and tracking semaphore per channel in that - * same order, so the rank is what ties a channel to its ring - * offset and its push-buffer base (see svc_ring_autopin / - * svc_pb_pair). + * Rank among the live CE channels OF THIS GUEST. The RM hands + * out object handles in creation order and the guest's UVM + * allocates one push buffer, ring and tracking semaphore per + * channel in that same order, so the rank is what ties a + * channel to its ring offset and its push-buffer base (see + * svc_ring_autopin / svc_pb_pair). The rank must not count the + * other guest's channels: their handles are independent and + * their allocators interleave. */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; @@ -1121,13 +1212,16 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && !svc.chan[k].engine_type && + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->gr_index = rank; } ch->valid = true; svc_sess_active = 1; + ch->vgpu = desc->vgpu; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; @@ -1145,6 +1239,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } + /* Per-guest session flag: this guest has a live channel. */ + { + struct svc_guest_sess *sess = + svc_guest_get(desc->vgpu, true); + + if (sess) + sess->active = true; + } } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -1169,25 +1271,30 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && desc->userd_pa) { + struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); + + if (!s) + goto scrub_done; /* table full: max guests */ /* Re-seed the counting on a fresh scrubber object. */ - if (!svc_scrub.userd_ok || - svc_scrub.h_object != desc->h_object || - svc_scrub.gpfifo_va != desc->gpfifo_va) { - svc_scrub.seeded = false; - svc_scrub.submitted = 0; + if (!s->userd_ok || + s->h_object != desc->h_object || + s->gpfifo_va != desc->gpfifo_va) { + s->seeded = false; + s->submitted = 0; } - svc_scrub.userd_ok = true; - svc_scrub.h_object = desc->h_object; - svc_scrub.userd_pa = desc->userd_pa; - svc_scrub.userd_as = desc->userd_as ? desc->userd_as - : SVC_APERTURE_SYS; - svc_scrub.gpfifo_va = desc->gpfifo_va; - svc_scrub.entries = desc->gpfifo_entries; + s->userd_ok = true; + s->h_object = desc->h_object; + s->userd_pa = desc->userd_pa; + s->userd_as = desc->userd_as ? desc->userd_as + : SVC_APERTURE_SYS; + s->gpfifo_va = desc->gpfifo_va; + s->entries = desc->gpfifo_entries; pr_info("sema-svc: scrubber channel 0x%x detected (gpfifo=0x%llx entries=%u userd=0x%llx as=%u)\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, - svc_scrub.userd_as); + s->userd_as); } +scrub_done: pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, @@ -1199,7 +1306,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } /* P6b wave-1.5 hold state is defined with the forwarding section below. */ -static void svc_fwd_hold_abort(u32 h_object); +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object); /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. @@ -1218,7 +1325,7 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 entry_idx); static void svc_opdef_reset_all(void); -void gsp_sema_service_channel_del(u32 h_object) +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { unsigned long flags; int i; @@ -1228,7 +1335,8 @@ void gsp_sema_service_channel_del(u32 h_object) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); @@ -1238,19 +1346,27 @@ void gsp_sema_service_channel_del(u32 h_object) /* * ISSUE-0002: when the last channel closes, the next client is a new * session in a new UVM VA space; its discovery state must not inherit - * this session's map/pins/page-directory latch. Request the flush - * here; it executes at the next rebuild (the only writer of that - * state) just like a fresh boot would start empty. Only the transition - * from "a session existed" to "empty" may request it: redundant - * deletes for handles that never registered would otherwise re-flush - * (and wipe) a session that started after the real one ended. + * this session's map/pins/page-directory latch. The active decision is + * per guest (svc_guests) and the flush request is raised only when NO + * guest has a live channel left: the discovery tables are still shared + * between guests (staged G4), so flushing while another guest is live + * would wipe that guest's pins and stall it. One guest: the last-channel + * close IS the global empty transition, so behaviour is unchanged. */ if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false, own = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) - break; + if (!svc.chan[i].valid) + continue; + any = true; + if (svc.chan[i].vgpu == vgpu) + own = true; } - if (i == SVC_MAX_CHANNELS) { + if (sess && !own) + sess->active = false; + if (!any) { svc_sess_active = 0; svc_sess_flush = 1; } @@ -1265,9 +1381,10 @@ void gsp_sema_service_channel_del(u32 h_object) * P6b wave 1.5: a deleted channel's held releases must not wait out * the timeout; mark the hold aborted so the pump applies them (or * drops them if the snapshot already lost the channel) on its next - * pass. + * pass. The hold is matched per (vgpu, handle): the other guest's + * identical handle must not abort this guest's hold. */ - svc_fwd_hold_abort(h_object); + svc_fwd_hold_abort(vgpu, h_object); } /* @@ -5078,7 +5195,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { /* * The release targets the slot the push itself named * (release_pa learned from the release VA); the @@ -5150,6 +5268,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && + svc.chan[i].vgpu == vgpu && svc.chan[i].h_object == h) { if (payload > svc.chan[i].pending_rel_pay) { @@ -5229,7 +5348,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * harmless for an absolute release (its guard drops it), but an increment * would advance the slot twice. */ -static void svc_counter_pending_clear(u32 h) +static void svc_counter_pending_clear(struct nv_vgpu_device *vgpu, u32 h) { unsigned long flags; int i; @@ -5238,7 +5357,8 @@ static void svc_counter_pending_clear(u32 h) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { svc.chan[i].pending_rel_pay = 0; svc.chan[i].pending_rel_inc = 0; break; @@ -5272,13 +5392,13 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, if (payload > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); return; } } else if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (cur > ch->counter_last) ch->counter_last = cur; return; @@ -5289,7 +5409,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, return; ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; - svc_counter_pending_clear(ch->h_object); + svc_counter_pending_clear(ch->vgpu, ch->h_object); if (nv > ch->counter_last) ch->counter_last = nv; svc.releases_applied++; @@ -5428,6 +5548,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = ipa & ~0x3ULL; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = ch->counter_pa; @@ -5476,6 +5597,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, spin_lock_irqsave(&svc.chan_lock, flags); for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object != ch->h_object && svc.chan[j].counter_pa == pa) { collision = true; @@ -5501,6 +5623,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = pa; break; @@ -5560,7 +5683,8 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].h_object == h) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h) { if (svc.chan[i].release_pa != (pa & ~0x3ULL)) { svc.chan[i].release_pa = pa & ~0x3ULL; svc.chan[i].relearn_n++; @@ -7694,7 +7818,8 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) u32 slot, k; bool have = false; - if (!svc.chan[i].valid || !svc.chan[i].engine_type) + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + !svc.chan[i].engine_type) continue; ce = true; if (svc.chan[i].ce_index >= 32) @@ -8975,7 +9100,8 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) int pap = 0; u32 r = c->gr_index; - if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS) + if (!c->valid || c->vgpu != vgpu || c->engine_type || + r >= SVC_GR_CONTENT_RANKS) continue; if (test_bit(r, &svc_gr_ring_done)) continue; @@ -9316,7 +9442,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, */ spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) - if (svc.chan[k].valid && svc.chan[k].engine_type && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && + svc.chan[k].engine_type && svc.chan[k].ce_index + 1 > max_ce) max_ce = svc.chan[k].ce_index + 1; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -10792,7 +10919,7 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, } if (kind != SVC_APERTURE_SYS) return -EINVAL; - ret = gsp_phys_gpa_to_hpa(page, hpa); + ret = gsp_phys_gpa_to_hpa_vgpu(vgpu, page, hpa); if (ret) return ret; *out_off = page_off; @@ -10961,9 +11088,10 @@ struct svc_fwd_hold { static struct svc_fwd_hold svc_fwd_hold; -static void svc_fwd_hold_abort(u32 h_object) +static void svc_fwd_hold_abort(struct nv_vgpu_device *vgpu, u32 h_object) { if (READ_ONCE(svc_fwd_hold.active) && + svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == h_object) WRITE_ONCE(svc_fwd_hold.aborted, true); } @@ -11334,18 +11462,22 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, } } -/* A slot whose channel is gone from this pass's snapshot: drop it. */ -static void svc_opdef_drop_gone(const struct svc_channel *local, int n) +/* A slot of THIS pump's guest whose channel is gone from the pass snapshot: + * drop it. Slots of other guests are never touched (their channels are not in + * this snapshot by construction). */ +static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, + const struct svc_channel *local, int n) { int i, k; for (i = 0; i < SVC_OPDEF_SLOTS; i++) { struct svc_opdef_ent *e = &svc_opdef[i]; - if (!e->active) + if (!e->active || e->vgpu != vgpu) continue; for (k = 0; k < n; k++) - if (local[k].h_object == e->h_object) + if (local[k].vgpu == e->vgpu && + local[k].h_object == e->h_object) break; if (k == n) svc_opdef_clear(e); @@ -11787,6 +11919,31 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, } /* + * Is this mdev still on the driver list with an open VFIO device? Decides + * whether a hold that names it can still be resolved through it: a removed + * device must never be dereferenced (its struct can be freed before the next + * pump pass would see the hold). Pointer comparisons only - safe to call + * with a possibly-stale pointer. + */ +static bool svc_vgpu_open(const struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_device *d; + bool open = false; + + if (!vgpu) + return false; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (d == vgpu && atomic_read(&d->vfio_dev_open_count) > 0) { + open = true; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return open; +} + +/* * Channel vanished while a hold was active. Any resolved hold (completed, * aborted, failed or expired) applies the recorded releases through a shim * channel - holds exist only for CE channels and the vanished channel object @@ -11806,6 +11963,18 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + /* + * The owning mdev was removed mid-hold (the mdev cleanup path + * already released its session state): its guest can never + * observe the releases and dereferencing the device would be a + * use-after-free. Drop the hold; the request slot frees for the + * next guest. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } if (!done && !aborted && !svc_fwd_hold_expired()) return; /* @@ -11858,6 +12027,15 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) if (!svc_fwd_hold.active) return; + /* + * A hold whose owning mdev is already gone must never be dereferenced: + * drop it (the guest cannot observe the releases anyway). + */ + if (svc_fwd_hold.vgpu && !svc_vgpu_open(svc_fwd_hold.vgpu)) { + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + return; + } done = smp_load_acquire(&svc_fwd_req.done); /* * P6b wave 2: the backend is going away; a skipped candidate whose @@ -13471,7 +13649,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -13507,7 +13685,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * pass's snapshot belongs to a vanished channel; its waiter is gone, * so drop the set instead of resolving it later. */ - svc_opdef_drop_gone(local, n); + svc_opdef_drop_gone(vgpu, local, n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; @@ -15020,7 +15198,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { - if (svc.chan[k].valid && + if (svc.chan[k].valid && svc.chan[k].vgpu == vgpu && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; @@ -15334,7 +15512,7 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) - if (svc.chan[i].valid) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -15725,9 +15903,18 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish * payload the guest's scrub wait polls; hsem_pa is the channel's host * semaphore (channelWaitForFreeEntry polls it). Plain state update. + * + * Dual-guest: the words are written through a specific guest's address space + * (see svc_scrub_pass), so the registration binds to the FIRST open mdev - + * the same single-guest convention the scanner tooling follows (it tracks + * one client). Refused with no open mdev rather than parked globally. */ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) { + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct svc_scrub *s; + if (!svc.task) return -ENODEV; /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ @@ -15736,12 +15923,28 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) return -EINVAL; - svc_scrub.fin_pa = fin_pa; - svc_scrub.hsem_pa = hsem_pa; - svc_scrub.pa_ok = true; - svc_scrub.pa_gen++; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) { + pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n"); + return -ENODEV; + } + s = svc_scrub_get(vgpu, true); + if (!s) + return -ENOSPC; + + s->fin_pa = fin_pa; + s->hsem_pa = hsem_pa; + s->pa_ok = true; + s->pa_gen++; pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", - fin_pa, hsem_pa, svc_scrub.pa_gen); + fin_pa, hsem_pa, s->pa_gen); return 0; } @@ -15769,11 +15972,11 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) */ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) { - struct svc_scrub *s = &svc_scrub; + struct svc_scrub *s = svc_scrub_get(vgpu, false); u32 put = 0; u64 delta; - if (!s->userd_ok || !s->pa_ok || !s->entries) + if (!s || !s->userd_ok || !s->pa_ok || !s->entries) return; if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { @@ -15830,7 +16033,9 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) static int svc_thread(void *arg) { + struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; struct nv_vgpu_device *vgpu; + int n_vgpus, g; pr_info("sema-svc: thread started\n"); @@ -15850,130 +16055,174 @@ static int svc_thread(void *arg) complete_all(&svc_sess_flush_done); } - vgpu = NULL; + /* + * Service EVERY open mdev each pass, in list order. With one + * mdev this is exactly the historical single-vgpu pass; with + * two, both guests get a pump turn per iteration (hardware + * forwarding stays serialized by the single-inflight slot; see + * the forwarding section). The list walk is the historical + * bounded scan under nv_vgpu_driver.lock. + */ + n_vgpus = 0; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { - if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; - break; - } + if (atomic_read(&d->vfio_dev_open_count) > 0 && + n_vgpus < SVC_MAX_GUESTS) + vgpus[n_vgpus++] = d; } } mutex_unlock(&nv_vgpu_driver.lock); - if (!vgpu) { + if (!n_vgpus) { msleep(500); continue; } - svc_handle_probe(vgpu); - svc_handle_walk_probe(vgpu); + /* Host-injection queue drain: once per pass, not per guest. */ svc_handle_map_add(); - svc_handle_chan_dump(vgpu); - svc_scrub_pass(vgpu); - if (svc_sess_flush || !svc.direct_base || - time_after(jiffies, - (unsigned long)(svc.last_build + 10 * HZ))) - svc_rebuild(vgpu); + for (g = 0; g < n_vgpus; g++) { + struct svc_guest_sess *sess; + u64 done_before; - /* - * Channel-driven execution: notice submissions as they are made - * and run the pushed work immediately, so the guest's UVM - * progress waits complete in milliseconds rather than in the - * time it takes the fallback scan to find the push. A doorbell - * token means a channel was just scheduled; run everything now. - */ - { - u64 done_before = svc.gpfifo_entries_done; + vgpu = vgpus[g]; + svc_handle_probe(vgpu); + svc_handle_walk_probe(vgpu); + svc_handle_chan_dump(vgpu); + svc_scrub_pass(vgpu); + + /* + * Per-guest rebuild freshness: the discovery map is + * still shared between guests (staged G4), but each + * guest's RAM must be scanned on its own clock, or the + * first guest's 10 s tick would starve the second + * guest of rebuilds. The freshness stamp is the one + * svc_rebuild() publishes (svc.last_build). + */ + sess = svc_guest_get(vgpu, true); + if (svc_sess_flush || !svc.direct_base || + !sess || + time_after(jiffies, + (unsigned long)(sess->last_build + + 10 * HZ))) { + svc_rebuild(vgpu); + if (sess) + sess->last_build = svc.last_build; + } + /* + * Channel-driven execution: notice submissions as they + * are made and run the pushed work immediately, so the + * guest's UVM progress waits complete in milliseconds + * rather than in the time it takes the fallback scan + * to find the push. A doorbell token means a channel + * was just scheduled; run everything now. + */ + done_before = svc.gpfifo_entries_done; svc.doorbell_pending = false; svc_pump_channels(vgpu); - pumped = (svc.gpfifo_entries_done != done_before); - } + if (svc.gpfifo_entries_done != done_before) + pumped = true; - /* - * Host CUDA bridge: marshal one queued launch into a daemon - * job and resolve completions. Inert until a launch is seen. - */ - gsp_cuda_bridge_service(vgpu); + /* + * Host CUDA bridge: marshal one queued launch into a + * daemon job and resolve completions. Inert until a + * launch is seen. + */ + gsp_cuda_bridge_service(vgpu); - /* - * Releases whose target VA was not translatable when the scan - * saw them: retry every pass, so a pin or a learned pair applies - * them immediately instead of on the next full sweep. - */ - svc_apply_pending(vgpu); + /* + * Releases whose target VA was not translatable when + * the scan saw them: retry every pass, so a pin or a + * learned pair applies them immediately instead of on + * the next full sweep. + */ + svc_apply_pending(vgpu); - /* - * A fresh VA->PA injection changes which releases are - * applicable: run the fast sliced sweep while the forced window - * is open (the pump's stall path also uses it). - */ - if (time_before(jiffies, (unsigned long)svc.fast_force_until)) - svc_apply_sweep_slice(vgpu); + /* + * A fresh VA->PA injection changes which releases are + * applicable: run the fast sliced sweep while the + * forced window is open (the pump's stall path also + * uses it). + */ + if (time_before(jiffies, + (unsigned long)svc.fast_force_until)) + svc_apply_sweep_slice(vgpu); + } /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip - * the discovery/sweep passes, which otherwise dominate the pass - * period (the pump cadence is what the client's context init - * deadline sees). Scans resume the first pass without progress. + * the discovery/sweep passes, which otherwise dominate the + * pass period (the pump cadence is what the client's context + * init deadline sees). Scans resume the first pass without + * progress. (With two guests, any guest's progress keeps the + * loop tight; the next iteration pumps both.) */ if (pumped) { msleep(1); continue; } - /* - * Push-buffer base discovery: a chunked walk of guest RAM - * finds the pages whose first push names a channel, so a - * channel that appears with work already waiting is pinned and - * pumped without waiting for the rolling sweep. - */ - svc_pb_discover(vgpu); + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; - /* - * Completion hint: re-post the events for a few seconds after a - * completion, so a client that starts waiting after the - * edge-triggered wake-up still observes it. - */ - if (time_before(jiffies, (unsigned long)svc.post_hint_until) && - time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { - int rc; + /* + * Push-buffer base discovery: a chunked walk of guest + * RAM finds the pages whose first push names a + * channel, so a channel that appears with work + * already waiting is pinned and pumped without + * waiting for the rolling sweep. + */ + svc_pb_discover(vgpu); - svc.last_hint = jiffies; - rc = gsp_rm_post_all_events(vgpu); - if (gsp_sema_dbg) - pr_info_ratelimited("sema-svc: completion hint rc=%d\n", - rc); - } + /* + * Completion hint: re-post the events for a few + * seconds after a completion, so a client that starts + * waiting after the edge-triggered wake-up still + * observes it. + */ + if (time_before(jiffies, + (unsigned long)svc.post_hint_until) && + time_after(jiffies, + (unsigned long)svc.last_hint + HZ / 2)) { + int rc; + + svc.last_hint = jiffies; + rc = gsp_rm_post_all_events(vgpu); + if (gsp_sema_dbg) + pr_info_ratelimited("sema-svc: completion hint rc=%d\n", + rc); + } - /* - * Fast path for the pushes themselves: every mapped page is a - * candidate pushbuffer page, and there are far fewer of those - * than there are pages of guest RAM. - */ - if (svc.direct_base && svc.map_n) - svc_apply_mapped(vgpu); + /* + * Fast path for the pushes themselves: every mapped + * page is a candidate pushbuffer page, and there are + * far fewer of those than there are pages of guest + * RAM. + */ + if (svc.direct_base && svc.map_n) + svc_apply_mapped(vgpu); - /* - * Rolling RAM scan: the pushbuffers of some clients are not - * covered by the VA map, so sweep guest RAM in bounded windows - * in parallel with the targeted pass. Work per pass is fixed, - * which keeps the emulation responsive while a full sweep - * completes every few seconds. - */ - svc_apply_pushes(vgpu); + /* + * Rolling RAM scan: the pushbuffers of some clients + * are not covered by the VA map, so sweep guest RAM in + * bounded windows in parallel with the targeted pass. + * Work per pass is fixed, which keeps the emulation + * responsive while a full sweep completes every few + * seconds. + */ + svc_apply_pushes(vgpu); - /* - * Deferred readability probes for faked CE pushes (observation - * only; see svc_retry_poll). - */ - svc_retry_poll(vgpu); + /* + * Deferred readability probes for faked CE pushes + * (observation only; see svc_retry_poll). + */ + svc_retry_poll(vgpu); + } msleep(10); } @@ -16096,6 +16345,97 @@ void gsp_sema_service_stop(void) svc_vas_free_all(); } +/* + * Release every piece of service state owned by one mdev (dual-guest goal + * 1.0). Called from the mdev cleanup path after that mdev's emulation thread + * stopped and before its struct can be freed: + * - quiesce forwarding so no in-flight work item can still be reading the + * request's vgpu, and abort a release hold that names it; + * - drop any hold that survives (a request that never queued leaves no work + * item to clear it) BEFORE the device can go away - resolution would + * dereference it; + * - drop the guest's channels; request the global session flush only when no + * guest has live channels left (the discovery tables are still shared; + * staged G4); + * - drop the guest's held-operand release slots (its waiters are gone); + * - free the per-guest session and scrubber records so the vgpu pointer can + * never match a newly allocated device. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + if (!vgpu) + return; + + /* + * The forward work item runs on its own kthread and may be using the + * request's vgpu; quiesce waits for it (bounded by the submit's + * 200 ms poll cap) and aborts an active hold. Resume afterwards so the + * other guests keep forwarding. + */ + gsp_sema_service_work_quiesce(); + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) + WRITE_ONCE(svc_fwd_hold.aborted, true); + gsp_sema_service_work_resume(); + + if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.vgpu == vgpu) { + /* + * Pure state clear (never a resolution through the vanishing + * device): the releases belong to a guest that is gone. + */ + svc.fwd_hold_canceled++; + svc_fwd_hold_clear(); + } + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + + if (!ch->valid || ch->vgpu != vgpu) + continue; + pr_info("sema-svc: channel 0x%x dropped (mdev cleanup; submissions=%llu)\n", + ch->h_object, ch->submissions); + memset(ch, 0, sizeof(*ch)); + } + if (svc_sess_active) { + struct svc_guest_sess *sess = svc_guest_get(vgpu, false); + bool any = false; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid) { + any = true; + break; + } + if (sess) + sess->active = false; + if (!any) { + svc_sess_active = 0; + svc_sess_flush = 1; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + /* Held-operand release slots of this guest cannot be delivered. */ + for (i = 0; i < SVC_OPDEF_SLOTS; i++) + if (svc_opdef[i].active && svc_opdef[i].vgpu == vgpu) + svc_opdef_clear(&svc_opdef[i]); + + /* Free the per-guest session and scrubber records. */ + spin_lock_irqsave(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_guests[i].vgpu == vgpu) + memset(&svc_guests[i], 0, sizeof(svc_guests[i])); + spin_unlock_irqrestore(&svc_guest_lock, flags); + for (i = 0; i < SVC_MAX_GUESTS; i++) + if (svc_scrubs[i].vgpu == vgpu) + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + + if (svc_sess_flush && svc.task) + wake_up_process(svc.task); +} + /* ------------------------------------------------------------------ */ /* host CUDA bridge exports (gsp_cuda_bridge.c) */ /* ------------------------------------------------------------------ */ diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 6822bd5..c1dd763 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -18,8 +18,16 @@ * * All addresses are guest-physical except gpfifo_va, which is the GPU VA of * the channel's GPFIFO ring (translated through the discovered VA->PA map). + * + * vgpu names the owning mdev (guest session). Two guests' stock allocators + * choose the same object handles, so the channel table is keyed by + * (vgpu, h_object): a registration can never overwrite another guest's slot, + * and ranks/session bookkeeping are computed per guest. */ +struct nv_vgpu_device; + struct sema_channel_desc { + struct nv_vgpu_device *vgpu; /* owning mdev (guest session) */ u32 h_object; u32 engine_type; u32 cid; /* RM channel id (the doorbell token vector) */ @@ -33,7 +41,17 @@ struct sema_channel_desc { }; void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); -void gsp_sema_service_channel_del(u32 h_object); +void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object); + +/* + * Release every piece of service state that belongs to one mdev before its + * guest session disappears: drop the mdev's channels (per-vgpu session end), + * clear a release hold that names it (never dereference a removed device), + * and free its per-guest session record. Called from the mdev cleanup path + * after the device's emulation thread stopped and before the mdev struct can + * be freed. No-op for a vgpu that never registered a channel. + */ +void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu); /* * Register the guest PMA scrubber's channel-buffer words (the CeUtils diff --git a/kernel/main.c b/kernel/main.c index d0308c4..257638a 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -1740,6 +1740,16 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work) gsp_emu_cleanup(vgpu); /* + * Dual-guest (goal 1.0): release everything the sema service still holds + * for this mdev (channel entries, session/scrubber records, a release + * hold that names it - quiescing any in-flight forwarding work) and drop + * its per-mdev guest-RAM record, BEFORE the vgpu struct can be freed: a + * stale pointer in the service's tables must never match a later device. + */ + gsp_sema_service_vgpu_cleanup(vgpu); + gsp_phys_guest_ram_vgpu_clear(vgpu); + + /* * Last-reference backend teardown: gsp_emu_cleanup() above has stopped * this mdev's emulation thread, so no forward can be in flight when the * claim is released. The backend session stays up until the last mdev |