diff options
Diffstat (limited to 'kernel/gsp_phys_gpu.c')
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 432 |
1 files changed, 361 insertions, 71 deletions
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 9b7235b..642a743 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1256,16 +1256,28 @@ EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The - * ranges map guest GPA windows to file offsets; up to four windows of one - * file per module cycle. + * ranges map guest GPA windows to file offsets; up to four windows per file. + * + * Per-mdev records (dual-guest goal 1.0): a guest RAM backing file belongs + * to ONE guest session, so the record moves with the mdev. Slot 0 is the + * legacy record (vgpu == NULL): it is what the historical top-level + * gsp-phys-guest-ram debugfs write configures and it serves the first/only + * open mdev exactly as the pre-dual-guest code did (one guest: byte-identical + * behaviour). A per-mdev record is configured with the mdev UUID as an extra + * first token on the same debugfs file and serves only that mdev; the + * record is released when the mdev is removed (gsp_phys_guest_ram_vgpu_clear) + * or when the physical cycle ends (gsp_phys_guest_ram_clear). * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. - * One file per module cycle: a second path is refused -EBUSY because a fresh - * cycle reloads the module; more ranges of the same file may be added. + * Within one record the historical rule is unchanged: the first configure + * opens the file, a different path is refused -EBUSY, more ranges of the + * same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 +/* Per-mdev guest-RAM records (a handful of guests; 4 is far beyond 1.0). */ +#define GSP_PHYS_GUEST_RAM_MDEVS 4 struct gsp_phys_guest_ram_range { u64 gpa_base; @@ -1273,35 +1285,121 @@ struct gsp_phys_guest_ram_range { u64 size; }; -static struct file *guest_ram_file; -static char guest_ram_path[128]; -static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; -static unsigned int guest_ram_ranges_n; +struct gsp_phys_guest_ram_rec { + struct nv_vgpu_device *vgpu; /* NULL = legacy (first/only open mdev) */ + struct file *file; + char path[128]; + struct gsp_phys_guest_ram_range ranges[GSP_PHYS_GUEST_RAM_RANGES]; + unsigned int ranges_n; +}; + +static struct gsp_phys_guest_ram_rec guest_ram_legacy; +static struct gsp_phys_guest_ram_rec guest_ram_mdevs[GSP_PHYS_GUEST_RAM_MDEVS]; static DEFINE_MUTEX(guest_ram_lock); +/* Caller holds guest_ram_lock. NULL when this vgpu has no bound record. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_find_locked(const struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return NULL; + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].vgpu == vgpu) + return &guest_ram_mdevs[i]; + return NULL; +} + +/* Caller holds guest_ram_lock. NULL when every per-mdev slot is taken. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_alloc_locked(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (!guest_ram_mdevs[i].vgpu) { + guest_ram_mdevs[i].vgpu = vgpu; + return &guest_ram_mdevs[i]; + } + return NULL; +} + /* - * Configure the guest RAM backing file and add (or replace, keyed by - * gpa_base) one GPA window. The first call opens the file read-only and - * fixes the path for this module cycle; later calls may only add ranges of - * that same file. The file must be a regular file and every range must fit - * inside its current size, so a translation can never index a page beyond - * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges - * in use, or the file/argument error. + * Number of mdevs whose VFIO device is currently open. Shared by the + * per-mdev debugfs selector (which mdev does this UUID name?) and the + * single-guest guard on the legacy test-only paths. */ -int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, - u64 file_offset, u64 size) +int gsp_phys_open_mdev_count(void) +{ + struct nv_vgpu_device *d; + int n = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) + n++; + } + mutex_unlock(&nv_vgpu_driver.lock); + return n; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_count); + +/* + * Resolve an OPEN mdev by its device name (the mdev UUID, e.g. + * "b6acdc5e-..."). Returns the device or NULL. The caller must use the + * result only while that mdev's VFIO fd is known to be open (the same + * lifetime rule as the first-open resolution below): the driver list is + * walked under nv_vgpu_driver.lock, but the returned pointer is not + * reference-counted. + */ +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name) +{ + struct nv_vgpu_device *d, *found = NULL; + + if (!name || !*name) + return NULL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0 && d->mdev && + !strcmp(dev_name(&d->mdev->dev), name)) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return found; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_by_name); + +/* + * Shared configure body for one record: open the record's file (first call) + * and add (or replace, keyed by gpa_base) one GPA window. The first call + * opens the file read-only and fixes the path for the record's lifetime; + * later calls may only add ranges of that same file. The file must be a + * regular file and every range must fit inside its current size, so a + * translation can never index a page beyond EOF. Returns 0, -EBUSY for a + * different path, -ENOSPC with all four ranges in use, or the + * file/argument error. `who` is "" for the legacy record and + * " for mdev <uuid>" for a per-mdev record; the legacy strings stay + * byte-identical. + */ +static int gsp_phys_guest_ram_rec_configure(struct gsp_phys_guest_ram_rec *rec, + const char *path, u64 gpa_base, + u64 file_offset, u64 size, + const char *who) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; - if (!path || !*path) + if (!rec || !path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { + if (!rec->file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond @@ -1331,35 +1429,38 @@ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, ret = -ERANGE; goto out_fput; } - guest_ram_file = f; - strscpy(guest_ram_path, path, sizeof(guest_ram_path)); - } else if (strcmp(guest_ram_path, path)) { - pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", - guest_ram_path, path); + rec->file = f; + strscpy(rec->path, path, sizeof(rec->path)); + } else if (strcmp(rec->path, path)) { + if (who[0]) + pr_err("vgsp: guest RAM already configured from %s; one file per guest session (reload or mdev remove to switch); refusing %s\n", + rec->path, path); + else + pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + rec->path, path); ret = -EBUSY; goto out; } - for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { - if (guest_ram_ranges[i].gpa_base == gpa_base) { - range = &guest_ram_ranges[i]; + for (i = 0, range = NULL; i < rec->ranges_n; i++) { + if (rec->ranges[i].gpa_base == gpa_base) { + range = &rec->ranges[i]; break; } } if (!range) { - if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + if (rec->ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } - range = &guest_ram_ranges[guest_ram_ranges_n++]; + range = &rec->ranges[rec->ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; - pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", - guest_ram_path, gpa_base, file_offset, size, - guest_ram_ranges_n); + pr_info("vgsp: guest RAM configured%s: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + who, rec->path, gpa_base, file_offset, size, rec->ranges_n); out: mutex_unlock(&guest_ram_lock); @@ -1369,35 +1470,148 @@ out_fput: fput(f); goto out; } + +/* + * Configure the legacy guest RAM record (the historical top-level + * gsp-phys-guest-ram write). Serves the first/only open mdev exactly as + * before the per-mdev records existed. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + return gsp_phys_guest_ram_rec_configure(&guest_ram_legacy, path, + gpa_base, file_offset, size, + ""); +} EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* - * Close the configured guest RAM file (if any) and drop every range. Called - * from the physical-GPU release path so module unload and a backend release - * end the file reference; a fresh cycle reconfigures from debugfs. + * Configure the guest RAM record bound to one mdev (per-mdev selector on the + * debugfs write). A record for this vgpu is created on first use and closed + * by gsp_phys_guest_ram_vgpu_clear() when the mdev is removed. A failed + * first configure leaves no bound record behind. + */ +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_rec *rec; + char who[80]; + const char *name = (vgpu && vgpu->mdev) ? + dev_name(&vgpu->mdev->dev) : "?"; + bool fresh = false; + int ret; + + if (!vgpu) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec) { + rec = gsp_phys_guest_ram_rec_alloc_locked(vgpu); + if (!rec) { + mutex_unlock(&guest_ram_lock); + pr_err("vgsp: guest RAM: no free per-mdev record slot (max %u); refusing mdev %s\n", + GSP_PHYS_GUEST_RAM_MDEVS, name); + return -ENOSPC; + } + fresh = true; + } + mutex_unlock(&guest_ram_lock); + + scnprintf(who, sizeof(who), " for mdev %s", name); + ret = gsp_phys_guest_ram_rec_configure(rec, path, gpa_base, + file_offset, size, who); + + /* + * A fresh record whose first configure failed must not hold the slot + * (the vgpu pointer would otherwise be reserved with no file and the + * next attempt could not create a clean record). + */ + if (ret && fresh) { + mutex_lock(&guest_ram_lock); + if (rec->file) + ret = 0; /* raced with another configure; keep it */ + else + rec->vgpu = NULL; + mutex_unlock(&guest_ram_lock); + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure_vgpu); + +/* Lock held: close one record's file and drop its ranges. */ +static void gsp_phys_guest_ram_rec_close(struct gsp_phys_guest_ram_rec *rec, + const char *release_log) +{ + if (rec->file) { + pr_info("%s: path=%s ranges=%u\n", release_log, + rec->path, rec->ranges_n); + fput(rec->file); + rec->file = NULL; + } + rec->path[0] = '\0'; + memset(rec->ranges, 0, sizeof(rec->ranges)); + rec->ranges_n = 0; +} + +/* + * Close every configured guest RAM record (legacy and per-mdev) and drop + * every range. Called from the physical-GPU release path so module unload + * and a backend release end the file references; a fresh cycle reconfigures + * from debugfs. The legacy release log line is unchanged. */ void gsp_phys_guest_ram_clear(void) { + unsigned int i; + mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", - guest_ram_path, guest_ram_ranges_n); - fput(guest_ram_file); - guest_ram_file = NULL; - } - guest_ram_path[0] = '\0'; - memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); - guest_ram_ranges_n = 0; + gsp_phys_guest_ram_rec_close(&guest_ram_legacy, + "vgsp: guest RAM released"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); +/* + * Release the guest RAM record bound to one mdev. Called from the mdev + * cleanup path so the file reference and the vgpu pointer never outlive the + * session (a removed mdev's translations must stop, and the record slot must + * free up for a restarted guest). + */ +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return; + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + if (guest_ram_mdevs[i].vgpu == vgpu) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } + } + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_vgpu_clear); + bool gsp_phys_guest_ram_active(void) { - bool active; + bool active = false; + unsigned int i; mutex_lock(&guest_ram_lock); - active = guest_ram_file != NULL; + if (guest_ram_legacy.file) + active = true; + for (i = 0; !active && i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) + active = true; mutex_unlock(&guest_ram_lock); return active; } @@ -1405,33 +1619,78 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, - * path, range count and every range (gpa/offset/size). The caller supplies - * the buffer; nothing is written when the file is not configured beyond the - * active=0 marker. + * path, range count and every range (gpa/offset/size). With only the legacy + * record configured (the one-mdev campaign) the output is byte-identical to + * the pre-dual-guest format; a per-mdev record adds its mdev UUID so a + * dual-guest operator can tell the records apart. */ +static void gsp_phys_guest_ram_rec_state(char *buf, size_t len, + unsigned int *n, + const struct gsp_phys_guest_ram_rec *rec, + const char *label) +{ + unsigned int i; + + if (*n >= len) + return; + *n += scnprintf(buf + *n, len - *n, "%s path=%s ranges=%u", + label, rec->path, rec->ranges_n); + for (i = 0; i < rec->ranges_n && *n < len; i++) + *n += scnprintf(buf + *n, len - *n, + " gpa=0x%llx off=0x%llx size=0x%llx", + rec->ranges[i].gpa_base, + rec->ranges[i].file_offset, + rec->ranges[i].size); +} + void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; + bool any_mdev = false; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { - scnprintf(buf, len, "active=0"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) { + any_mdev = true; + break; + } + + if (!any_mdev) { + /* + * Legacy-only (or nothing): keep the historical shape - the + * campaign's parser (if any) sees the same text as before. + */ + if (!guest_ram_legacy.file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + n += scnprintf(buf + n, len - n, "active=1"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_legacy, ""); mutex_unlock(&guest_ram_lock); return; } - n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", - guest_ram_path, guest_ram_ranges_n); - for (i = 0; i < guest_ram_ranges_n && n < len; i++) - n += scnprintf(buf + n, len - n, - " gpa=0x%llx off=0x%llx size=0x%llx", - guest_ram_ranges[i].gpa_base, - guest_ram_ranges[i].file_offset, - guest_ram_ranges[i].size); + n += scnprintf(buf + n, len - n, "active=1"); + if (guest_ram_legacy.file) + gsp_phys_guest_ram_rec_state(buf, len, &n, &guest_ram_legacy, + " legacy"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS && n < len; i++) { + char label[64]; + + if (!guest_ram_mdevs[i].file) + continue; + scnprintf(label, sizeof(label), " mdev=%s", + (guest_ram_mdevs[i].vgpu && guest_ram_mdevs[i].vgpu->mdev) ? + dev_name(&guest_ram_mdevs[i].vgpu->mdev->dev) : "?"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_mdevs[i], label); + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); @@ -1452,12 +1711,20 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * - * Guards kept from the original path: the active-mdev walk + * Guards kept from the original path: the mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - - * the engine path must only run with a live guest session; no such session - * returns -ENODEV. The IOMMU-domain query remains as a documented fallback - * for a setup whose mdev has a real domain (it is inert with the - * emulated-IOMMU mdev used here). + * the engine path must only run with a live guest session; a vgpu given by + * the caller must itself be open, and no open session at all returns + * -ENODEV. The IOMMU-domain query remains as a documented fallback for a + * setup whose mdev has a real domain (it is inert with the emulated-IOMMU + * mdev used here). + * + * Per-mdev resolution (dual-guest goal 1.0): the caller names the owning + * vgpu (a captured forwarding request, a held release, ...). The vgpu's own + * record resolves the translation; the legacy record (configured by the + * historical top-level debugfs write) still serves the FIRST open mdev so + * single-guest scripts are unchanged. vgpu == NULL keeps the historical + * resolution: the first open mdev. * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file @@ -1467,13 +1734,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ -int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, u64 *hpa_out) { - struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; + struct nv_vgpu_device *first = NULL; + struct gsp_phys_guest_ram_rec *rec = NULL; struct iommu_domain *dom; phys_addr_t p; unsigned int i; + bool vgpu_gone = false; if (!hpa_out) return -EINVAL; @@ -1481,19 +1750,28 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; + first = d; break; } } + if (vgpu && atomic_read(&vgpu->vfio_dev_open_count) <= 0) + vgpu_gone = true; /* named session is gone: never retarget */ mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu_gone) + return -ENODEV; + if (!vgpu) + vgpu = first; /* historical resolution: first open mdev */ if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - for (i = 0; i < guest_ram_ranges_n; i++) { + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec && vgpu == first) + rec = &guest_ram_legacy; /* legacy file: first/only mdev */ + if (rec && rec->file) { + for (i = 0; i < rec->ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = - &guest_ram_ranges[i]; + &rec->ranges[i]; struct page *page; u64 file_off; @@ -1502,7 +1780,7 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) continue; file_off = r->file_offset + (gpa - r->gpa_base); - page = read_cache_page(guest_ram_file->f_mapping, + page = read_cache_page(rec->file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); @@ -1538,6 +1816,18 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) *hpa_out = (u64)p; return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa_vgpu); + +/* + * Historical entry point: resolve through the first open mdev (and that + * mdev's own record, else the legacy record). Used by the single-guest + * boot-ladder guest-GPA pin (test-only; see gsp_phys_guest_gpa_write) and + * kept as the compatibility wrapper for callers without a request context. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + return gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, hpa_out); +} EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* |