diff options
Diffstat (limited to 'kernel/gsp_phys_gpu.c')
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 643 |
1 files changed, 565 insertions, 78 deletions
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index c5419a5..e896664 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -638,13 +638,20 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value); /* - * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's - * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by - * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does - * not emit these (it relies on the IQ2M window's implied copy PCAS); setting - * this to 0 emits the pure vendor window-only form for the A/B test. - */ -static int gsp_phys_rmcall_gr_pcas_param = 1; + * Explicit-PCAS gate (default 0 since r343 - the double-execution fix). + * The rebuilt launch used to carry the vendor's turingPushPcas + * (CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by + * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE) ADDED to the inline-QMD run + * that the stock guest push already carries. The QMD's + * REQUIRE_SCHEDULING_PCAS bit is clear, so the inline run schedules by + * itself - the explicit pair was a SECOND scheduling edge and the same QMD + * was dispatched twice (launch_count: one atomicAdd +100 landed as +200; + * victory_reduce exactly 2x; scale/saxpy/memset2d doubled). The stock guest + * push has NO PCAS pair (captured push dumps); default 0 restores that + * single-edge form. 1 = re-add the pair (A/B debug only - with it on, the + * grid executes twice). + */ +static int gsp_phys_rmcall_gr_pcas_param; module_param_named(gsp_phys_rmcall_gr_pcas, gsp_phys_rmcall_gr_pcas_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas, @@ -949,6 +956,174 @@ int gsp_phys_rmcall_gr_skedrec_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value); /* + * m153 no-write/wrong-place launch retry (gsp_phys_rmcall_gr_nowrite_retry; + * default 1). After a forwarded launch reports completion, the output + * retirement wait measures how many resolved data pages the engine actually + * changed. Two mis-execution signatures are known to be intermittent per + * boot: the engine writes nothing at all, or it writes into a lower operand + * window while the highest (output) operand stays untouched (a stale + * constant-source state). Both re-execute correctly on a fresh ring entry + * often enough to be worth a bounded retry: refill the bounce window from + * the guest inputs, re-publish the same method block, re-issue the schedule + * and re-poll, then re-wait. Set to 0 to disable the retry and accept the + * first outcome unconditionally. + */ +static int gsp_phys_rmcall_gr_nowrite_retry_param = 1; +module_param_named(gsp_phys_rmcall_gr_nowrite_retry, + gsp_phys_rmcall_gr_nowrite_retry_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_nowrite_retry, + "Re-execute a forwarded GR launch whose retirement wait saw zero/wrong-place engine writes (default 1)"); + +int gsp_phys_rmcall_gr_nowrite_retry_value(void) +{ + return gsp_phys_rmcall_gr_nowrite_retry_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_nowrite_retry_value); + +/* + * F2 fast re-arm (gsp_phys_gr_retry_fast; default 1). r198: the first + * forwarded-launch timeout costs 2x5 s before the fresh-entry ladder runs, + * yet the m155 extended re-arm (lane-enable replay + GPFIFO/TSG schedule + + * doorbell) plus one fresh ring entry recovers most of these launches. With + * this on, the first timeout immediately runs that re-arm + a fresh entry + * and re-polls with a short window; only if that still fails does the + * historical timeout ladder continue. Set to 0 for today's exact behavior. + */ +static int gsp_phys_gr_retry_fast_param = 1; +module_param_named(gsp_phys_gr_retry_fast, + gsp_phys_gr_retry_fast_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_retry_fast, + "On the first GR launch timeout, run the extended re-arm + a fresh ring entry immediately and re-poll briefly (default 1; 0 = legacy ladder timing)"); + +int gsp_phys_gr_retry_fast_value(void) +{ + return gsp_phys_gr_retry_fast_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_retry_fast_value); + +/* + * F2 first-poll window (gsp_phys_gr_first_poll_ms; default 2500, clamped to + * [1000, 30000]). With gsp_phys_gr_retry_fast=1 the FIRST poll window of a + * forwarded GR launch (before any re-arm) uses this instead of the fixed 5 s, + * so the fast re-arm path above starts earlier; with retry_fast=0 the launch + * keeps today's exact behavior and this knob is not read. The later ladder + * windows keep their own constants. + */ +static unsigned int gsp_phys_gr_first_poll_ms_param = 2500u; +module_param_named(gsp_phys_gr_first_poll_ms, + gsp_phys_gr_first_poll_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_first_poll_ms, + "First GR launch poll window in ms, clamped [1000,30000], read when gsp_phys_gr_retry_fast=1 (default 2500)"); + +unsigned int gsp_phys_gr_first_poll_ms_value(void) +{ + unsigned int v = gsp_phys_gr_first_poll_ms_param; + + if (v < 1000u) + v = 1000u; + if (v > 30000u) + v = 30000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_first_poll_ms_value); + +/* + * M91 icache coherence (gsp_phys_gr_icache_inv; default 1). The m91 class is + * the transient SM instruction-fetch lottery: the first attempt of a + * mid-sequence launch faults on SM garbage ("Illegal Instruction Encoding" / + * "Out Of Range Register") while a retry of the identical QMD and identical + * shadow content succeeds. The window-0 shadow pool reuses code VAs across + * launches (two 8-page banks alternate by submit epoch), so the bytes at a + * program VA change between launches, and every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE (word 5 + * bit 30) CLEAR - the field the CUDA driver arms only under its + * forceIcacheInvalidate knob. An SM fetch line cached for launch N can then + * execute the previous kernel's bytes at the same VA in launch N+2. With this + * on, the forwarded QMD sets that bit (only when the QMD carries the driver's + * standard cache-control signature, bits 26..29 of word 5, so an unknown QMD + * revision is never touched), forcing a refetch at dispatch. 0 restores the + * exact pre-change QMD for a one-flip A/B. + */ +static int gsp_phys_gr_icache_inv_param = 1; +module_param_named(gsp_phys_gr_icache_inv, gsp_phys_gr_icache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_icache_inv, + "Set QMDV02_03 INVALIDATE_INSTRUCTION_CACHE (word5 bit30) on forwarded GR launches whose QMD has the driver cache-control signature (default 1; 0 = pre-m91 QMD words)"); + +int gsp_phys_gr_icache_inv_value(void) +{ + return gsp_phys_gr_icache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_icache_inv_value); + +/* + * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped + * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch + * whose retirement wait saw zero engine writes; it shared the timeout ladder's + * attempt counter, so a launch that had already spent attempts on the ladder + * (fast re-arm + fresh ring entries) reached the accept path with NO + * re-execution at all - the r200/r201/r202 memcpy_roundtrip / multikernel + * class (a lost launch silently accepted, client output all wrong). This + * budget is dedicated to the no-write loop and independent of the ladder; it + * is read once per submission. 0 keeps the accept-without-retry behavior + * (still logged loudly). The clamp keeps the worst-case host publishes per + * launch (initial + fast re-arm + 2 fresh entries + budget) within the + * 8-entry GPFIFO ring. + */ +static int gsp_phys_gr_nowrite_retries_param = 3; +module_param_named(gsp_phys_gr_nowrite_retries, + gsp_phys_gr_nowrite_retries_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_nowrite_retries, + "Dedicated re-execution budget for a data-bearing GR launch that wrote nothing tracked, clamped [0,4] (default 3)"); + +/* + * STALLFREE: absolute per-launch wall-clock budget (gsp_phys_gr_wait_max_ms; + * default 60000, clamped [10000, 600000]; 0 disables the extra cap and the + * built-in ladder bounds stand alone). The GR submit ladder and its retirement + * (no-write) retries are individually bounded, but their SUM grows with every + * re-arm/re-publish: a launch whose engine ack never arrives could hold the + * single-inflight work item for tens of seconds while the guest client waits + * on its QMD completion word. When this budget is crossed the submit stops its + * ladder immediately (no further re-arm/publish/retry), logs + * 'vgsp-gr: GR-STALL-wait-max ... action=fail' and returns -ETIMEDOUT; the + * sema service's failure edge then authors the QMD completion release in + * software, so the client always completes. A green launch (~0.2-11 s end to + * end, including the fast-re-arm recovery) never approaches the default, so + * the default does not change green behavior. + */ +static unsigned int gsp_phys_gr_wait_max_ms_param = 60000u; +module_param_named(gsp_phys_gr_wait_max_ms, + gsp_phys_gr_wait_max_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wait_max_ms, + "Absolute per-launch GR submit budget in ms, clamped [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000)"); + +unsigned int gsp_phys_gr_wait_max_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wait_max_ms_param; + + if (!v) + return 0; + if (v < 10000u) + v = 10000u; + if (v > 600000u) + v = 600000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wait_max_ms_value); + +int gsp_phys_gr_nowrite_retries_value(void) +{ + int v = gsp_phys_gr_nowrite_retries_param; + + if (v < 0) + v = 0; + if (v > 4) + v = 4; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted @@ -1050,6 +1225,28 @@ int gsp_phys_rmcall_gr_data_prewin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_data_prewin_value); /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region; default + * 1). When set, gsp_phys_rmcall_pdb_build plants the 16384-page (64 MiB) + * region at GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 (PD0 80..111) and the SVC + * relocates operand pages into it; the booter's copy-in/copy-out index the + * grown chunk. When 0, everything falls back to the legacy 205-page / + * 68-pages-per-pointer window below the mail page, for one-flip A/B + * trialing. Boot-fixed: it is read by the stage-3 plant, so set it at + * insmod, not at runtime. + */ +static int gsp_phys_gr_data_region_param = 1; +module_param_named(gsp_phys_gr_data_region, + gsp_phys_gr_data_region_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_data_region, + "Use the grown 64 MiB data-bounce region for GR operands (default 1; 0 = legacy 205-page window)"); + +int gsp_phys_gr_data_region_value(void) +{ + return gsp_phys_gr_data_region_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_data_region_value); + +/* * m113: after planting the launch's data mappings, re-register the VAS page * directory (UNSET+SET) to force the server walker to re-snapshot the tree; * all evidence says the SM walker resolves only the commit-time tree, and @@ -1231,16 +1428,28 @@ EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The - * ranges map guest GPA windows to file offsets; up to four windows of one - * file per module cycle. + * ranges map guest GPA windows to file offsets; up to four windows per file. + * + * Per-mdev records (dual-guest goal 1.0): a guest RAM backing file belongs + * to ONE guest session, so the record moves with the mdev. Slot 0 is the + * legacy record (vgpu == NULL): it is what the historical top-level + * gsp-phys-guest-ram debugfs write configures and it serves the first/only + * open mdev exactly as the pre-dual-guest code did (one guest: byte-identical + * behaviour). A per-mdev record is configured with the mdev UUID as an extra + * first token on the same debugfs file and serves only that mdev; the + * record is released when the mdev is removed (gsp_phys_guest_ram_vgpu_clear) + * or when the physical cycle ends (gsp_phys_guest_ram_clear). * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. - * One file per module cycle: a second path is refused -EBUSY because a fresh - * cycle reloads the module; more ranges of the same file may be added. + * Within one record the historical rule is unchanged: the first configure + * opens the file, a different path is refused -EBUSY, more ranges of the + * same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 +/* Per-mdev guest-RAM records (a handful of guests; 4 is far beyond 1.0). */ +#define GSP_PHYS_GUEST_RAM_MDEVS 4 struct gsp_phys_guest_ram_range { u64 gpa_base; @@ -1248,35 +1457,121 @@ struct gsp_phys_guest_ram_range { u64 size; }; -static struct file *guest_ram_file; -static char guest_ram_path[128]; -static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; -static unsigned int guest_ram_ranges_n; +struct gsp_phys_guest_ram_rec { + struct nv_vgpu_device *vgpu; /* NULL = legacy (first/only open mdev) */ + struct file *file; + char path[128]; + struct gsp_phys_guest_ram_range ranges[GSP_PHYS_GUEST_RAM_RANGES]; + unsigned int ranges_n; +}; + +static struct gsp_phys_guest_ram_rec guest_ram_legacy; +static struct gsp_phys_guest_ram_rec guest_ram_mdevs[GSP_PHYS_GUEST_RAM_MDEVS]; static DEFINE_MUTEX(guest_ram_lock); +/* Caller holds guest_ram_lock. NULL when this vgpu has no bound record. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_find_locked(const struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return NULL; + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].vgpu == vgpu) + return &guest_ram_mdevs[i]; + return NULL; +} + +/* Caller holds guest_ram_lock. NULL when every per-mdev slot is taken. */ +static struct gsp_phys_guest_ram_rec * +gsp_phys_guest_ram_rec_alloc_locked(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (!guest_ram_mdevs[i].vgpu) { + guest_ram_mdevs[i].vgpu = vgpu; + return &guest_ram_mdevs[i]; + } + return NULL; +} + /* - * Configure the guest RAM backing file and add (or replace, keyed by - * gpa_base) one GPA window. The first call opens the file read-only and - * fixes the path for this module cycle; later calls may only add ranges of - * that same file. The file must be a regular file and every range must fit - * inside its current size, so a translation can never index a page beyond - * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges - * in use, or the file/argument error. + * Number of mdevs whose VFIO device is currently open. Shared by the + * per-mdev debugfs selector (which mdev does this UUID name?) and the + * single-guest guard on the legacy test-only paths. */ -int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, - u64 file_offset, u64 size) +int gsp_phys_open_mdev_count(void) +{ + struct nv_vgpu_device *d; + int n = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) + n++; + } + mutex_unlock(&nv_vgpu_driver.lock); + return n; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_count); + +/* + * Resolve an OPEN mdev by its device name (the mdev UUID, e.g. + * "b6acdc5e-..."). Returns the device or NULL. The caller must use the + * result only while that mdev's VFIO fd is known to be open (the same + * lifetime rule as the first-open resolution below): the driver list is + * walked under nv_vgpu_driver.lock, but the returned pointer is not + * reference-counted. + */ +struct nv_vgpu_device *gsp_phys_open_mdev_by_name(const char *name) +{ + struct nv_vgpu_device *d, *found = NULL; + + if (!name || !*name) + return NULL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0 && d->mdev && + !strcmp(dev_name(&d->mdev->dev), name)) { + found = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return found; +} +EXPORT_SYMBOL_GPL(gsp_phys_open_mdev_by_name); + +/* + * Shared configure body for one record: open the record's file (first call) + * and add (or replace, keyed by gpa_base) one GPA window. The first call + * opens the file read-only and fixes the path for the record's lifetime; + * later calls may only add ranges of that same file. The file must be a + * regular file and every range must fit inside its current size, so a + * translation can never index a page beyond EOF. Returns 0, -EBUSY for a + * different path, -ENOSPC with all four ranges in use, or the + * file/argument error. `who` is "" for the legacy record and + * " for mdev <uuid>" for a per-mdev record; the legacy strings stay + * byte-identical. + */ +static int gsp_phys_guest_ram_rec_configure(struct gsp_phys_guest_ram_rec *rec, + const char *path, u64 gpa_base, + u64 file_offset, u64 size, + const char *who) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; - if (!path || !*path) + if (!rec || !path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { + if (!rec->file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond @@ -1306,35 +1601,38 @@ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, ret = -ERANGE; goto out_fput; } - guest_ram_file = f; - strscpy(guest_ram_path, path, sizeof(guest_ram_path)); - } else if (strcmp(guest_ram_path, path)) { - pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", - guest_ram_path, path); + rec->file = f; + strscpy(rec->path, path, sizeof(rec->path)); + } else if (strcmp(rec->path, path)) { + if (who[0]) + pr_err("vgsp: guest RAM already configured from %s; one file per guest session (reload or mdev remove to switch); refusing %s\n", + rec->path, path); + else + pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + rec->path, path); ret = -EBUSY; goto out; } - for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { - if (guest_ram_ranges[i].gpa_base == gpa_base) { - range = &guest_ram_ranges[i]; + for (i = 0, range = NULL; i < rec->ranges_n; i++) { + if (rec->ranges[i].gpa_base == gpa_base) { + range = &rec->ranges[i]; break; } } if (!range) { - if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + if (rec->ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } - range = &guest_ram_ranges[guest_ram_ranges_n++]; + range = &rec->ranges[rec->ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; - pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", - guest_ram_path, gpa_base, file_offset, size, - guest_ram_ranges_n); + pr_info("vgsp: guest RAM configured%s: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + who, rec->path, gpa_base, file_offset, size, rec->ranges_n); out: mutex_unlock(&guest_ram_lock); @@ -1344,35 +1642,148 @@ out_fput: fput(f); goto out; } + +/* + * Configure the legacy guest RAM record (the historical top-level + * gsp-phys-guest-ram write). Serves the first/only open mdev exactly as + * before the per-mdev records existed. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + return gsp_phys_guest_ram_rec_configure(&guest_ram_legacy, path, + gpa_base, file_offset, size, + ""); +} EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* - * Close the configured guest RAM file (if any) and drop every range. Called - * from the physical-GPU release path so module unload and a backend release - * end the file reference; a fresh cycle reconfigures from debugfs. + * Configure the guest RAM record bound to one mdev (per-mdev selector on the + * debugfs write). A record for this vgpu is created on first use and closed + * by gsp_phys_guest_ram_vgpu_clear() when the mdev is removed. A failed + * first configure leaves no bound record behind. + */ +int gsp_phys_guest_ram_configure_vgpu(struct nv_vgpu_device *vgpu, + const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_rec *rec; + char who[80]; + const char *name = (vgpu && vgpu->mdev) ? + dev_name(&vgpu->mdev->dev) : "?"; + bool fresh = false; + int ret; + + if (!vgpu) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec) { + rec = gsp_phys_guest_ram_rec_alloc_locked(vgpu); + if (!rec) { + mutex_unlock(&guest_ram_lock); + pr_err("vgsp: guest RAM: no free per-mdev record slot (max %u); refusing mdev %s\n", + GSP_PHYS_GUEST_RAM_MDEVS, name); + return -ENOSPC; + } + fresh = true; + } + mutex_unlock(&guest_ram_lock); + + scnprintf(who, sizeof(who), " for mdev %s", name); + ret = gsp_phys_guest_ram_rec_configure(rec, path, gpa_base, + file_offset, size, who); + + /* + * A fresh record whose first configure failed must not hold the slot + * (the vgpu pointer would otherwise be reserved with no file and the + * next attempt could not create a clean record). + */ + if (ret && fresh) { + mutex_lock(&guest_ram_lock); + if (rec->file) + ret = 0; /* raced with another configure; keep it */ + else + rec->vgpu = NULL; + mutex_unlock(&guest_ram_lock); + } + return ret; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure_vgpu); + +/* Lock held: close one record's file and drop its ranges. */ +static void gsp_phys_guest_ram_rec_close(struct gsp_phys_guest_ram_rec *rec, + const char *release_log) +{ + if (rec->file) { + pr_info("%s: path=%s ranges=%u\n", release_log, + rec->path, rec->ranges_n); + fput(rec->file); + rec->file = NULL; + } + rec->path[0] = '\0'; + memset(rec->ranges, 0, sizeof(rec->ranges)); + rec->ranges_n = 0; +} + +/* + * Close every configured guest RAM record (legacy and per-mdev) and drop + * every range. Called from the physical-GPU release path so module unload + * and a backend release end the file references; a fresh cycle reconfigures + * from debugfs. The legacy release log line is unchanged. */ void gsp_phys_guest_ram_clear(void) { + unsigned int i; + mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", - guest_ram_path, guest_ram_ranges_n); - fput(guest_ram_file); - guest_ram_file = NULL; - } - guest_ram_path[0] = '\0'; - memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); - guest_ram_ranges_n = 0; + gsp_phys_guest_ram_rec_close(&guest_ram_legacy, + "vgsp: guest RAM released"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); +/* + * Release the guest RAM record bound to one mdev. Called from the mdev + * cleanup path so the file reference and the vgpu pointer never outlive the + * session (a removed mdev's translations must stop, and the record slot must + * free up for a restarted guest). + */ +void gsp_phys_guest_ram_vgpu_clear(struct nv_vgpu_device *vgpu) +{ + unsigned int i; + + if (!vgpu) + return; + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) { + if (guest_ram_mdevs[i].vgpu == vgpu) { + gsp_phys_guest_ram_rec_close(&guest_ram_mdevs[i], + "vgsp: guest RAM released for mdev"); + guest_ram_mdevs[i].vgpu = NULL; + } + } + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_vgpu_clear); + bool gsp_phys_guest_ram_active(void) { - bool active; + bool active = false; + unsigned int i; mutex_lock(&guest_ram_lock); - active = guest_ram_file != NULL; + if (guest_ram_legacy.file) + active = true; + for (i = 0; !active && i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) + active = true; mutex_unlock(&guest_ram_lock); return active; } @@ -1380,33 +1791,78 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, - * path, range count and every range (gpa/offset/size). The caller supplies - * the buffer; nothing is written when the file is not configured beyond the - * active=0 marker. + * path, range count and every range (gpa/offset/size). With only the legacy + * record configured (the one-mdev campaign) the output is byte-identical to + * the pre-dual-guest format; a per-mdev record adds its mdev UUID so a + * dual-guest operator can tell the records apart. */ +static void gsp_phys_guest_ram_rec_state(char *buf, size_t len, + unsigned int *n, + const struct gsp_phys_guest_ram_rec *rec, + const char *label) +{ + unsigned int i; + + if (*n >= len) + return; + *n += scnprintf(buf + *n, len - *n, "%s path=%s ranges=%u", + label, rec->path, rec->ranges_n); + for (i = 0; i < rec->ranges_n && *n < len; i++) + *n += scnprintf(buf + *n, len - *n, + " gpa=0x%llx off=0x%llx size=0x%llx", + rec->ranges[i].gpa_base, + rec->ranges[i].file_offset, + rec->ranges[i].size); +} + void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; + bool any_mdev = false; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); - if (!guest_ram_file) { - scnprintf(buf, len, "active=0"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS; i++) + if (guest_ram_mdevs[i].file) { + any_mdev = true; + break; + } + + if (!any_mdev) { + /* + * Legacy-only (or nothing): keep the historical shape - the + * campaign's parser (if any) sees the same text as before. + */ + if (!guest_ram_legacy.file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + n += scnprintf(buf + n, len - n, "active=1"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_legacy, ""); mutex_unlock(&guest_ram_lock); return; } - n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", - guest_ram_path, guest_ram_ranges_n); - for (i = 0; i < guest_ram_ranges_n && n < len; i++) - n += scnprintf(buf + n, len - n, - " gpa=0x%llx off=0x%llx size=0x%llx", - guest_ram_ranges[i].gpa_base, - guest_ram_ranges[i].file_offset, - guest_ram_ranges[i].size); + n += scnprintf(buf + n, len - n, "active=1"); + if (guest_ram_legacy.file) + gsp_phys_guest_ram_rec_state(buf, len, &n, &guest_ram_legacy, + " legacy"); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS && n < len; i++) { + char label[64]; + + if (!guest_ram_mdevs[i].file) + continue; + scnprintf(label, sizeof(label), " mdev=%s", + (guest_ram_mdevs[i].vgpu && guest_ram_mdevs[i].vgpu->mdev) ? + dev_name(&guest_ram_mdevs[i].vgpu->mdev->dev) : "?"); + gsp_phys_guest_ram_rec_state(buf, len, &n, + &guest_ram_mdevs[i], label); + } mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); @@ -1427,12 +1883,20 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * - * Guards kept from the original path: the active-mdev walk + * Guards kept from the original path: the mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - - * the engine path must only run with a live guest session; no such session - * returns -ENODEV. The IOMMU-domain query remains as a documented fallback - * for a setup whose mdev has a real domain (it is inert with the - * emulated-IOMMU mdev used here). + * the engine path must only run with a live guest session; a vgpu given by + * the caller must itself be open, and no open session at all returns + * -ENODEV. The IOMMU-domain query remains as a documented fallback for a + * setup whose mdev has a real domain (it is inert with the emulated-IOMMU + * mdev used here). + * + * Per-mdev resolution (dual-guest goal 1.0): the caller names the owning + * vgpu (a captured forwarding request, a held release, ...). The vgpu's own + * record resolves the translation; the legacy record (configured by the + * historical top-level debugfs write) still serves the FIRST open mdev so + * single-guest scripts are unchanged. vgpu == NULL keeps the historical + * resolution: the first open mdev. * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file @@ -1442,13 +1906,15 @@ EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ -int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +int gsp_phys_gpa_to_hpa_vgpu(struct nv_vgpu_device *vgpu, u64 gpa, u64 *hpa_out) { - struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; + struct nv_vgpu_device *first = NULL; + struct gsp_phys_guest_ram_rec *rec = NULL; struct iommu_domain *dom; phys_addr_t p; unsigned int i; + bool vgpu_gone = false; if (!hpa_out) return -EINVAL; @@ -1456,19 +1922,28 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { - vgpu = d; + first = d; break; } } + if (vgpu && atomic_read(&vgpu->vfio_dev_open_count) <= 0) + vgpu_gone = true; /* named session is gone: never retarget */ mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu_gone) + return -ENODEV; + if (!vgpu) + vgpu = first; /* historical resolution: first open mdev */ if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); - if (guest_ram_file) { - for (i = 0; i < guest_ram_ranges_n; i++) { + rec = gsp_phys_guest_ram_rec_find_locked(vgpu); + if (!rec && vgpu == first) + rec = &guest_ram_legacy; /* legacy file: first/only mdev */ + if (rec && rec->file) { + for (i = 0; i < rec->ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = - &guest_ram_ranges[i]; + &rec->ranges[i]; struct page *page; u64 file_off; @@ -1477,7 +1952,7 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) continue; file_off = r->file_offset + (gpa - r->gpa_base); - page = read_cache_page(guest_ram_file->f_mapping, + page = read_cache_page(rec->file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); @@ -1513,6 +1988,18 @@ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) *hpa_out = (u64)p; return 0; } +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa_vgpu); + +/* + * Historical entry point: resolve through the first open mdev (and that + * mdev's own record, else the legacy record). Used by the single-guest + * boot-ladder guest-GPA pin (test-only; see gsp_phys_guest_gpa_write) and + * kept as the compatibility wrapper for callers without a request context. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + return gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, hpa_out); +} EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* |