diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 509 |
1 files changed, 347 insertions, 162 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 9edb66d..72480a2 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1088,7 +1088,7 @@ static void svc_add_map(u64 gpu_va, u64 pa); * finishes. VAs whose full scan found nothing are remembered and not rescanned * for SVC_TR_FAIL_HOLD. */ -#define SVC_TR_SLICE (128ULL << 20) /* guest RAM covered per call */ +#define SVC_TR_SLICE (512ULL << 20) /* guest RAM covered per call */ #define SVC_TR_CHUNK (64 * 1024) /* one guest read */ #define SVC_TR_FAIL_MAX 16 #define SVC_TR_FAIL_HOLD (60 * HZ) @@ -1096,6 +1096,7 @@ static void svc_add_map(u64 gpu_va, u64 pa); static u64 tr_va; /* page VA of the active scan, 0 = idle */ static u64 tr_cursor; /* next GPA to read */ static unsigned long tr_start; /* active scan start (staleness bound) */ +static unsigned long tr_last; /* last lookup of the active scan's VA */ static u64 tr_scans; static struct { u64 va; @@ -1134,13 +1135,27 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, if (svc_tr_recently_failed(page_va)) return -ENOENT; - if (tr_va && tr_va != page_va) - return -EAGAIN; /* another VA owns the active scan */ + if (tr_va && tr_va != page_va) { + /* + * The scan slot is owned by another page. If that page is + * still being looked up (the pump retries its GPFIFO entry + * every pass) let it finish; if it has not been seen for a + * second, its entry was consumed or skipped and the slot + * would otherwise stay wedged for the full 30 s scan window, + * starving every live page behind it (observed: a skipped GR + * entry left CE memops pages unresolvable until the scan + * timed out). + */ + if (time_before(jiffies, tr_last + HZ)) + return -EAGAIN; /* another VA owns the active scan */ + tr_va = 0; + } if (!tr_va) { tr_va = page_va; tr_cursor = 0; tr_start = jiffies; } + tr_last = jiffies; if (time_after(jiffies, tr_start + 30 * HZ)) { /* The guest stopped giving us its memory: abandon the scan. */ svc_tr_note_failure(tr_va); @@ -1213,6 +1228,22 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, } /* + * True while a full-RAM targeted scan for this VA's page owns the scan slot + * (and has not aged out). The pump uses it to hold an entry whose translation + * may land within the scan instead of consuming it as stale. + */ +static bool svc_resolve_in_progress(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + + if (!tr_va || tr_va != page) + return false; + if (time_after(jiffies, tr_start + 30 * HZ)) + return false; + return true; +} + +/* * Translate a GPU VA using trusted sources only: a pinned translation, the * learned cache, the discovered map, the directories the driver published and * the self-discovered directory. The final vidmem window convention is @@ -1722,6 +1753,7 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 +#define SEM_RELEASE_FOUR_WORD 2 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ @@ -1947,25 +1979,6 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) } /* - * Relaxed push-prefix check for rescue candidate validation only. Real pushes - * observed in this workload start with methods at non-zero addresses (e.g. - * 0x2001001e = count 1, addr 0x78), which the strict check above rejects - * because it requires addr 0. Rescue only needs the candidate page to begin - * a plausible method stream; work execution still uses the strict check. - */ -static bool svc_stream_prefix_ok(const u8 *buf, u32 len) -{ - u32 hdr = 0, op, count; - - if (len < 8) - return false; - memcpy(&hdr, buf, 4); - op = (hdr >> 29) & 0x7; - count = (hdr >> 16) & 0xfff; - return op == 1 && count && count <= 64 && 4 + count * 4 <= len; -} - -/* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page @@ -2320,8 +2333,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * Both release encodings write the payload word first + * (NvGpuSemaphore.data[0] for the four-word format, the + * single word for the one-word format), and the client's + * trackers poll exactly that u32: apply the payload for + * either. The four-word format's timestamp/status words + * that follow are not observed by the waiters. + */ if (DMA_SEMAPHORE_TYPE(dma) == - SEM_RELEASE_ONE_WORD) { + SEM_RELEASE_ONE_WORD || + DMA_SEMAPHORE_TYPE(dma) == + SEM_RELEASE_FOUR_WORD) { u64 sem_va; u32 cur = 0; int pending = 0; @@ -3325,6 +3348,129 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) } /* + * Match one buffer for this channel's pool-semaphore release group and append + * candidate pin bases. The group is: + * +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4) + * +0x04 hi == 1 + * +0x08 lo & 0xfff == this channel's pool slot + * +0x0c payload (any small value; the tracked expectation can lag) + * +0x10 LAUNCH_DMA semaphore-release follower (transfer none) + * Candidates are the page the group sits in (correct when the push starts in + * that page, e.g. a stale release push) and the pages implied by treating the + * group as the end of this entry's push, walking back a page at a time so a + * trailing method run after the release does not defeat the guess. Returns the + * updated candidate count. + */ +static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, + u64 entry_len, u64 *cand, u32 ncand) +{ + u32 o; + + for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) { + u32 hdr, hi, lo, payload, w5, v5; + u64 rel_pa, d; + + memcpy(&hdr, buf + o, 4); + if (hdr != 0x20038090u) + continue; + memcpy(&hi, buf + o + 4, 4); + if (hi != 1) + continue; + memcpy(&lo, buf + o + 8, 4); + if ((lo & 0xfff) != slot) + continue; + memcpy(&payload, buf + o + 12, 4); + if (!payload || payload >= (1u << 24)) + continue; + memcpy(&w5, buf + o + 16, 4); + memcpy(&v5, buf + o + 20, 4); + if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 || + ((w5 & 0x1fff) << 2) != 0x300 || + (v5 & 0x3) != 0) + continue; + + rel_pa = base_pa + o; + cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + for (d = entry_len >= 24 ? entry_len - 24 : 0; + d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) { + u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1); + + if (s != cand[ncand - 1]) + cand[ncand++] = s; + } + } + return ncand; +} + +/* + * Full method-stream parse: every word run must be a valid INC_METHOD header + * covering the entire buffer, nothing left over. Real pushes satisfy this by + * construction; arbitrary data (firmware pages, page tables, allocator free + * lists) practically never does - a false candidate must pass this test for + * the whole push, not just for its first header. + */ +static bool svc_stream_full_ok(const u8 *buf, u32 len) +{ + u32 off = 0, methods = 0; + + while (off + 4 <= len) { + u32 hdr, op, count; + + memcpy(&hdr, buf + off, 4); + if (hdr == 0) + break; /* trailing zero padding */ + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + if (op != 1 || !count || count > 64 || + off + 4 + count * 4 > len) + return false; + off += 4 + count * 4; + if (++methods > 1024) + return false; + } + /* Trailing zero padding is common in recycled push slots. */ + while (off < len && buf[off] == 0) + off++; + return methods > 0 && off == len; +} + +/* + * Pin a candidate page as the translation of the entry VA's page and verify + * by reading the whole push back and parsing it as a method stream. Requires + * the push to end exactly on a method boundary - the stream replaces a weak + * prefix check that accepted page-table pages (whose first word can decode as + * a header). Returns 0 and leaves the pin installed on success. + */ +static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u64 entry_va, u32 entry_len, u32 pin_len, u64 cand_pa, + u32 cand_i, u32 ncand) +{ + u8 *full; + u32 j; + int rc = -ENOENT; + + for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) { + svc_pin_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); + svc_lat_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); + } + + full = kmalloc(entry_len, GFP_KERNEL); + if (!full) + return -ENOMEM; + if (!svc_read_va_trusted(vgpu, entry_va, full, entry_len) && + svc_stream_full_ok(full, entry_len)) { + for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) + svc_mark_hot(cand_pa + j); + pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n", + ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, + entry_len, cand_pa, cand_i + 1, ncand); + rc = 0; + } + kfree(full); + return rc; +} + +/* * Content rescue for unmapped push-buffer pool VAs. * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the * VA->PA map, we scan guest RAM for a page containing a push whose semaphore @@ -3347,103 +3493,95 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, return -ENOENT; if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI) return -EINVAL; - if (time_before(jiffies, ch->rescue_last + 5 * HZ)) - return -EAGAIN; - ch->rescue_last = jiffies; /* * Slot-aligned entries pin the whole 1 MB slot at once (all of its * pushes share the mapping); an entry in the middle of a slot only - * needs the pages its own push occupies - candidate 2 below is the - * PA of the entry VA's page in both cases. + * needs the pages its own push occupies - the page-walking candidates + * in svc_rescue_match_buf cover the push starting on a page before the + * release. */ pin_len = (entry_va & 0xfffff) ? ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000; - buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return -ENOMEM; /* - * Scan guest RAM for the exact release group at any 4-byte offset: - * +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4) - * +0x04 hi == 1 - * +0x08 lo & 0xfff == this channel's pool slot - * +0x0c payload (any small value; the tracked expectation can lag) - * +0x10 LAUNCH_DMA semaphore-release follower (type 0) - * The header is checked first so the inner loop is one compare for - * almost every offset; the remaining exact fields make false - * positives (the firmware-page matches of the old matcher) - * negligible. - * - * Each match yields two candidate pin bases: the page the group sits - * in (correct when the matched group is in the slot's first page, - * e.g. a stale release push) and the page implied by treating the - * group as the end of THIS entry's push (correct when the push starts - * at the slot base and is longer than one page). Each candidate is - * validated by pinning and reading the entry VA back; the first that - * parses as a push wins. + * Hot pages first: pushbuffer pages are recycled and this workload + * touches only a small working set, so the page holding a just-written + * push is almost always already marked hot. Scanning the hot set is + * milliseconds - the client's waits expire long before the multi- + * second full-RAM scan below can run, and that scan also competes + * with the targeted resolver and the sweeps for the service thread. */ - for (gpa = 0; gpa < SEMA_PA_MAX && ncand < SVC_MBUF_MAX; gpa += SVC_SCAN_CHUNK) { - u32 o; + { + unsigned long h; - if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) - continue; - for (o = 0; o + 24 <= SVC_SCAN_CHUNK && ncand < SVC_MBUF_MAX; o += 4) { - u32 hdr, hi, lo, payload, w5, v5; - u64 rel_pa, s; + for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0); + h < SEMA_HOT_PAGES; + h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) { + u32 n; - memcpy(&hdr, buf + o, 4); - if (hdr != 0x20038090u) - continue; - memcpy(&hi, buf + o + 4, 4); - if (hi != 1) - continue; - memcpy(&lo, buf + o + 8, 4); - if ((lo & 0xfff) != slot) - continue; - memcpy(&payload, buf + o + 12, 4); - if (!payload || payload >= (1u << 24)) + gpa = (u64)h << SEMA_PAGE_SHIFT; + if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; - memcpy(&w5, buf + o + 16, 4); - memcpy(&v5, buf + o + 20, 4); - if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 || - ((w5 & 0x1fff) << 2) != 0x300 || - (v5 & 0x3) != 0) - continue; - - rel_pa = gpa + o; - cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); - s = (rel_pa - (entry_len >= 24 ? entry_len - 24 : 0)) & - ~(u64)(SEMA_PAGE_SIZE - 1); - if (ncand < SVC_MBUF_MAX && - (ncand == 0 || cand[ncand - 1] != s)) - cand[ncand++] = s; + ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa, + slot, entry_len, cand, 0); + for (n = 0; n < ncand; n++) { + if (!svc_rescue_try(vgpu, ch, entry_va, + entry_len, pin_len, + cand[n], n, ncand)) { + kfree(buf); + return 0; + } + } } } + + /* + * A full-RAM targeted scan for this exact page is already running (it + * is triggered by the failed read above): let it finish instead of + * duplicating the RAM scan, and let the pump retry the entry next pass + * while the scan's result is still pending. + */ + if (svc_resolve_in_progress(entry_va)) { + kfree(buf); + return -EAGAIN; + } + if (time_before(jiffies, ch->rescue_last + 2 * HZ)) { + kfree(buf); + return -EAGAIN; + } + ch->rescue_last = jiffies; + kfree(buf); + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return -ENOMEM; - for (i = 0; i < ncand; i++) { - u8 probe[64]; - u32 j; - - for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) { - svc_pin_store(entry_va + j, cand[i] + j, - SVC_APERTURE_SYS); - svc_lat_store(entry_va + j, cand[i] + j, - SVC_APERTURE_SYS); - } - if (!svc_read_va_trusted(vgpu, entry_va, probe, - sizeof(probe)) && - svc_stream_prefix_ok(probe, sizeof(probe))) { - for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) - svc_mark_hot(cand[i] + j); - pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n", - ch->h_object, ch->ce_index, slot, entry_va, - entry_len, cand[i], i + 1, ncand); - return 0; + /* + * Full-RAM fallback: the header is checked first so the inner loop is + * one compare for almost every offset; the remaining exact fields make + * false positives (the firmware-page matches of the old matcher) + * negligible. Each candidate is validated by pinning and reading the + * entry VA back; the first that parses as a push wins. + */ + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) + continue; + ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot, + entry_len, cand, 0); + for (i = 0; i < ncand; i++) { + if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, + pin_len, cand[i], i, ncand)) { + kfree(buf); + return 0; + } } } + kfree(buf); if (ncand) pr_info("sema-svc: content rescue ch=0x%x entry_va=0x%llx len=%u: %u candidate(s) did not validate\n", @@ -3711,7 +3849,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.gpfifo_nomap++; kfree(pb); break; - } else if (++ch->rescue_fail >= 3) { + } else if (++ch->rescue_fail >= + (ch->engine_type ? 2 : 1)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -3719,6 +3858,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * candidate. Consume the entry so * the pump moves on instead of * rescanning RAM every 5 s. + * + * GR (compute) channels have no + * rescue at all - their pushes + * are compute-class method + * streams the CE-shaped matcher + * cannot validate - and their + * completion signal is the + * per-channel counter we publish + * from `put`. Consume on the + * first failed attempt: a slow + * skip delays GPGet and the + * counter long enough for the + * client's submission wait to + * expire (observed at ~12 s with + * the 3-attempt path). */ pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n", ch->h_object, va, len, @@ -3772,8 +3926,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * Persist the per-entry rescue state and the * expected payload: the pump works on a snapshot * copy, so without the write-back the stale-entry - * skip (rescue_fail >= 3) and the 5 s rescue rate - * limit (rescue_last) can never fire and the + * skip (rescue_fail >= threshold) and the rescue + * rate limit (rescue_last) can never fire and the * payload expectation is reinitialized every * pass. */ @@ -4037,90 +4191,121 @@ int gsp_sema_service_dump_channels(void) * discoverable from the structures the service scans; this lets a session * feed a pair that was verified from the guest and watch push execution use * it. + * + * The injection is fully asynchronous: the sysfs write only enqueues the + * pair and returns, so a batch of mappings lands in microseconds even while + * the service thread is deep in a scan. Timed-out single-slot injections + * used to be cancelled (the pair was silently lost), and single-slot params + * could not take a batch at all. */ -static struct completion map_add_done; -static u64 map_add_va; -static u64 map_add_pa; -static u64 map_add_len; -static u32 map_add_ap; -static bool map_add_pending; #define SVC_PIN_REWIND 8 +#define SVC_MAP_ADD_MAX 64 + +struct svc_map_add_req { + u64 va; + u64 pa; + u64 len; + u32 ap; +}; + +static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX]; +static u32 svc_map_add_head; /* producer */ +static u32 svc_map_add_tail; /* consumer (service thread) */ +static DEFINE_SPINLOCK(svc_map_add_lock); static void svc_handle_map_add(void) { - u32 ap = map_add_ap ? map_add_ap : SVC_APERTURE_SYS; unsigned long flags; - u64 off; - int i; - if (!map_add_pending) - return; + for (;;) { + struct svc_map_add_req req; + u32 ap, i; + u64 off; - for (off = 0; off < (map_add_len ? map_add_len : 1); - off += SEMA_PAGE_SIZE) { - svc_pin_store(map_add_va + off, map_add_pa + off, ap); - svc_lat_store(map_add_va + off, map_add_pa + off, ap); - } - /* - * A translation can be replaced after the pump already consumed - * pushes that release through it (a new client maps new physical - * pages at the same VA, or the first translation was wrong): the - * releases resolved against the old page, compared as stale against - * its values, and were dropped. Rewind every channel a few ring - * entries so those pushes are re-read under the new translation; - * the release/fill application paths are forward-progress guarded, - * so re-executing them is idempotent. - */ - spin_lock_irqsave(&svc.chan_lock, flags); - for (i = 0; i < SVC_MAX_CHANNELS; i++) { - struct svc_channel *ch = &svc.chan[i]; - u32 get; + spin_lock_irqsave(&svc_map_add_lock, flags); + if (svc_map_add_tail == svc_map_add_head) { + spin_unlock_irqrestore(&svc_map_add_lock, flags); + break; + } + req = svc_map_add_q[svc_map_add_tail % + SVC_MAP_ADD_MAX]; + svc_map_add_tail++; + spin_unlock_irqrestore(&svc_map_add_lock, flags); + + ap = req.ap ? req.ap : SVC_APERTURE_SYS; + for (off = 0; off < (req.len ? req.len : 1); + off += SEMA_PAGE_SIZE) { + svc_pin_store(req.va + off, req.pa + off, ap); + svc_lat_store(req.va + off, req.pa + off, ap); + } + /* + * A translation can be replaced after the pump already + * consumed pushes that release through it (a new client maps + * new physical pages at the same VA, or the first translation + * was wrong): the releases resolved against the old page, + * compared as stale against its values, and were dropped. + * Rewind every channel a few ring entries so those pushes are + * re-read under the new translation; the release/fill + * application paths are forward-progress guarded, so + * re-executing them is idempotent. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + u32 get; + + if (!ch->valid || !ch->gpfifo_entries) + continue; + /* + * Clamp the rewind at 0: wrapping below zero (get=0 -> + * entries - N) makes the pump read ring slots past the + * producer that hold stale entries, so replay only + * recently consumed slots or from the start of the + * ring, never around the wrap. + */ + get = ch->last_get; + ch->last_get = (get >= SVC_PIN_REWIND) ? + get - SVC_PIN_REWIND : 0; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); - if (!ch->valid || !ch->gpfifo_entries) - continue; /* - * Clamp the rewind at 0: wrapping below zero (get=0 -> entries - * - N) makes the pump read ring slots past the producer that - * hold stale entries, so replay only recently consumed slots - * or from the start of the ring, never around the wrap. + * Releases that missed translation before this injection may + * be applicable now: keep the fast full-RAM sweep armed for a + * few seconds so they land without waiting for an unrelated + * stall. */ - get = ch->last_get; - ch->last_get = (get >= SVC_PIN_REWIND) ? get - SVC_PIN_REWIND : 0; + svc.fast_force_until = jiffies + 10 * HZ; + pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s), rewound channels by %u\n", + req.va, req.pa, req.len, + ap == SVC_APERTURE_FB ? "vidmem" : "sysmem", + SVC_PIN_REWIND); } - spin_unlock_irqrestore(&svc.chan_lock, flags); - - /* - * Releases that missed translation before this injection may be - * applicable now: keep the fast full-RAM sweep armed for a few - * seconds so they land without waiting for an unrelated stall. - */ - svc.fast_force_until = jiffies + 10 * HZ; - pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s), rewound channels by %u\n", - map_add_va, map_add_pa, map_add_len, - ap == SVC_APERTURE_FB ? "vidmem" : "sysmem", SVC_PIN_REWIND); - map_add_pending = false; - complete(&map_add_done); } int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) { + unsigned long flags; + u32 next; + if (!svc.task) return -ENODEV; if (len > (64ULL << 20) || ap > SVC_APERTURE_FB) return -EINVAL; - init_completion(&map_add_done); - map_add_va = gpu_va; - map_add_pa = pa; - map_add_len = len; - map_add_ap = ap; - map_add_pending = true; - - if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) { - map_add_pending = false; - return -ETIMEDOUT; + spin_lock_irqsave(&svc_map_add_lock, flags); + next = svc_map_add_head + 1; + if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) { + spin_unlock_irqrestore(&svc_map_add_lock, flags); + return -ENOSPC; } + svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va; + svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa; + svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len; + svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap; + svc_map_add_head = next; + spin_unlock_irqrestore(&svc_map_add_lock, flags); return 0; } |