summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_display_engine.c237
1 files changed, 197 insertions, 40 deletions
diff --git a/kernel/gsp_display_engine.c b/kernel/gsp_display_engine.c
index 30257d5..e79a9f0 100644
--- a/kernel/gsp_display_engine.c
+++ b/kernel/gsp_display_engine.c
@@ -99,7 +99,8 @@
/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */
#define GSP_DISP_NOTIFIER_FINISHED 0x80000000u
#define GSP_DISP_NOTIFIER_CTRL_NOTIFY BIT(12)
-#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff)
+/* The CONTROL offset field is in 16-byte units; expose it in bytes. */
+#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) ((((c) >> 4) & 0xff) * 16)
/* Display instance memory (v03_00 layout). */
#define GSP_DISP_INST_HASH_OFF 0x0000
@@ -131,6 +132,7 @@ struct gsp_disp_chan {
u32 notifier_ctrl;
u64 notifier_pa;
bool notifier_pa_valid;
+ u8 notifier_target; /* instance-table target: 1 FB, 2/3 sysmem */
u32 notifier_writes;
u32 notifier_posts;
u32 update_logs;
@@ -221,6 +223,59 @@ static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}
+/*
+ * A notifier whose context-DMA object targets ADDR_FBMEM (1) lives in the
+ * emulated FB shadow; targets 2/3 are guest sysmem. The instance table
+ * stores vidmem offsets, but accept a physical FB address by rebasing it
+ * on vgpu->fb_base when the direct interpretation is out of range.
+ */
+static bool gsp_disp_fb_off(struct nv_vgpu_device *vgpu, u64 pa, size_t len,
+ u64 *off)
+{
+ u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
+
+ if (!vgpu->bars[1].vaddr)
+ return false;
+ if (pa + len <= fb_size) {
+ *off = pa;
+ return true;
+ }
+ if (vgpu->fb_base && pa >= vgpu->fb_base &&
+ pa - vgpu->fb_base + len <= fb_size) {
+ *off = pa - vgpu->fb_base;
+ return true;
+ }
+ return false;
+}
+
+static int gsp_disp_notifier_read(struct nv_vgpu_device *vgpu, u64 pa,
+ u32 target, void *buf, size_t len)
+{
+ u64 off;
+
+ if (target == 1) {
+ if (!gsp_disp_fb_off(vgpu, pa, len, &off))
+ return -ENOENT;
+ memcpy(buf, (u8 *)vgpu->bars[1].vaddr + off, len);
+ return 0;
+ }
+ return gsp_disp_read(vgpu, pa, buf, len);
+}
+
+static int gsp_disp_notifier_write(struct nv_vgpu_device *vgpu, u64 pa,
+ u32 target, void *buf, size_t len)
+{
+ u64 off;
+
+ if (target == 1) {
+ if (!gsp_disp_fb_off(vgpu, pa, len, &off))
+ return -ENOENT;
+ memcpy((u8 *)vgpu->bars[1].vaddr + off, buf, len);
+ return 0;
+ }
+ return gsp_disp_write(vgpu, pa, buf, len);
+}
+
/* BAR0 shadow access. The guest mmap writes it without our lock, so the
* register words are read with READ_ONCE; writes are single u32 stores. */
static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off)
@@ -313,7 +368,7 @@ static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle)
e < GSP_DISP_INST_HASH_END; e += 8) {
u32 obj = *(u32 *)(fb + blk + e);
u32 ctx = *(u32 *)(fb + blk + e + 4);
- u32 inst = (ctx >> 14) & 0x3ff;
+ u32 inst = (ctx >> 14) & 0x7ff;
u32 chn = (ctx >> 25) & 0x7f;
u64 ent;
@@ -407,8 +462,10 @@ void gsp_disp_engine_inst_mem(struct nv_vgpu_device *vgpu, u64 pa, u64 size,
* (word0 = ht_ObjectHandle, word1 = {chn[31:25], instance[24:14],
* client[13:0]}) + 32-byte object entries at instance*32 holding
* {target, base_lo, base_hi, limit_lo, limit_hi} in 256-byte units.
+ * *limit is the byte END address, matching the table encoding.
*/
-static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa, u32 *limit)
+static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa,
+ u64 *limit, u32 *target)
{
u64 e;
@@ -439,7 +496,8 @@ static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa, u32 *limit)
if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2)))
continue;
*pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
- *limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff;
+ *limit = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8;
+ *target = w0 & 0x3;
return 0;
}
return -ENOENT;
@@ -452,7 +510,8 @@ static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa, u32 *limit)
* by searching the shadow for the ctxDMA handle itself.
*/
static int gsp_disp_find_block_fb(struct nv_vgpu_device *vgpu, u32 handle,
- u64 *base, u64 *pa, u32 *limit)
+ u64 *base, u64 *pa, u64 *limit,
+ u32 *target)
{
u8 *fb = vgpu->bars[1].vaddr;
u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
@@ -467,7 +526,7 @@ static int gsp_disp_find_block_fb(struct nv_vgpu_device *vgpu, u32 handle,
if (a != handle)
continue;
if (!gsp_disp_parse_hash(handle, fb + (off & ~0xffffULL),
- pa, limit)) {
+ pa, limit, target)) {
*base = off & ~0xffffULL;
return 0;
}
@@ -477,7 +536,8 @@ static int gsp_disp_find_block_fb(struct nv_vgpu_device *vgpu, u32 handle,
/* Same search through guest RAM (the coherent-mapping fallback). */
static int gsp_disp_find_block_ram(struct nv_vgpu_device *vgpu, u32 handle,
- u64 *base, u64 *pa, u32 *limit)
+ u64 *base, u64 *pa, u64 *limit,
+ u32 *target)
{
u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
const size_t chunk = 1 << 20;
@@ -500,7 +560,7 @@ static int gsp_disp_find_block_ram(struct nv_vgpu_device *vgpu, u32 handle,
continue;
if (!gsp_disp_parse_hash(handle,
buf + (i & ~0xffffULL),
- pa, limit)) {
+ pa, limit, target)) {
*base = off + (i & ~0xffffULL);
ret = 0;
goto out;
@@ -513,15 +573,89 @@ out:
}
/*
+ * The guest RM writes the display instance block through its BAR2 aperture
+ * (memdescMapInternal -> BAR2), which vgsp exposes as a flat host-RAM
+ * shadow. BAR2 offsets are only 4 KiB-aligned, so locate the hash entry by
+ * handle and derive the block base from its position: the table is the
+ * first 8 KiB of the block, hence the entry lies in the 4 KiB page at
+ * block+0 or block+0x1000.
+ */
+static int gsp_disp_find_block_bar2(struct nv_vgpu_device *vgpu, u32 handle,
+ u64 *base, u64 *pa, u64 *limit,
+ u32 *target)
+{
+ u8 *bar2 = vgpu->bars[2].vaddr;
+ u64 size = vgpu->bars[2].size;
+ u64 s;
+
+ if (!bar2 || !size)
+ return -ENODEV;
+ for (s = 0; s + 8 <= size; s += 8) {
+ u32 obj, ctx, inst;
+ u64 cand[2];
+ int c;
+
+ memcpy(&obj, bar2 + s, 4);
+ if (obj != handle)
+ continue;
+ memcpy(&ctx, bar2 + s + 4, 4);
+ inst = (ctx >> 14) & 0x7ff;
+ if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
+ inst >= (GSP_DISP_INST_OBJ_END / 0x20))
+ continue;
+
+ cand[0] = s & ~0xfffULL;
+ cand[1] = cand[0] - 0x1000;
+ for (c = 0; c < 2; c++) {
+ u64 b = cand[c];
+ u64 ent, base64, lim64;
+ u32 w0, w1, w2, w3, w4, tgt;
+
+ if (s < b || s - b >= 0x2000)
+ continue;
+ ent = b + (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
+ if (ent + 5 * 4 > size)
+ continue;
+ memcpy(&w0, bar2 + ent + 0, 4);
+ memcpy(&w1, bar2 + ent + 4, 4);
+ memcpy(&w2, bar2 + ent + 8, 4);
+ memcpy(&w3, bar2 + ent + 12, 4);
+ memcpy(&w4, bar2 + ent + 16, 4);
+ tgt = w0 & 0x3;
+ if (tgt < 1 || tgt > 3)
+ continue;
+ base64 = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
+ lim64 = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8;
+ if (lim64 <= base64 || lim64 - base64 > (1 << 20))
+ continue;
+ *base = b;
+ *pa = base64;
+ *limit = lim64;
+ *target = tgt;
+ if (disp.im_logs < 8) {
+ disp.im_logs++;
+ pr_info("inst mem: handle 0x%x via bar2 block 0x%llx -> base=0x%llx limit=0x%llx target=%u\n",
+ handle, (unsigned long long)b,
+ (unsigned long long)base64,
+ (unsigned long long)lim64, tgt);
+ }
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+/*
* Resolve a context-DMA handle to its surface guest PA. Prefers the block
* announced by 0x20800a49; if the handle is not there (BAR1 aperture shift
- * / coherent mapping), searches the shadow and then guest RAM for the block
- * that actually contains it. Falls back to the legacy whole-FB scan when
- * the announce control was never seen.
- * Returns 0 on success and fills *pa, *limit (bytes-1).
+ * / coherent mapping / BAR2 write path), searches the shadow, then guest
+ * RAM, then the BAR2 shadow for the block that actually contains it. Falls
+ * back to the legacy whole-FB scan when the announce control was never seen.
+ * Returns 0 on success and fills *pa, *limit (byte end address) and
+ * *target (1 = FB, 2/3 = guest sysmem).
*/
static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
- u64 *pa, u32 *limit)
+ u64 *pa, u64 *limit, u32 *target)
{
u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
u8 *fb = vgpu->bars[1].vaddr;
@@ -538,7 +672,8 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
return -ENODEV;
if (gsp_disp_notifier_pa) {
*pa = gsp_disp_notifier_pa;
- *limit = 0xfff;
+ *limit = *pa + 0x1000;
+ *target = 2;
return 0;
}
@@ -564,7 +699,7 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
GSP_DISP_INST_OBJ_END))
return -ENOENT;
- if (!gsp_disp_parse_hash(handle, disp.ibuf, pa, limit)) {
+ if (!gsp_disp_parse_hash(handle, disp.ibuf, pa, limit, target)) {
if (gsp_disp_dbg && disp.im_logs < 8) {
disp.im_logs++;
pr_info("inst mem: handle 0x%x at announced pa=0x%llx space=%u\n",
@@ -593,11 +728,16 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
}
spin_unlock_irqrestore(&disp.lock, flags);
- if (!gsp_disp_find_block_fb(vgpu, handle, &base2, pa, limit)) {
+ if (!gsp_disp_find_block_fb(vgpu, handle, &base2, pa, limit,
+ target)) {
src = "shadow";
} else if (!gsp_disp_find_block_ram(vgpu, handle, &base2, pa,
- limit)) {
+ limit, target)) {
src = "ram";
+ } else if (!gsp_disp_find_block_bar2(vgpu, handle, &base2, pa,
+ limit, target)) {
+ /* The BAR2 scan emits its own single log line. */
+ return 0;
} else {
spin_lock_irqsave(&disp.lock, flags);
for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
@@ -629,8 +769,8 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
}
if (gsp_disp_dbg && disp.im_logs < 8) {
disp.im_logs++;
- pr_info("inst mem: handle 0x%x via %s block 0x%llx -> pa=0x%llx limit=0x%x\n",
- handle, src, base2, *pa, *limit);
+ pr_info("inst mem: handle 0x%x via %s block 0x%llx -> pa=0x%llx limit=0x%llx target=%u\n",
+ handle, src, base2, *pa, *limit, *target);
}
return 0;
}
@@ -645,8 +785,16 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
}
}
if (slot < 0 || !disp.im[slot].found) {
- if (gsp_disp_instmem_find(vgpu, handle))
+ if (gsp_disp_instmem_find(vgpu, handle)) {
+ /*
+ * The guest may write the instance block through its
+ * BAR2 aperture instead of the FB shadow.
+ */
+ if (!gsp_disp_find_block_bar2(vgpu, handle, &base, pa,
+ limit, target))
+ return 0;
return -ENOENT;
+ }
slot = -1;
for (i = 0; i < ARRAY_SIZE(disp.im); i++)
if (disp.im[i].vgpu == vgpu) {
@@ -665,7 +813,7 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
u64 e = base + i;
u32 obj = *(u32 *)(fb + e);
u32 ctx = *(u32 *)(fb + e + 4);
- u32 inst = (ctx >> 14) & 0x3ff;
+ u32 inst = (ctx >> 14) & 0x7ff;
u64 ent;
u32 w0, w1, w2, w3, w4;
@@ -683,7 +831,8 @@ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2)))
continue;
*pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
- *limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff;
+ *limit = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8;
+ *target = w0 & 0x3;
found = true;
break;
}
@@ -855,17 +1004,19 @@ static void gsp_disp_method(struct nv_vgpu_device *vgpu,
/* Resolve the surface now: the handle is the instance-table
* key and the surface is bound before any notifier wait. */
if (value && gsp_disp_finish) {
- u64 pa = 0;
- u32 lim = 0;
+ u64 pa = 0, lim = 0;
+ u32 target = 0;
- if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) {
+ if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim,
+ &target) == 0) {
ch->notifier_pa = pa;
+ ch->notifier_target = target;
ch->notifier_pa_valid = true;
}
}
- pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n",
+ pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx target=%u valid=%d\n",
ch->hclass, push_off, value, ch->notifier_pa,
- ch->notifier_pa_valid);
+ ch->notifier_target, ch->notifier_pa_valid);
return;
}
if (addr == notifier_ctrl) {
@@ -878,8 +1029,9 @@ static void gsp_disp_method(struct nv_vgpu_device *vgpu,
return;
}
if (addr == GSP_DISP_M_UPDATE) {
- u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl);
- u64 pa = ch->notifier_pa + (u64)off_words * 4;
+ u32 off_bytes = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl);
+ u64 pa = ch->notifier_pa + off_bytes;
+ u32 target = ch->notifier_target;
u32 finished = GSP_DISP_NOTIFIER_FINISHED;
bool armed = (ch->notifier_ctrl &
GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0;
@@ -897,22 +1049,25 @@ static void gsp_disp_method(struct nv_vgpu_device *vgpu,
* push buffer. Nothing else is ever written.
*/
if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) ||
- gsp_disp_read(vgpu, pa, &before, sizeof(before)) ||
+ gsp_disp_notifier_read(vgpu, pa, target, &before,
+ sizeof(before)) ||
(before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) {
if (ch->update_logs < 8) {
ch->update_logs++;
- pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n",
- ch->hclass, pa, before);
+ pr_warn("h=0x%x UPDATE notifier pa=0x%llx target=%u rejected (val=0x%x)\n",
+ ch->hclass, pa, target, before);
}
- } else if (gsp_disp_write(vgpu, pa, &finished,
- sizeof(finished)) == 0) {
+ } else if (gsp_disp_notifier_write(vgpu, pa, target,
+ &finished,
+ sizeof(finished)) == 0) {
ch->notifier_writes++;
wrote = true;
- gsp_disp_read(vgpu, pa, &after, sizeof(after));
+ gsp_disp_notifier_read(vgpu, pa, target, &after,
+ sizeof(after));
if (after != finished && ch->update_logs < 8) {
ch->update_logs++;
- pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n",
- ch->hclass, after, pa);
+ pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx target=%u\n",
+ ch->hclass, after, pa, target);
}
}
}
@@ -930,10 +1085,11 @@ static void gsp_disp_method(struct nv_vgpu_device *vgpu,
}
if (ch->update_logs < 8 || gsp_disp_dbg) {
ch->update_logs++;
- pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n",
+ pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u target=%u -> %s (posts=%u)\n",
ch->hclass, push_off, armed,
- ch->notifier_handle, ch->notifier_pa, off_words,
- wrote ? "FINISHED" : "none", ch->notifier_posts);
+ ch->notifier_handle, ch->notifier_pa, off_bytes,
+ target, wrote ? "FINISHED" : "none",
+ ch->notifier_posts);
}
}
}
@@ -1201,6 +1357,7 @@ void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu)
c->notifier_handle = s->notifier_handle;
c->notifier_ctrl = s->notifier_ctrl;
c->notifier_pa = s->notifier_pa;
+ c->notifier_target = s->notifier_target;
c->notifier_pa_valid = s->notifier_pa_valid;
c->notifier_writes = s->notifier_writes;
c->notifier_posts = s->notifier_posts;
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-08 23:55:45 +0000