diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 40 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 279 |
2 files changed, 303 insertions, 16 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 85bdb89..4e6acc5 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1428,6 +1428,17 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_PM_FB 0xcaf000b8u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u +/* + * Mode 35, graphics-execution plan §F first experiment: the TURING_A + * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7 + * user-lane channel AFTER its promote - the missing half the mode-35 + * capture never provided ("mode 35: ... the capture carries no 0xc597 on + * the user lane"). Best-effort like the TURING_COMPUTE_A: a rejection is + * logged and the lane keeps running; a construction on V7 must not hit + * the wired-golden deadlock class (m3a-m3g, that channel). 0xcaf00094 is + * free between 0xcaf00090 and 0xcaf000a0. + */ +#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D 0xcaf00094u /* * Mode-35 V7 channel flags/privilege and fault/PM geometry, decoded from the @@ -14778,6 +14789,35 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ; } + /* + * Graphics-execution plan §F first 3D-lane experiment: + * construct the TURING_A (0xc597) class object on V7 + * after the accepted promote, through the same generic + * alloc helper the golden lanes use. The plan's open + * question is whether a 0xc597 construct on the + * execution lane is accepted without the wired-golden + * deadlock class; the result line below is the + * experiment's evidence either way, and the construct + * never gates the lane. + */ + { + u32 g3_rpc = 0, g3_status = 0; + int g3_ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, + GSP_PHYS_RMCALL_CLASS_GR_3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + "mode 35 TURING_A on V7 (plan-F first 3D-lane experiment, after promote)", + &g3_rpc, &g3_status); + + pr_info("vgsp-boot: rm-call: mode 35 plan-F experiment: TURING_A (0xc597) on V7 hChannel=0x%x -> ret=%d hObject=0x%x status=0x%x rpc_result=0x%x (%s)\n", + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + g3_ret, + g3_ret ? 0 : + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + g3_status, g3_rpc, + g3_ret ? "rejected/absent; lane continues" : + "accepted; V7 carries a 3D class object for the method-stream replay"); + } pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", boot_stage.rmcall_channel, GSP_PHYS_RMCALL_HANDLE_USER_LANE, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3581ce9..22e7be1 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6992,6 +6992,47 @@ struct svc_gl_sec { u32 blob_len; }; static struct svc_gl_sec svc_gl; /* service thread only */ +/* + * SESSION 153 (G-B gate: the client's own C597 draws must open sections). + * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which + * returns true when the push VA sits in the GR candidate ring - even when + * no GL section was ever captured for that push. The GL client's draw + * pushes are admitted as GR candidates during driver init (they are read + * as ordinary GR pushes first), so every later replay of them is rejected + * as "replay already captured" and no GL section ever opens. Track GL + * section captures in their own table: a replay only skips when THIS table + * holds the exact (push_va, content hash) pair. The hash keeps a later, + * different push at the same VA (a frame loop: same slot, new content) + * capturable, matching the r383 reasoning for the GR seen table. + */ +#define SVC_GL_CAP_HIST 256 +static struct { + u64 push_va; + u64 hash; +} svc_gl_cap_hist[SVC_GL_CAP_HIST]; +static u32 svc_gl_cap_next; +static u32 svc_gl_cap_seen_log; + +static bool svc_gl_cap_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GL_CAP_HIST; i++) + if (svc_gl_cap_hist[i].push_va == push_va && + svc_gl_cap_hist[i].hash == hash) + return true; + return false; +} + +static void svc_gl_cap_store(u64 push_va, u64 hash) +{ + u32 i = svc_gl_cap_next++ % SVC_GL_CAP_HIST; + + svc_gl_cap_hist[i].push_va = push_va; + svc_gl_cap_hist[i].hash = hash; +} static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ @@ -7042,6 +7083,97 @@ static void svc_gl_prog_apply_param(void) pr_info("sema-gl: prog-hex param: %u bytes stored\n", svc_gl_prog_len); } + +/* + * SESSION 153 (GL client draw sections): the GL client's own per-frame + * pushes bind TURING_A (0xC597) exactly like the X server's one-shot + * section, but carry no program/param/release operands at all (measured: + * `0001fff0 20010000 0000c597 0001fff0 20016043 00000000 20016040 + * 00000000` on h=0xbfef0002 - the C597 SET_OBJECT plus 2D-class + * NoOp/Notify only). The old flush guard required the full operand trio + * and silently dropped every client section, so no client capture ever + * appeared. Two additive sources close that gap: + * + * 1. a per-channel operand HOLD: the last valid prog/param/rel operands + * seen in a section on this channel are remembered and adopted by a + * later draw section on the same channel (still marked src=hold); + * 2. a dev fallback (gsp_sema_gl_draw_synth, default 1): when the client + * never supplied operands at all, the draw section captures with the + * operand values the X server's one-shot GL section supplied in the + * same boot class (r891/r923). Every such capture is marked + * src=draw-synth in the witness so it is never mistaken for a + * stream-derived capture. + */ +#define SVC_GL_HOLD_MAX 8 +struct svc_gl_hold { + u32 h; + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; +}; +static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; +static u32 svc_gl_open_log; /* bounded section-open witness */ +static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static bool gsp_sema_gl_draw_synth = true; +module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); +MODULE_PARM_DESC(sema_gl_draw_synth, + "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 1; witness marks src=draw-synth)"); +#define SVC_GL_SYNTH_PROG_VA 0x121201b00ULL +#define SVC_GL_SYNTH_PARAM_VA 0x1214fa220ULL +#define SVC_GL_SYNTH_PARAM_SIZE 0x10u +#define SVC_GL_SYNTH_REL_VA 0x120001000ULL +#define SVC_GL_SYNTH_REL_PAY 1u + +static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, + u64 rel, u32 rpay) +{ + u32 i, free = SVC_GL_HOLD_MAX; + + if (!h || !prog || !param || !rel) + return; + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h == h) { + free = i; + break; + } + if (!svc_gl_hold_tab[i].h && free == SVC_GL_HOLD_MAX) + free = i; + } + if (free == SVC_GL_HOLD_MAX) + free = h % SVC_GL_HOLD_MAX; + svc_gl_hold_tab[free].h = h; + svc_gl_hold_tab[free].prog_va = prog; + svc_gl_hold_tab[free].param_va = param; + svc_gl_hold_tab[free].param_size = psize; + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; +} + +static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, + u64 *rel, u32 *rpay) +{ + u32 i; + + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h != h || !h) + continue; + if (prog) + *prog = svc_gl_hold_tab[i].prog_va; + if (param) + *param = svc_gl_hold_tab[i].param_va; + if (psize) + *psize = svc_gl_hold_tab[i].param_size; + if (rel) + *rel = svc_gl_hold_tab[i].rel_va; + if (rpay) + *rpay = svc_gl_hold_tab[i].rel_payload; + return true; + } + return false; +} + /* * SESSION 149 R17 (M4): standalone GL-class upload tracker. A context-init * upload push (the one-shot program upload) is NOT required to sit inside a @@ -7208,20 +7340,78 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, int blob_rc = -1; bool blob_trusted = false; const char *blob_src = "none"; + const char *op_src = "stream"; svc_gl_prog_apply_param(); - if (!svc_gl.seen || !svc_gl.prog_va || !svc_gl.param_va || - !svc_gl.rel_va || !trusted) + if (!svc_gl.seen || !trusted) goto out; + { + u64 hp = 0, hm = 0, hr = 0; + u32 hms = 0, hrp = 0; + bool held = false; + + if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { + held = svc_gl_hold_load(ch->h_object, &hp, &hm, &hms, + &hr, &hrp); + if (!svc_gl.prog_va && held) { + svc_gl.prog_va = hp; + op_src = "hold"; + } + if (!svc_gl.param_va && held) { + svc_gl.param_va = hm; + svc_gl.param_size = hms; + op_src = "hold"; + } + if (!svc_gl.rel_va && held) { + svc_gl.rel_va = hr; + svc_gl.rel_payload = hrp; + op_src = "hold"; + } + if ((!svc_gl.prog_va || !svc_gl.param_va) && + gsp_sema_gl_draw_synth && !ch->engine_type) { + if (!svc_gl.prog_va) + svc_gl.prog_va = SVC_GL_SYNTH_PROG_VA; + if (!svc_gl.param_va) { + svc_gl.param_va = SVC_GL_SYNTH_PARAM_VA; + svc_gl.param_size = + SVC_GL_SYNTH_PARAM_SIZE; + } + op_src = "draw-synth"; + } + if (!svc_gl.rel_va && gsp_sema_gl_draw_synth && + !ch->engine_type) { + svc_gl.rel_va = SVC_GL_SYNTH_REL_VA; + svc_gl.rel_payload = SVC_GL_SYNTH_REL_PAY; + op_src = "draw-synth"; + } + if (!svc_gl.prog_va || !svc_gl.param_va || + !svc_gl.rel_va) { + if (gsp_sema_dbg && svc_gl_skip_log < 16) { + svc_gl_skip_log++; + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va); + } + goto out; + } + } + } /* * SESSION 149 R15 (M1.1a): admit a rewind replay when the same slot * re-read valid bytes whose content hash matches the recorded pass * (the pump sets cap_trust only for such a hash-verified replay; see - * the trust admission at the decode call site). The session-wide - * (push_va, hash) capture history is the exactly-once guard: a push - * already captured/forwarded makes every later replay a no-op. - */ - if (replay && svc_gr_cand_push_seen(push_va, svc_push_hash(buf, len))) { + * the trust admission at the decode call site). The exactly-once guard + * for GL capture is GL-specific (SESSION 153): svc_gr_cand_push_seen() + * also matches the GR candidate ring and the session seen table, and + * the GL client's C597 draw pushes are admitted as GR candidates + * during driver init - so using it here suppressed every later GL + * section capture on those push VAs ("replay already captured" for + * every client draw). svc_gl_cap_seen() only matches pushes whose GL + * section was actually captured, with the content hash so a later + * different push in the same slot (a frame loop) stays capturable. + */ + if (replay && svc_gl_cap_seen(push_va, svc_push_hash(buf, len))) { if (svc_gl_rep_log < 16) { svc_gl_rep_log++; pr_info("sema-gl: replay already captured h=0x%x push=0x%llx (no re-capture)\n", @@ -7376,11 +7566,26 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, memcpy(pr.data, blob, blob_n); n_recs = 1; } - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u\n", + /* + * SESSION 153: record this GL section capture in the GL-specific + * exactly-once table so only an actual capture suppresses later + * replays of the same (push_va, content hash). + */ + svc_gl_cap_store(push_va, svc_push_hash(buf, len)); + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u); + replay ? 1u : 0u, op_src); + /* + * SESSION 153: remember real operands for this channel so a later + * draw-only section on the same channel can still capture. A + * draw-synth capture carries no channel truth and is never stored. + */ + if (op_src[0] != 'd') + svc_gl_hold_store(ch->h_object, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, + svc_gl.rel_va, svc_gl.rel_payload); /* R16 bounded witness: what the synthesized QMD now schedules and * where the code bytes came from. */ if (gsp_sema_dbg && svc_gl_prog_log < 8) { @@ -11440,26 +11645,50 @@ static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the * driver relied on is only provable by reading the push's own methods. * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + * SESSION 153: gsp_sema_push_raw_small extends the same dump to the GL + * client's small pushes (16..511 B - the C597 draw/notify pushes are 32/64 B + * and were excluded by the >= 512 gate), with its own budget and a per + * (VA,len) once-only guard so a re-read of the same push shape does not + * flood. */ unsigned gsp_sema_push_raw; +unsigned gsp_sema_push_raw_small; +module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); +MODULE_PARM_DESC(sema_push_raw_small, + "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); static u64 svc_pushraw_last_va; +static u32 svc_pushraw_last_len; static u32 svc_pushraw_n; +static u32 svc_pushraw_small_n; static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { u32 off = 0, gi = 0; - if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) - return; - if (len < 512 || len > 8192) + if (len > 8192) return; - if (push_va == svc_pushraw_last_va) + if (len < 512) { + if (!gsp_sema_push_raw_small || + svc_pushraw_small_n >= gsp_sema_push_raw_small) + return; + } else { + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + } + if (push_va == svc_pushraw_last_va && len == svc_pushraw_last_len) return; svc_pushraw_last_va = push_va; - svc_pushraw_n++; - pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", - h_object, push_va, len, svc_pushraw_n); + svc_pushraw_last_len = len; + if (len < 512) { + svc_pushraw_small_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u (small)\n", + h_object, push_va, len, svc_pushraw_small_n); + } else { + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + } while (off + 4 <= len && gi < 160) { u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; @@ -11706,9 +11935,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* * r481: verbatim method-stream dump for large GR-class pushes * (default 0 = off). See svc_push_raw_dump. + * SESSION 153: the small-push variant (sema_push_raw_small) covers the + * GL client's 16..511 B C597 pushes with its own budget. */ if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) svc_push_raw_dump(buf, len, push_va, ch->h_object); + else if (gsp_sema_push_raw_small && ch && !ch->engine_type && + len >= 16 && len < 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -12152,6 +12386,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_gl.subch = subch; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* + * SESSION 153: witness every C597 section open + * (bounded) - it is the recognizer-side proof + * that the client's own draw pushes are seen; + * the capture itself is witnessed by + * "sema-gl: section captured". + */ + if (gsp_sema_dbg && svc_gl_open_log < 32) { + svc_gl_open_log++; + pr_info("sema-gl: section open h=0x%x push=0x%llx subch=%u trusted=%d\n", + ch->h_object, push_va, subch, + trusted); + } } else if (svc_gl.seen) { svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, replay, off); |