diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 233 |
1 files changed, 220 insertions, 13 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1f89029..abb865d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -9215,6 +9215,13 @@ struct svc_gr_cand_rec { */ bool stream_chunked; bool entry_done; + /* + * eng89o: this slice carries the window's DRAW CALL (CALL_MME_MACRO + * slot 34, method 0x3910, or CALL_MME_DATA 0x3914). The chunker forces + * the slice to start at the draw run header and stamps order 0 so the + * pump offers it before every other slice of the window. + */ + bool draw_slice; u64 order; /* * SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair @@ -11897,11 +11904,132 @@ static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, return last; } +/* + * eng89o: one-shot-per-push census of the DRAW CALL method over the WHOLE + * admitted window (and, when absent there, the whole push buffer). + * + * The chunk walker and svc_gl_chunk_has_draw_ia() only understood op 1/3/5 + * runs, so a DRAW encoded in an op 0/2/4 header - the other method-bearing + * opcodes - was invisible and no slice was ever named draw=1. Walk the real + * grammar: op 0/1/3 are incrementing method runs (method + 4 bytes per data + * word), op 2 is a non-incrementing run (every data word hits m0), op 4 is + * an immediate (one data word at m0), op 5 is a single-method increment; + * op 6/7 carry no method. Log every 0x3910/0x3914 occurrence with its word + * offset and one DRAW-SCAN summary so a capture can prove whether the method + * is absent, mis-scanned, or outside the admitted window. + */ +static u32 svc_gl_draw_scan_log; +static u64 svc_gl_draw_scan_seen[16]; +static u32 svc_gl_draw_scan_pos; + +static bool svc_gl_draw_scan_new_push(u64 push_va) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_gl_draw_scan_seen); i++) + if (svc_gl_draw_scan_seen[i] == push_va) + return false; + svc_gl_draw_scan_seen[svc_gl_draw_scan_pos++ % + ARRAY_SIZE(svc_gl_draw_scan_seen)] = push_va; + return true; +} + +static u32 svc_gl_draw_walk(const u32 *win, u32 n, u32 base_word, + u32 *at, u32 *span_out, u32 *runs_out, + bool log_it, struct svc_channel *ch, u64 push_va) +{ + u32 r = 0, runs = 0, found = 0; + + *at = 0; + *span_out = 0; + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 span = 1u, k; + u32 m0 = (h & 0xfffu) << 2; + bool incr = (op == 0u || op == 1u || op == 3u); + + if ((op <= 3u || op == 5u) && cnt && r + 1u + cnt <= n) + span = 1u + cnt; + runs++; + for (k = 0; k < span; k++) { + u32 m; + + if (k && !incr) + break; + m = m0 + 4u * k; + if (m != 0x3910u && m != 0x3914u) + continue; + if (!found) { + *at = r; + *span_out = span; + } + found++; + if (log_it && svc_gl_draw_scan_log < 8u) { + svc_gl_draw_scan_log++; + pr_info("sema-gl: DRAW-HIT h=0x%x push=0x%llx off=%u m=0x%x op=%u cnt=%u span=%u\n", + ch->h_object, push_va, + base_word + r, m, op, cnt, span); + } + } + r += span; + } + *runs_out = runs; + return found; +} + +static void svc_gl_draw_scan(struct svc_channel *ch, const u8 *buf, u32 len, + u32 start_off, u32 at_off, u32 *draw_rel, + u32 *draw_span) +{ + u32 win_words, push_words, at = 0, span = 0, runs = 0, found; + u32 at2 = 0, span2 = 0, runs2 = 0; + u64 push_va = svc_gr_cand.push_va; + bool log_it, outside = false; + + *draw_rel = 0; + *draw_span = 0; + log_it = svc_gl_draw_scan_new_push(push_va); + win_words = at_off > start_off ? (at_off - start_off) / 4u : 0u; + push_words = len / 4u; + found = 0; + if (win_words) + found = svc_gl_draw_walk((const u32 *)(buf + start_off), + win_words, start_off / 4u, &at, &span, + &runs, log_it, ch, push_va); + if (found) { + *draw_rel = at; + *draw_span = span; + } else if (push_words) { + /* Not in the admitted window: census the whole push so the + * receipt can say "present but outside [start_off, at_off)". */ + found = svc_gl_draw_walk((const u32 *)buf, push_words, 0, + &at2, &span2, &runs2, log_it, ch, + push_va); + if (found) { + outside = true; + at = at2; + span = span2; + runs = runs2; + } + } + if (log_it && svc_gl_draw_scan_log < 64u) { + svc_gl_draw_scan_log++; + pr_info("sema-gl: DRAW-SCAN h=0x%x push=0x%llx win_off=%u win_words=%u found=%u at=%u runs=%u%s\n", + ch->h_object, push_va, start_off / 4u, win_words, + found, at, runs, + outside ? " found_outside=1" : ""); + } +} + static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, u32 len, u32 at_off) { u32 cur = svc_gl.start_off; u32 n = 0, pending = 0, ri; + u32 draw_rel = 0, draw_span = 0, draw_at = 0; + bool have_draw = false; if (!svc_gr_cand.valid) return; @@ -11929,15 +12057,48 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, svc_gr_cand.valid = false; return; } + /* + * eng89o: locate the DRAW CALL before slicing. The census walks the + * whole admitted window (falling back to the whole push) with the full + * method grammar, so the draw slice can be forced and launched first. + */ + svc_gl_draw_scan(ch, buf, len, svc_gl.start_off, at_off, + &draw_rel, &draw_span); + if (draw_span && svc_gl.start_off + draw_rel * 4u < at_off) { + draw_at = svc_gl.start_off + draw_rel * 4u; + have_draw = true; + } while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { u32 cw = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; u32 end, rel, nw; + bool is_draw = false; struct svc_gr_cand_rec *c = &svc_gr_cand; if (cw > 448u) cw = 448u; end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + /* + * eng89o: force a chunk boundary around the DRAW run so one + * slice starts at its header and contains its full op span. + * A chunk that would straddle the draw is cut before it; the + * next iteration starts exactly at the run header. + */ + if (have_draw) { + if (cur < draw_at && draw_at < end) { + end = draw_at; + } else if (cur == draw_at) { + u32 dend = draw_at + draw_span * 4u; + + if (dend > at_off) + dend = at_off; + if (dend > len) + dend = len; + if (dend > cur) + end = dend; + is_draw = true; + } + } rel = (cur - svc_gl.start_off) / 4u; nw = (end - cur) / 4u; @@ -11966,6 +12127,12 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, } } if (have) { + /* eng89o: a re-decode must keep the draw slice's + * forced-first identity, not just its bytes. */ + if (is_draw) { + svc_gr_cand_ring[ri].draw_slice = true; + svc_gr_cand_ring[ri].order = 0; + } n++; cur = end; continue; @@ -11998,6 +12165,14 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, if (!c->order) c->order = ++svc_gr_cand_order; } + /* + * eng89o: the draw slice is stamped order 0 so the chunk pump + * (lowest order first) offers it before every other slice of + * the window; the receipt then shows draw=1 on that launch. + */ + c->draw_slice = is_draw; + if (is_draw) + c->order = 0; c->valid = true; c->offered = false; c->stream_3d = true; @@ -12072,6 +12247,11 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, * same run grammar as the booter's * gsp_phys_gr_mme_window_has_draw_ia(): op 1/3 carry 1+cnt words and walk * methods past the header, op 5 is a single-method increment. + * + * eng89o: the draw can also arrive as CALL_MME_DATA (0x3914) and the client + * may encode the run with op 0 (incrementing) or op 2 (non-incrementing) + * headers, which the original walker treated as one-word resyncs and thus + * never scanned. Accept every method-bearing opcode (0..5) here. */ static bool svc_gl_chunk_has_draw_ia(const u32 *win, u32 n) { @@ -12082,14 +12262,15 @@ static bool svc_gl_chunk_has_draw_ia(const u32 *win, u32 n) u32 op = (h >> 29) & 7u; u32 cnt = (h >> 16) & 0x1fffu; u32 span = 1u, k, m0 = (h & 0xfffu) << 2; + bool incr = (op == 0u || op == 1u || op == 3u); - if ((op == 1u || op == 3u || op == 5u) && cnt && - r + 1u + cnt <= n) + if ((op <= 3u || op == 5u) && cnt && r + 1u + cnt <= n) span = 1u + cnt; for (k = 0; k < span; k++) { - if (k && op != 1u && op != 3u) + if (k && !incr) break; - if (m0 + 4u * k == 0x3910u) + if (m0 + 4u * k == 0x3910u || + m0 + 4u * k == 0x3914u) return true; } r += span; @@ -12116,8 +12297,12 @@ static void svc_gl_chunk_launch_note(const struct svc_gr_cand_rec *c) n = c->stream_n; total = c->push_len / 4u; base = total > c->push_n ? total - c->push_n : 0; - if (n && off >= base && off - base + n <= c->push_n && - svc_gl_chunk_has_draw_ia(c->raw + (off - base), n)) + /* eng89o: the chunker's forced draw slice is authoritative; the scan + * remains the fallback for captures that were not re-sliced. */ + if (c->draw_slice) + draw = 1; + else if (n && off >= base && off - base + n <= c->push_n && + svc_gl_chunk_has_draw_ia(c->raw + (off - base), n)) draw = 1; svc_gl_chunk_launch_log++; pr_info("sema-gl: 3D chunk launch h=0x%x push=0x%llx off=%u n=%u draw=%u\n", @@ -43633,16 +43818,38 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * slots, select THIS channel's oldest unconsumed candidate first. */ if (gsp_phys_gr_forward_enabled()) { - u32 ri; + u32 ri, draw = SVC_GR_CAND_RING; + /* + * eng89o: the forced draw slice (order 0) is offered before + * any other slice of this channel, so the DRAW CALL reaches + * the engine on the first launch of the admitted window even + * when the entry path would otherwise pick the base capture. + */ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { - u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[ri]; - if (svc_gr_cand_ring[rp].valid && - !svc_gr_cand_ring[rp].offered && - svc_gr_cand_ring[rp].h_object == ch->h_object) { - svc_gr_cand_pos = rp; - break; + if (!c->valid || c->offered || !c->draw_slice || + c->h_object != ch->h_object) + continue; + if (draw == SVC_GR_CAND_RING || + c->order < svc_gr_cand_ring[draw].order) + draw = ri; + } + if (draw != SVC_GR_CAND_RING) { + svc_gr_cand_pos = draw; + } else { + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (svc_gr_cand_ring[rp].valid && + !svc_gr_cand_ring[rp].offered && + svc_gr_cand_ring[rp].h_object == + ch->h_object) { + svc_gr_cand_pos = rp; + break; + } } } } |
