diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 121 |
1 files changed, 103 insertions, 18 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1add702..a62e3e4 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -212,6 +212,18 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; + /* Content witness for slot_ops_ident: the + * FNV-1a hash of the push bytes the identity + * was recorded from. Entry words alone are + * not enough - a client that rewrites its + * push buffer in place reuses the same (VA, + * length) words, and suppressing that new + * entry's ops loses its work (observed: a + * suppressed rewritten push left a staged + * chunk overwritten and the vadd input + * shifted by one chunk). Suppression requires + * the content to match too. */ u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ @@ -461,6 +473,7 @@ struct sema_svc { u32 op_trace_log; u64 ring_ops; /* op runs from ring-decoded pushes */ u64 ops_once_skipped; /* ops skipped by the done bitmap on a retried entry */ + u64 ops_reident; /* suppressions cancelled by a content mismatch */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 rescue_rej_log; /* rescue rejection budget */ u64 sem256_log; /* 256-byte op trace budget */ @@ -2723,10 +2736,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, - svc.ops_once_skipped); + svc.ops_once_skipped, svc.ops_reident); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback); @@ -6652,6 +6665,10 @@ u64 gsp_sema_copy_dst2; * copies the client will read back) must run before the release the client * synchronizes on; the translation usually lands within a service pass, so * keep the entry pending inside the window instead of dropping the work. + * The window stays generous on purpose: a faked entry's work is never run + * (running it late, after the client refilled the staging buffer, would + * write the next chunk's data over a finished one), so the hold is the only + * chance the work has. */ unsigned gsp_sema_ce_grace_ms = 8000; @@ -7610,6 +7627,25 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) test_bit(idx, ch->slot_done_epoch); } +/* + * FNV-1a over the raw push bytes: the content witness that lets the + * once-only op gate tell a true rewind replay (identical bytes) from a + * rewritten push that happens to carry the same entry words (same push-buffer + * VA and length, new methods). Entry words alone cannot separate the two, and + * suppressing a rewritten push's ops loses its work. + */ +static u64 svc_push_hash(const u8 *p, u32 len) +{ + u64 h = 0xcbf29ce484222325ULL; + u32 i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + /* ------------------------------------------------------------------ */ /* P5/P6b wave-1 shadow work forwarding */ /* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data) */ @@ -9158,6 +9194,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * A pin-batch rewind replays recently consumed + * entries. Re-processing one (rescue, grace) costs + * seconds and delivers nothing, and a slot consumed + * without running its work (the grace fake) must + * never execute that work later: by the time the + * rewind re-reads it the client has been told the + * release completed and may have refilled the + * staging buffer, so a late copy writes the next + * chunk's data over a finished one (observed: + * deterministic vadd input shift). The epoch mark is + * the authority - it is set on every consume path + * and cleared only when put wraps - so a consumed + * slot is skipped regardless of the channel's + * deferral state. defer_since must not re-open it. + */ + if (ch->engine_type && svc_replay_seen(ch, idx, put)) { + ch->last_get = (idx + 1) % + ch->gpfifo_entries; + svc.gpfifo_skipped++; + continue; + } + + /* * Remember the payload this entry should deliver the * first time it is examined. If the pool advances past * it later, the release was already delivered (the @@ -9166,23 +9225,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * consumed instead of waiting out the grace window - * otherwise every push costs one full grace period. */ - if (!ch->defer_since) { + if (!ch->defer_since) ch->entry_want = ch->expected_sem_payload; - /* - * A pin-batch rewind replays recently consumed - * entries. Re-processing one (rescue, grace) - * costs seconds and delivers nothing: consume - * it at once when the producing put has not - * advanced since it was consumed. - */ - if (ch->engine_type && - svc_replay_seen(ch, idx, put)) { - ch->last_get = (idx + 1) % - ch->gpfifo_entries; - svc.gpfifo_skipped++; - continue; - } - } u32 rel_n_iter = svc_rel_count; @@ -9392,6 +9436,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; /* + * The entry words repeated, but the + * push bytes may not have: a client + * that rewrites its push buffer in + * place produces a new push with the + * same (VA, length) words. Suppress + * ops only when the content matches + * the recorded pass; a changed page + * is a new entry and its ops must + * run (observed: a suppressed reread + * of a rewritten push left a staged + * chunk overwritten, shifting the + * vadd input by one chunk). + */ + if (entry_ident && + (entry_replay || entry_done) && + svc_push_hash(pb, len) != + ch->slot_ops_hash[idx]) { + entry_replay = false; + entry_done = 0; + svc.ops_reident++; + } + /* * Fills/copies execute only for CE * channels: the push was written for the * copy engine, so its method space is @@ -9438,6 +9504,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->slot_ops_complete[idx] = svc.op_tr_pending ? 0 : 1; + ch->slot_ops_hash[idx] = + svc_push_hash(pb, + len); } /* * An operand of this push @@ -9972,6 +10041,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } ch->rescue_fail = 0; ch->defer_since = 0; + /* + * Mark the slot consumed in this + * put epoch before advancing: the + * work was not run (the release was + * faked), so a later pin-batch + * rewind must skip the slot instead + * of executing the work against a + * staging buffer the client has + * since refilled. Missing this mark + * was the deterministic vadd input + * shift. + */ + svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; @@ -10179,6 +10261,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(svc.chan[k].slot_ops_complete, ch->slot_ops_complete, sizeof(ch->slot_ops_complete)); + memcpy(svc.chan[k].slot_ops_hash, + ch->slot_ops_hash, + sizeof(ch->slot_ops_hash)); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives |