diff options
Diffstat (limited to 'wiki/Migration-Protocol.md')
| -rw-r--r-- | wiki/Migration-Protocol.md | 395 |
1 files changed, 395 insertions, 0 deletions
diff --git a/wiki/Migration-Protocol.md b/wiki/Migration-Protocol.md new file mode 100644 index 0000000..4fbc879 --- /dev/null +++ b/wiki/Migration-Protocol.md @@ -0,0 +1,395 @@ +# Migration Protocol (RPC 0x0B) + +## Overview +The MIGRATE RPC (type 0x0B) implements live migration for vGPU virtual functions. It uses **dual jump tables** for two-phase migration operations, accessed via the primary dispatcher at 0x057A1AC0. + +## Key Addresses + +### Migration Jump Tables + +#### Table 1: 0x04179ED8 (Primary MIGRATE branch at 0x057A1BDC) +| Index | Target | Decompiled | Description | +|-------|--------|------------|-------------| +| 0 | 0x057A2344 | ✅ | Phase 1: Prepare - returns ID 0x14, checks flag at +0x7F9 | +| 1 | 0x057A2320 | ✅ | Phase 1: Prepare variant | +| 2 | 0x057A230C | ✅ | Phase 1: Prepare variant | +| 3 | 0x057A22F8 | ✅ | Phase 1: Prepare - returns ID 0x10 | +| 4 | 0x057A2388 | ✅ | Phase 1: Prepare variant | +| 5 | 0x057A23D8 | ✅ | Phase 1: Prepare variant | +| 6 | 0x057A23AC | ✅ | Phase 1: Prepare variant | +| 7 | 0x057A2478 | ✅ | Phase 1: Prepare variant | +| 8 | 0x057A2428 | ✅ | Phase 1: Prepare variant | +| 9 | 0x057A2454 | ✅ | Phase 1: Prepare variant | +| 10 | 0x057A2404 | ✅ | Phase 1: Prepare variant | +| 11 | 0x057A2364 | ✅ | Phase 1: Prepare variant | +| 12 | 0x057A1F3C | (dup) | Duplicate entry | +| 13 | 0x057A299C | — | Phase 2: Transfer | +| 14 | 0x057A1F3C | (dup) | Duplicate entry | +| 15 | 0x057A1F3C | (dup) | Duplicate entry | + +#### Table 2: 0x04179FBC (Primary MIGRATE branch at 0x057A1D5C) +| Index | Target | Decompiled | Description | +|-------|--------|------------|-------------| +| 0 | 0x057A257C | ✅ | Phase 2: Transfer | +| 1 | 0x057A25E4 | ✅ | Phase 2: Transfer | +| 2 | 0x057A24EC | ✅ | Phase 2: Transfer | +| 3 | 0x057A2608 | ✅ | Phase 2: Transfer | +| 4 | 0x057A2534 | ✅ | Phase 2: Transfer | +| 5 | 0x057A25C0 | ✅ | Phase 2: Transfer | +| 6 | 0x057A24A4 | ✅ | Phase 2: Transfer | +| 7 | 0x057A259C | ✅ | Phase 2: Transfer | +| 8 | 0x057A2510 | ✅ | Phase 2: Transfer | +| 9 | 0x057A2558 | ✅ | Phase 2: Transfer | +| 10 | 0x057A24C8 | ✅ | Phase 2: Transfer | +| 11 | 0x057A262C | ✅ | Phase 2: Transfer | +| 12 | 0x057A345C | ✅ | Phase 3: Commit | +| 13 | 0x057A36D8 | ✅ | Phase 3: Commit | +| 14 | 0x057A35D8 | ✅ | Phase 3: Commit | +| 15 | 0x057A3658 | ✅ | Phase 3: Commit | + +## Migration Phases + +### Phase 0: Discovery (Pre-migration) +```c +// Query VF migratability +int gsp_migrate_query(struct gsp_vf *vf, struct migrate_query *out) { + migrate_payload_t payload = { + .phase = MIGRATE_PHASE_QUERY, + .subcmd = 0, + .src_vf = vf->id, + .dst_vf = 0, + }; + return gsp_rpc_send(vf->gsp, RPC_MIGRATE, &payload, sizeof(payload), out, sizeof(*out)); +} +``` + +### Phase 1: Prepare (Table 1, indices 0-11) +**Goal**: Quiesce VF, prepare memory state for transfer + +```c +// Subcmd 3 (0x057A22F8) - Returns ID 0x10 +int migrate_prepare_quiesce(vf_id) { + // 1. Pause VF scheduling + // 2. Flush pending commands + // 3. Pin all VF memory + // 4. Return quiesce token +} + +// Subcmd 0 (0x057A2344) - Returns ID 0x14, checks flag at in_t1+0x7F9 +int migrate_prepare_snapshot(vf_id, *token) { + // 1. Snapshot VF register state + // 2. Snapshot BAR configurations + // 3. Snapshot WPR regions + // 4. Snapshot page tables + // 5. Return snapshot handle +} +``` + +### Phase 2: Transfer (Table 1 index 13, Table 2 indices 0-11) +**Goal**: Transfer memory pages to destination + +```c +// Table 2, subcmd 0 (0x057A257C) +int migrate_transfer_pages(vf_id, page_list, *progress) { + // 1. Iterate page list + // 2. For each page: + // a. Copy via DMA (0x0588F9C4) + // b. Verify checksum + // c. Mark page dirty if modified + // 3. Return bytes transferred +} + +// Table 2, subcmd 11 (0x057A262C) - Dirty page tracking +int migrate_track_dirty(vf_id, *dirty_bitmap) { + // Return bitmap of pages dirtied since last transfer +} +``` + +### Phase 3: Commit (Table 2 indices 12-15) +**Goal**: Final synchronization, cutover to destination + +```c +// Subcmd 12 (0x057A345C) +int migrate_commit_finalize(vf_id, dst_vf_id) { + // 1. Final dirty page transfer + // 2. Transfer register state + // 3. Activate destination VF + // 4. Release source VF resources + // 5. Notify completion +} + +// Subcmd 13 (0x057A36D8) +int migrate_commit_verify(vf_id, *verify_result) { + // 1. Verify memory integrity + // 2. Verify register state + // 3. Verify device state + // 4. Return PASS/FAIL +} +``` + +### Phase 4: Cleanup (Post-migration) +```c +int migrate_cleanup(src_vf_id) { + // 1. Release pinned memory + // 2. Free snapshot buffers + // 3. Remove WPR regions + // 4. Destroy migration context +} +``` + +## Migration Message Format + +```c +typedef struct { + uint32_t phase; // MIGRATE_PHASE_* + uint32_t subcmd; // Sub-command index (0-15 per table) + uint64_t src_vf; // Source VF ID + uint64_t dst_vf; // Destination VF ID + uint64_t data_ptr; // Data buffer (page list, bitmap, etc.) + uint64_t data_size; // Buffer size + uint32_t flags; // MIGRATE_LIVE, MIGRATE_POSTCOPY, etc. + uint32_t reserved; + uint64_t token; // Opaque migration token +} __attribute__((packed)) migrate_payload_t; + +#define MIGRATE_PHASE_QUERY 0 +#define MIGRATE_PHASE_PREPARE 1 +#define MIGRATE_PHASE_TRANSFER 2 +#define MIGRATE_PHASE_COMMIT 3 +#define MIGRATE_PHASE_CLEANUP 4 + +#define MIGRATE_FLAG_LIVE (1<<0) // Live migration +#define MIGRATE_FLAG_POSTCOPY (1<<1) // Post-copy mode +#define MIGRATE_FLAG_AUTO_COMMIT (1<<2) // Auto-commit after transfer +#define MIGRATE_FLAG_VERIFY (1<<3) // Verify integrity +``` + +## Sub-Command Descriptors (Returned by Handlers) + +All migration handlers follow the primary dispatcher pattern: + +```c +typedef struct { + uint32_t subcmd_id; // Constant: 0x10, 0x14, etc. + uint8_t echo_flags_b; // Copy of input flags_b + uint8_t echo_flags_a; // Copy of input flags_a + uint8_t echo_flags_c; // Copy of input flags_c +} migrate_descriptor_t; +``` + +**Known Subcmd IDs**: +| Subcmd | ID | Phase | +|--------|-----|-------| +| 0x057A22F8 (Table1[3]) | 0x10 | Prepare | +| 0x057A2344 (Table1[0]) | 0x14 | Prepare | +| Others | TBD | Transfer/Commit | + +## vGPU-IO Integration + +### Migration State Machine +```c +typedef enum { + MIG_STATE_IDLE = 0, + MIG_STATE_QUERY, + MIG_STATE_PREPARE, + MIG_STATE_TRANSFER, + MIG_STATE_COMMIT, + MIG_STATE_CLEANUP, + MIG_STATE_FAILED, + MIG_STATE_COMPLETED +} mig_state_t; + +struct gsp_migration { + struct gsp_vf *vf; + mig_state_t state; + uint64_t token; + uint64_t dst_vf; + struct migrate_progress progress; + struct completion comp; + struct mutex lock; +}; +``` + +### Migration Coordinator +```c +int gsp_migration_start(struct gsp_vf *vf, uint64_t dst_vf) { + struct gsp_migration *mig = kzalloc(sizeof(*mig), GFP_KERNEL); + vf->migration = mig; + mig->vf = vf; + mig->dst_vf = dst_vf; + mig->state = MIG_STATE_QUERY; + + // Phase 0: Query + struct migrate_query qry; + gsp_migrate_query(vf, &qry); + if (!qry.migratable) return -EINVAL; + + // Phase 1: Prepare + mig->state = MIG_STATE_PREPARE; + migrate_payload_t prep = { + .phase = MIGRATE_PHASE_PREPARE, + .subcmd = 3, // Quiesce + .src_vf = vf->id, + .dst_vf = dst_vf, + }; + migrate_descriptor_t desc; + gsp_rpc_send(vf->gsp, RPC_MIGRATE, &prep, sizeof(prep), &desc, sizeof(desc)); + mig->token = desc.subcmd_id; // Use returned token + + // Phase 2: Transfer (iterative) + mig->state = MIG_STATE_TRANSFER; + while (!gsp_migration_transfer_iter(mig)) { + msleep(10); // Throttle + if (signal_pending(current)) break; + } + + // Phase 3: Commit + mig->state = MIG_STATE_COMMIT; + migrate_payload_t commit = { + .phase = MIGRATE_PHASE_COMMIT, + .subcmd = 12, + .src_vf = vf->id, + .dst_vf = dst_vf, + .token = mig->token, + .flags = MIGRATE_FLAG_VERIFY, + }; + gsp_rpc_send(vf->gsp, RPC_MIGRATE, &commit, sizeof(commit), NULL, 0); + + // Phase 4: Cleanup + mig->state = MIG_STATE_CLEANUP; + gsp_migration_cleanup(mig); + + mig->state = MIG_STATE_COMPLETED; + return 0; +} +``` + +### Transfer Iteration +```c +bool gsp_migration_transfer_iter(struct gsp_migration *mig) { + migrate_payload_t xfer = { + .phase = MIGRATE_PHASE_TRANSFER, + .subcmd = 0, + .src_vf = mig->vf->id, + .dst_vf = mig->dst_vf, + .data_ptr = (uint64_t)mig->progress.page_list, + .data_size = mig->progress.page_count * sizeof(uint64_t), + .token = mig->token, + }; + + struct migrate_progress prog; + int ret = gsp_rpc_send(mig->vf->gsp, RPC_MIGRATE, + &xfer, sizeof(xfer), &prog, sizeof(prog)); + + if (ret) return true; // Error = done + + mig->progress.bytes_transferred += prog.bytes_transferred; + mig->progress.pages_transferred += prog.pages_transferred; + mig->progress.dirty_pages = prog.dirty_pages; + + // Check if more dirty pages to transfer + if (prog.dirty_pages > 0 && !(mig->flags & MIGRATE_FLAG_POSTCOPY)) { + return false; // Continue iterating + } + + return true; // Transfer complete +} +``` + +### Post-Copy Mode +```c +int gsp_migration_postcopy(struct gsp_migration *mig) { + // Enable post-copy on destination + migrate_payload_t pc = { + .phase = MIGRATE_PHASE_TRANSFER, + .subcmd = 11, // Dirty tracking + .src_vf = mig->vf->id, + .dst_vf = mig->dst_vf, + .flags = MIGRATE_FLAG_POSTCOPY, + .token = mig->token, + }; + + // Page fault handler on destination + gsp_page_fault_handler = gsp_migration_page_fault; + + return gsp_rpc_send(mig->vf->gsp, RPC_MIGRATE, &pc, sizeof(pc), NULL, 0); +} +``` + +## Integration with BAR/WPR + +### BAR State Migration +```c +int gsp_migration_bar_save(struct gsp_migration *mig) { + // Save BAR1 aperture, force access state + struct bar_save_msg bar_save = { + .vf_id = mig->vf->id, + .bar = 1, + }; + return gsp_rpc_send(mig->vf->gsp, RPC_BAR_TRAP, + BAR_SUBCMD_SAVE, &bar_save, sizeof(bar_save), + mig->bar_state_buf, &mig->bar_state_size); +} + +int gsp_migration_bar_restore(struct gsp_migration *mig) { + // Restore BAR1 on destination + struct bar_restore_msg bar_restore = { + .vf_id = mig->dst_vf, + .bar = 1, + .data = mig->bar_state_buf, + .data_size = mig->bar_state_size, + }; + return gsp_rpc_send(mig->vf->gsp, RPC_BAR_TRAP, + BAR_SUBCMD_RESTORE, &bar_restore, sizeof(bar_restore)); +} +``` + +### WPR Region Migration +```c +int gsp_migration_wpr_save(struct gsp_migration *mig) { + // Save WPR regions for VF + migrate_payload_t wpr = { + .phase = MIGRATE_PHASE_PREPARE, + .subcmd = 4, // WPR snapshot + .src_vf = mig->vf->id, + .dst_vf = mig->dst_vf, + .token = mig->token, + }; + return gsp_rpc_send(mig->vf->gsp, RPC_MIGRATE, &wpr, sizeof(wpr), NULL, 0); +} +``` + +## Error Handling + +| Phase | Error | Recovery | +|-------|-------|----------| +| Prepare | EBUSY | Wait, retry | +| Prepare | ENOMEM | Reduce snapshot size | +| Transfer | ETIMEDOUT | Checkpoint, retry | +| Transfer | EIO | Failover to post-copy | +| Commit | EINVAL | Rollback to source | +| Commit | ENOSPC | Expand destination | + +## Debugging + +### Migration Trace Points +```c +trace_gsp_mig_phase(vf_id, phase, subcmd, token); +trace_gsp_mig_transfer(vf_id, pages, bytes, dirty); +trace_gsp_mig_commit(vf_id, status, verify); +trace_gsp_mig_error(vf_id, phase, error); +``` + +### GSP Log Strings +Search for migration-related strings in firmware: +- Migration phase markers +- Page transfer progress +- Commit verification results + +--- + +## Reference +- Primary Dispatcher MIGRATE branches: 0x057A1BDC, 0x057A1D54 +- Jump Table 1: 0x04179ED8 (16 entries) +- Jump Table 2: 0x04179FBC (16 entries) +- Decompiled handlers: 28 functions in decompiled/
\ No newline at end of file |