/* * Real-GSP boot bring-up stage 3b: boot-argument allocation and WPR layout. * * A real GSP boot needs three host-DMA pages - the WPR metadata the booter * consumes, the libos init arguments, and the GSP/bootloader arguments - plus * the video-memory layout recorded in that metadata. This file mirrors the * layout computation the distribution driver performs for Turing, fills the * 256-byte WprMeta structure and logs every field. * * Stage 3b writes nothing to the device: the only device access is a * read-only sample of the video-memory-size register in BAR0; the metadata * page is filled in host memory only. Disabled by default (gsp_phys_layout=N); * claim must be enabled too (gsp_phys_gpu=1). */ #include #include #include #include #include #include #include #include "gsp_phys_boot.h" static bool gsp_phys_layout_param; module_param_named(gsp_phys_layout, gsp_phys_layout_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_layout, "Allocate the GSP boot-argument pages and compute/log the WPR/FB layout on the claimed GPU (default 0, no device writes)"); bool gsp_phys_layout_enabled(void) { return gsp_phys_layout_param; } /* Boot-argument allocation sizes (mirror of the reference allocator). */ #define GSP_PHYS_PAGE_SIZE 0x1000 #define GSP_PHYS_LIBOS_ARGS_SIZE 0x1000 #define GSP_PHYS_GSP_ARGS_SIZE 0x1000 /* * The boot binary ("bootloader"/SK ucode) is not a section of the staged * .fwimage; the distribution driver carries it as compiled-in bindata. The * Turing boot ucode image is 4 KB, but the staged image cannot confirm the * exact size, so this stays a clearly-named stub used for layout only. */ #define GSP_PHYS_BOOTLOADER_SIZE_STUB 0x1000 /* Turing-era layout constants (HAL values for TU102..AD10x). */ #define GSP_PHYS_FRTS_SIZE (0x100 * 0x1000) #define GSP_PHYS_WPR_END_ALIGN 0x20000 #define GSP_PHYS_BOOTBIN_ALIGN 0x1000 #define GSP_PHYS_FW_ALIGN 0x10000 #define GSP_PHYS_MB (1ULL << 20) #define GSP_PHYS_PRAMIN_SIZE (1 * GSP_PHYS_MB) #define GSP_PHYS_NON_WPR_HEAP_SIZE (1 * GSP_PHYS_MB) #define GSP_PHYS_FW_HEAP_OS_CARVEOUT 0 #define GSP_PHYS_FW_HEAP_BASE_RM_SIZE (8 * GSP_PHYS_MB) #define GSP_PHYS_FW_HEAP_PER_GB_FB (96 * 1024) #define GSP_PHYS_FW_HEAP_CLIENT_ALLOC ((48 * 1024) * 2048) #define GSP_PHYS_FW_HEAP_MIN_MB 64 #define GSP_PHYS_FW_HEAP_MAX_MB 256 #define GSP_PHYS_PRESCRUBBED_TOP_FB (256 * GSP_PHYS_MB) /* * Video-memory size register in the PFB PRI MMU block (read-only). The VBIOS * devinit programs the lower bound of the usable local-memory range before * the driver runs: a 6-bit mantissa in bits 9:4 scaled by 2^(scale+20), with * the ECC carve-out reported in bit 30. */ #define GSP_PHYS_FB_LOCAL_MEMORY_RANGE 0x00100ce0 #define GSP_PHYS_FB_LOWER_SCALE 0x0000000f #define GSP_PHYS_FB_LOWER_SCALE_SHIFT 0 #define GSP_PHYS_FB_LOWER_MAG 0x000003f0 #define GSP_PHYS_FB_LOWER_MAG_SHIFT 4 #define GSP_PHYS_FB_ECC_MODE BIT(30) #define GSP_PHYS_FB_SIZE_MIN (256 * GSP_PHYS_MB) #define GSP_PHYS_FB_SIZE_MAX (64ULL << 30) #define GSP_PHYS_WPR_META_MAGIC 0xdc3aae21371a60b3ULL #define GSP_PHYS_WPR_META_REVISION 1 #define GSP_PHYS_WPR_META_FLAG_CLOCK_BOOST (1U << 0) /* * GspFwWprMeta, the 256-byte handoff structure the booter consumes from the * WPR metadata page. Field order and offsets follow the driver ABI (magic at * 0, revision at 8, layout fields from 88, verified at 248); the two unions * alias signature/free-list and partition/crash-queue members respectively. */ struct gsp_phys_wpr_meta { __u64 magic; __u64 revision; __u64 sysmem_addr_of_radix3_elf; __u64 size_of_radix3_elf; __u64 sysmem_addr_of_bootloader; __u64 size_of_bootloader; __u64 bootloader_code_offset; __u64 bootloader_data_offset; __u64 bootloader_manifest_offset; union { struct { __u64 sysmem_addr_of_signature; __u64 size_of_signature; }; struct { __u32 gsp_fw_heap_free_list_wpr_offset; __u32 unused0; __u64 unused1; }; }; __u64 gsp_fw_rsvd_start; __u64 non_wpr_heap_offset; __u64 non_wpr_heap_size; __u64 gsp_fw_wpr_start; __u64 gsp_fw_heap_offset; __u64 gsp_fw_heap_size; __u64 gsp_fw_offset; __u64 boot_bin_offset; __u64 frts_offset; __u64 frts_size; __u64 gsp_fw_wpr_end; __u64 fb_size; __u64 vga_workspace_offset; __u64 vga_workspace_size; __u64 boot_count; union { struct { __u64 partition_rpc_addr; __u16 partition_rpc_request_offset; __u16 partition_rpc_reply_offset; __u32 elf_code_offset; __u32 elf_data_offset; __u32 elf_code_size; __u32 elf_data_size; __u32 ls_ucode_version; }; struct { __u32 partition_rpc_padding[4]; __u64 sysmem_addr_of_crash_report_queue; __u32 size_of_crash_report_queue; __u32 ls_ucode_version_padding[1]; }; }; __u8 gsp_fw_heap_vf_partition_count; __u8 flags; __u8 padding[2]; __u32 pmu_reserved_size; __u64 verified; }; static u64 gsp_phys_align_down(u64 value, u64 align) { return value & ~(align - 1); } static u64 gsp_phys_align_up(u64 value, u64 align) { return (value + align - 1) & ~(align - 1); } /* * Decode the real usable FB size from the VBIOS-programmed local-memory-range * register: mantissa (bits 9:4) << (scale (bits 3:0) + 20), reduced by 1/16 * when ECC is enabled (bit 30). Read-only. Returns 0 with the decoded byte * size, or a negative error when the register decodes to zero or garbage - * the caller must fail the layout, never fall back to the BAR1 aperture. */ static int gsp_phys_fb_size_decode(void __iomem *bar0, u64 fw_image_size, u64 *out_fb_size) { u32 reg, mag, scale; bool ecc; u64 fb_size; reg = readl(bar0 + GSP_PHYS_FB_LOCAL_MEMORY_RANGE); mag = (reg & GSP_PHYS_FB_LOWER_MAG) >> GSP_PHYS_FB_LOWER_MAG_SHIFT; scale = (reg & GSP_PHYS_FB_LOWER_SCALE) >> GSP_PHYS_FB_LOWER_SCALE_SHIFT; ecc = !!(reg & GSP_PHYS_FB_ECC_MODE); fb_size = (u64)mag << (scale + 20); if (ecc) fb_size = fb_size / 16 * 15; pr_info("gsp-phys-boot: FB size register[0x%06x]=0x%08x lowerMag=%u lowerScale=%u ecc=%u -> real usable FB size 0x%llx (%llu GiB)\n", GSP_PHYS_FB_LOCAL_MEMORY_RANGE, reg, mag, scale, (unsigned int)ecc, fb_size, fb_size >> 30); if (!mag || !fb_size) { pr_err("gsp-phys-boot: FB size register decodes to zero (reg=0x%08x mag=%u scale=%u); refusing layout\n", reg, mag, scale); return -EINVAL; } if (fb_size < GSP_PHYS_FB_SIZE_MIN || fb_size > GSP_PHYS_FB_SIZE_MAX) { pr_err("gsp-phys-boot: decoded real FB size 0x%llx outside sane range [0x%llx..0x%llx]; refusing layout\n", fb_size, (u64)GSP_PHYS_FB_SIZE_MIN, (u64)GSP_PHYS_FB_SIZE_MAX); return -ERANGE; } if (!IS_ALIGNED(fb_size, GSP_PHYS_PAGE_SIZE)) { pr_err("gsp-phys-boot: decoded real FB size 0x%llx is not page-aligned; refusing layout\n", fb_size); return -ERANGE; } if (fw_image_size && fb_size <= fw_image_size) { pr_err("gsp-phys-boot: decoded real FB size 0x%llx does not fit the staged GSP-RM image (0x%llx); refusing layout\n", fb_size, fw_image_size); return -ERANGE; } *out_fb_size = fb_size; return 0; } /* * GSP FW (WPR) heap size for a bare-metal Turing part. Ported from the * reference: a base RM reservation, a per-GB-of-FB term, and a client-channel * allocation term, clamped to the architecture min/max and to the pre-scrubbed * top of FB (Turing has no scrubber ucode, so the heap must fit there). */ static u64 gsp_phys_fw_heap_size(u64 fb_size, u64 anterior, u64 posterior) { const u64 fb_size_gb = gsp_phys_align_up(fb_size, 1ULL << 30) >> 30; u64 max_scrubbed_mb = U32_MAX; u64 heap; if (anterior + posterior < GSP_PHYS_PRESCRUBBED_TOP_FB) max_scrubbed_mb = (GSP_PHYS_PRESCRUBBED_TOP_FB - anterior - posterior) >> 20; heap = GSP_PHYS_FW_HEAP_OS_CARVEOUT + GSP_PHYS_FW_HEAP_BASE_RM_SIZE + gsp_phys_align_up(GSP_PHYS_FW_HEAP_PER_GB_FB * fb_size_gb, GSP_PHYS_MB) + gsp_phys_align_up((u64)GSP_PHYS_FW_HEAP_CLIENT_ALLOC, GSP_PHYS_MB); heap = max(heap, (u64)GSP_PHYS_FW_HEAP_MIN_MB * GSP_PHYS_MB); heap = min(heap, min((u64)GSP_PHYS_FW_HEAP_MAX_MB, max_scrubbed_mb) * GSP_PHYS_MB); return heap; } static bool gsp_phys_wpr_layout_valid(const struct gsp_phys_wpr_meta *m) { return m->frts_size <= m->gsp_fw_wpr_end && m->gsp_fw_offset < m->boot_bin_offset && m->boot_bin_offset < m->frts_offset && m->frts_offset < m->gsp_fw_wpr_end && m->gsp_fw_wpr_end <= m->vga_workspace_offset && m->gsp_fw_wpr_start < m->gsp_fw_heap_offset && m->non_wpr_heap_offset < m->gsp_fw_wpr_start && m->gsp_fw_rsvd_start == m->non_wpr_heap_offset && m->gsp_fw_offset > 0 && m->non_wpr_heap_offset > 0; } static void gsp_phys_wpr_meta_dump(const struct gsp_phys_wpr_meta *m) { pr_info("gsp-phys-boot: GspFwWprMeta sizeof=%zu (must be 256) magic=0x%llx revision=%llu flags=0x%x\n", sizeof(*m), (u64)m->magic, (u64)m->revision, m->flags); pr_info("gsp-phys-boot: fbSize=0x%llx vgaWorkspace=0x%llx - 0x%llx (0x%llx)\n", m->fb_size, m->vga_workspace_offset, m->vga_workspace_offset + m->vga_workspace_size - 1, m->vga_workspace_size); pr_info("gsp-phys-boot: gspFwRsvdStart=0x%llx\n", m->gsp_fw_rsvd_start); pr_info("gsp-phys-boot: nonWprHeap=0x%llx - 0x%llx (0x%llx)\n", m->non_wpr_heap_offset, m->non_wpr_heap_offset + m->non_wpr_heap_size - 1, m->non_wpr_heap_size); pr_info("gsp-phys-boot: gspFwWprStart=0x%llx\n", m->gsp_fw_wpr_start); pr_info("gsp-phys-boot: gspFwHeap=0x%llx - 0x%llx (0x%llx)\n", m->gsp_fw_heap_offset, m->gsp_fw_heap_offset + m->gsp_fw_heap_size - 1, m->gsp_fw_heap_size); pr_info("gsp-phys-boot: gspFwOffset=0x%llx - 0x%llx (0x%llx)\n", m->gsp_fw_offset, m->gsp_fw_offset + m->size_of_radix3_elf - 1, m->size_of_radix3_elf); pr_info("gsp-phys-boot: bootBinOffset=0x%llx - 0x%llx (0x%llx)\n", m->boot_bin_offset, m->boot_bin_offset + m->size_of_bootloader - 1, m->size_of_bootloader); pr_info("gsp-phys-boot: frtsOffset=0x%llx - 0x%llx (0x%llx)\n", m->frts_offset, m->frts_offset + m->frts_size - 1, m->frts_size); pr_info("gsp-phys-boot: gspFwWprEnd=0x%llx bootCount=%llu verified=%llu\n", m->gsp_fw_wpr_end, (u64)m->boot_count, (u64)m->verified); pr_info("gsp-phys-boot: stubs: sysmemAddrOfRadix3Elf/Bootloader/Signature=0 crashReportQueue=0 (image not staged in 3b), bootloader offsets=0\n"); } /* * Stage 3b entry: allocate the boot-argument pages on the claimed device and * compute/record the WPR layout in the metadata page. No device writes. */ int gsp_phys_boot_layout_init(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0, u64 fw_image_size, resource_size_t bar1_len) { struct gsp_phys_wpr_meta *meta; u64 usable_fb_size; u64 vbios_reserved_offset; u64 non_wpr_heap_size, wpr_meta_size; u64 pre_wpr_heap, post_wpr_heap, wpr_heap_size; int ret; if (!gsp_phys_layout_enabled() || boot->initialized) return 0; if (!pdev || !bar0 || !fw_image_size) return -EINVAL; BUILD_BUG_ON(sizeof(struct gsp_phys_wpr_meta) != 256); pr_info("gsp-phys-boot: stage 3b layout+allocation on %s (BAR0 read-only, no device writes)\n", pci_name(pdev)); ret = gsp_phys_fb_size_decode(bar0, fw_image_size, &usable_fb_size); if (ret) return ret; pr_info("gsp-phys-boot: real FB size = 0x%llx (%llu GiB), replacing the BAR1-aperture approximation\n", usable_fb_size, usable_fb_size >> 30); pr_info("gsp-phys-boot: BAR1 aperture = 0x%llx (%llu MiB); decoded real FB size difference %+lld bytes\n", (u64)bar1_len, (u64)bar1_len >> 20, (long long)((s64)usable_fb_size - (s64)bar1_len)); pr_info("gsp-phys-boot: staged GSP-RM image (.fwimage) size = 0x%llx\n", fw_image_size); pr_info("gsp-phys-boot: stub: boot binary size assumed 0x%x (boot ucode is compiled-in bindata, not a .fwimage section)\n", GSP_PHYS_BOOTLOADER_SIZE_STUB); pr_info("gsp-phys-boot: wprEndMargin=0 (reference normal first-boot path; no override/retry margin)\n"); boot->wpr_meta_size = GSP_PHYS_PAGE_SIZE; boot->wpr_meta_va = dma_alloc_coherent(&pdev->dev, boot->wpr_meta_size, &boot->wpr_meta_dma, GFP_KERNEL); if (!boot->wpr_meta_va) { pr_err("gsp-phys-boot: WPR meta page allocation failed\n"); ret = -ENOMEM; goto err; } boot->libos_args_size = GSP_PHYS_LIBOS_ARGS_SIZE; boot->libos_args_va = dma_alloc_coherent(&pdev->dev, boot->libos_args_size, &boot->libos_args_dma, GFP_KERNEL); if (!boot->libos_args_va) { pr_err("gsp-phys-boot: libos init arguments allocation failed\n"); ret = -ENOMEM; goto err; } boot->gsp_args_size = GSP_PHYS_GSP_ARGS_SIZE; boot->gsp_args_va = dma_alloc_coherent(&pdev->dev, boot->gsp_args_size, &boot->gsp_args_dma, GFP_KERNEL); if (!boot->gsp_args_va) { pr_err("gsp-phys-boot: GSP/bootloader arguments allocation failed\n"); ret = -ENOMEM; goto err; } pr_info("gsp-phys-boot: WPR meta page 0x%zx bytes dma=0x%llx va=%p (coherent/uncached, zeroed)\n", boot->wpr_meta_size, (u64)boot->wpr_meta_dma, boot->wpr_meta_va); pr_info("gsp-phys-boot: libos args 0x%zx bytes dma=0x%llx va=%p (coherent, zeroed; filled in a later stage)\n", boot->libos_args_size, (u64)boot->libos_args_dma, boot->libos_args_va); pr_info("gsp-phys-boot: gsp args 0x%zx bytes dma=0x%llx va=%p (coherent, zeroed; filled in a later stage)\n", boot->gsp_args_size, (u64)boot->gsp_args_dma, boot->gsp_args_va); meta = boot->wpr_meta_va; /* * Layout from the top of FB downward. Turing has no VBIOS MMU-lock * region (the lock read is invalid on this generation), so the VGA * workspace bottom is the reserved-area boundary. */ meta->fb_size = usable_fb_size; meta->vga_workspace_offset = usable_fb_size - GSP_PHYS_PRAMIN_SIZE; meta->vga_workspace_size = usable_fb_size - meta->vga_workspace_offset; vbios_reserved_offset = meta->vga_workspace_offset; meta->size_of_radix3_elf = fw_image_size; meta->gsp_fw_wpr_end = gsp_phys_align_down(vbios_reserved_offset, GSP_PHYS_WPR_END_ALIGN); meta->frts_size = GSP_PHYS_FRTS_SIZE; meta->frts_offset = meta->gsp_fw_wpr_end - meta->frts_size; meta->size_of_bootloader = GSP_PHYS_BOOTLOADER_SIZE_STUB; meta->boot_bin_offset = gsp_phys_align_down(meta->frts_offset - meta->size_of_bootloader, GSP_PHYS_BOOTBIN_ALIGN); meta->gsp_fw_offset = gsp_phys_align_down(meta->boot_bin_offset - meta->size_of_radix3_elf, GSP_PHYS_FW_ALIGN); non_wpr_heap_size = gsp_phys_align_up((u64)GSP_PHYS_NON_WPR_HEAP_SIZE, GSP_PHYS_MB); wpr_meta_size = gsp_phys_align_up((u64)sizeof(*meta), GSP_PHYS_MB); pre_wpr_heap = wpr_meta_size + non_wpr_heap_size; post_wpr_heap = gsp_phys_align_up(usable_fb_size - meta->gsp_fw_offset, GSP_PHYS_MB); wpr_heap_size = gsp_phys_fw_heap_size(usable_fb_size, pre_wpr_heap, post_wpr_heap); if (meta->gsp_fw_offset < pre_wpr_heap + wpr_heap_size) { pr_err("gsp-phys-boot: real FB size 0x%llx too small: image (0x%llx) + WPR heap (0x%llx) + pre-WPR regions (0x%llx) do not fit below the VGA workspace\n", usable_fb_size, meta->size_of_radix3_elf, wpr_heap_size, pre_wpr_heap); ret = -ERANGE; goto err; } meta->gsp_fw_heap_offset = gsp_phys_align_down(meta->gsp_fw_offset - wpr_heap_size, GSP_PHYS_MB); meta->gsp_fw_heap_size = gsp_phys_align_down(meta->gsp_fw_offset - meta->gsp_fw_heap_offset, GSP_PHYS_MB); meta->gsp_fw_heap_vf_partition_count = 0; meta->gsp_fw_wpr_start = meta->gsp_fw_heap_offset - wpr_meta_size; meta->non_wpr_heap_size = non_wpr_heap_size; meta->non_wpr_heap_offset = meta->gsp_fw_wpr_start - non_wpr_heap_size; meta->gsp_fw_rsvd_start = meta->non_wpr_heap_offset; /* Sysmem image addresses are not staged yet (stage 3d fills them). */ meta->sysmem_addr_of_radix3_elf = 0; meta->sysmem_addr_of_bootloader = 0; meta->bootloader_code_offset = 0; meta->bootloader_data_offset = 0; meta->bootloader_manifest_offset = 0; meta->sysmem_addr_of_signature = 0; meta->size_of_signature = 0; meta->sysmem_addr_of_crash_report_queue = 0; meta->size_of_crash_report_queue = 0; meta->boot_count = 0; meta->verified = 0; meta->revision = GSP_PHYS_WPR_META_REVISION; meta->magic = GSP_PHYS_WPR_META_MAGIC; meta->flags = GSP_PHYS_WPR_META_FLAG_CLOCK_BOOST; pr_info("gsp-phys-boot: FW heap size = 0x%llx (pre=0x%llx post=0x%llx, min %uMB max %uMB, pre-scrubbed top 0x%llx)\n", wpr_heap_size, pre_wpr_heap, post_wpr_heap, GSP_PHYS_FW_HEAP_MIN_MB, GSP_PHYS_FW_HEAP_MAX_MB, (u64)GSP_PHYS_PRESCRUBBED_TOP_FB); if (!gsp_phys_wpr_layout_valid(meta)) { pr_err("gsp-phys-boot: computed layout is inconsistent (frame too small for image/heap?)\n"); ret = -EINVAL; goto err; } gsp_phys_wpr_meta_dump(meta); boot->fb_size = meta->fb_size; boot->vga_workspace_offset = meta->vga_workspace_offset; boot->vga_workspace_size = meta->vga_workspace_size; boot->gsp_fw_wpr_end = meta->gsp_fw_wpr_end; boot->frts_offset = meta->frts_offset; boot->frts_size = meta->frts_size; boot->boot_bin_offset = meta->boot_bin_offset; boot->size_of_bootloader = meta->size_of_bootloader; boot->gsp_fw_offset = meta->gsp_fw_offset; boot->size_of_radix3_elf = meta->size_of_radix3_elf; boot->gsp_fw_heap_offset = meta->gsp_fw_heap_offset; boot->gsp_fw_heap_size = meta->gsp_fw_heap_size; boot->gsp_fw_wpr_start = meta->gsp_fw_wpr_start; boot->non_wpr_heap_offset = meta->non_wpr_heap_offset; boot->non_wpr_heap_size = meta->non_wpr_heap_size; boot->gsp_fw_rsvd_start = meta->gsp_fw_rsvd_start; boot->initialized = true; pr_info("gsp-phys-boot: stage 3b complete: boot args allocated, WPR meta filled (no device writes)\n"); return 0; err: gsp_phys_boot_layout_fini(boot, pdev); return ret; } void gsp_phys_boot_layout_fini(struct gsp_phys_boot *boot, struct pci_dev *pdev) { if (!boot->initialized && !boot->wpr_meta_va && !boot->libos_args_va && !boot->gsp_args_va) return; if (!pdev) return; if (boot->wpr_meta_va) dma_free_coherent(&pdev->dev, boot->wpr_meta_size, boot->wpr_meta_va, boot->wpr_meta_dma); if (boot->libos_args_va) dma_free_coherent(&pdev->dev, boot->libos_args_size, boot->libos_args_va, boot->libos_args_dma); if (boot->gsp_args_va) dma_free_coherent(&pdev->dev, boot->gsp_args_size, boot->gsp_args_va, boot->gsp_args_dma); boot->wpr_meta_va = NULL; boot->libos_args_va = NULL; boot->gsp_args_va = NULL; boot->initialized = false; pr_info("gsp-phys-boot: boot-argument pages freed\n"); }