/* * SPDX-FileCopyrightText: Copyright (c) 1999-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ /************************************************************************************************************** * * Description: * UNIX-general, device-independent initialization code for * the resource manager. * * **************************************************************************************************************/ #include #include // NV device driver interface #include #include #include #include #include #include #include #include #include #include "gpu/gpu.h" #include #include "gpu/bus/kern_bus.h" #include "nverror.h" #include #include #include #include #include "kernel/gpu/intr/intr.h" #include #include #include #include #include #include #include #include "liblogdecode.h" #include #include #include #include #include #include #include #include #include #include #include #include "platform/nbsi/nbsi_read.h" #include "gpu_mgr/gpu_db.h" #include #include #include #include #include // RMCONFIG: need definition of REGISTER_ALL_HALS() #include "g_hal_register.h" typedef enum { RM_INIT_OK, /* general os errors */ RM_INIT_REG_SETUP_FAILED = 0x10, RM_INIT_SYS_ENVIRONMENT_FAILED, /* gpu errors */ RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED = 0x20, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED, RM_INIT_GPU_PRE_INIT_FAILED, RM_INIT_GPU_STATE_INIT_FAILED, RM_INIT_GPU_LOAD_FAILED, RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED, RM_INIT_GPU_DMA_CONFIGURATION_FAILED, /* vbios errors */ RM_INIT_VBIOS_FAILED = 0x30, RM_INIT_VBIOS_POST_FAILED, RM_INIT_VBIOS_X86EMU_FAILED, /* scalability errors */ RM_INIT_SCALABILITY_FAILED = 0x40, /* general core rm errors */ RM_INIT_WATCHDOG_FAILED, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED, RM_GPUDB_REGISTER_FAILED, RM_INIT_ALLOC_RMAPI_FAILED, RM_INIT_GPUINFO_WITH_RMAPI_FAILED, /* rm firmware errors */ RM_INIT_FIRMWARE_POLICY_FAILED = 0x60, RM_INIT_FIRMWARE_FETCH_FAILED, RM_INIT_FIRMWARE_INIT_FAILED, RM_INIT_MAX_FAILURES } rm_init_status; typedef rm_init_status RM_INIT_STATUS; typedef struct { RM_INIT_STATUS initStatus; NV_STATUS rmStatus; NvU32 line; } UNIX_STATUS; #define INIT_UNIX_STATUS { RM_INIT_OK, NV_OK, 0 } #define RM_INIT_SUCCESS(init) ((init) == RM_INIT_OK) #define RM_SET_ERROR(status, err) { (status).initStatus = (err); \ (status).line = __LINE__; } // // GPU architectures support DMA addressing up to a certain address width, // above which all other bits in any given DMA address must not vary // (e.g., all 0). This value is the minimum of the DMA addressing // capabilities, in number of physical address bits, for all supported // GPU architectures. // #define NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 36 // // All GPU architectures with GSP support at least 47 physical address bits. // #define NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH 47 static inline NvU64 nv_encode_pci_info(nv_pci_info_t *pci_info) { return gpuEncodeDomainBusDevice(pci_info->domain, pci_info->bus, pci_info->slot); } static inline NvU32 nv_generate_id_from_pci_info(nv_pci_info_t *pci_info) { return gpuGenerate32BitId(pci_info->domain, pci_info->bus, pci_info->slot); } static inline void nv_os_map_kernel_space(nv_state_t *nv, nv_aperture_t *aperture) { NV_ASSERT(aperture->map == NULL); // let's start off assuming a standard device and map the registers // normally. It is unfortunate to hard-code the register size here, but we don't // want to fail trying to map all of a multi-devices' register space aperture->map = osMapKernelSpace(aperture->cpu_address, aperture->size, NV_MEMORY_UNCACHED, NV_PROTECT_READ_WRITE); aperture->map_u = (nv_phwreg_t)aperture->map; } // local prototypes static NV_STATUS initCoreLogic(OBJGPU *); static void initVendorSpecificRegistry(OBJGPU *, NvU16); static NV_STATUS teardownCoreLogic(OBJOS *, OBJGPU *); static void initUnixSpecificRegistry(OBJGPU *); NvBool osRmInitRm(OBJOS *pOS) { OBJSYS *pSys = SYS_GET_INSTANCE(); NvU64 system_memory_size = (NvU64)-1; NV_PRINTF(LEVEL_INFO, "init rm\n"); if (os_is_efi_enabled()) { pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_TRUE); } // have to init this before the debug subsystem, which will // try to check the value of ResmanDebugLevel RmInitRegistry(); NvU32 data = 0; if ((osReadRegistryDword(NULL, NV_REG_ENABLE_PCIE_GEN3, &data) == NV_OK) && (data != 0)) { data = DRF_DEF(_REG_STR, _RM_PCIE, _LINK_SPEED_ALLOW_GEN3, _ENABLE); osWriteRegistryDword(NULL, NV_REG_STR_RM_PCIE_LINK_SPEED, data); } if (!(osReadRegistryDword(NULL, NV_REG_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, &data) == NV_OK) || data) { pSys->setProperty(pSys, PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS, NV_TRUE); } if ((osReadRegistryDword(NULL, NV_REG_ENABLE_STREAM_MEMOPS, &data) == NV_OK) && (data != 0)) { data = DRF_DEF(_REG_STR, _RM_STREAM_MEMOPS, _ENABLE, _YES); osWriteRegistryDword(NULL, NV_REG_STR_RM_STREAM_MEMOPS, data); } if ((osReadRegistryDword(NULL, NV_REG_STR_RM_FORCE_BAR_PATH, &data) == NV_OK) && (data == 1)) { // // Exposing the GPU memory as NUMA node memory requires coherent CPU // mappings over NVLink. If those mappings are disabled, also disable // NUMA-onlining of the device memory. // osWriteRegistryDword(NULL, NV_REG_ENABLE_USER_NUMA_MANAGEMENT, 0); } if ((osReadRegistryDword(NULL, NV_REG_NVLINK_DISABLE, &data) == NV_OK) && (data == 1)) { osWriteRegistryDword(NULL, NV_REG_STR_RM_NVLINK_CONTROL, 0x1); } if ((osReadRegistryDword(NULL, NV_REG_ENABLE_PCIE_RELAXED_ORDERING_MODE, &data) == NV_OK) && (data == 1)) { osWriteRegistryDword(NULL, NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING, NV_REG_STR_RM_SET_PCIE_TLP_RELAXED_ORDERING_ENABLE); } // init the debug subsystem if necessary os_dbg_init(); nvDbgInitRmMsg(NULL); // Force nvlog reinit since module params are now available NVLOG_UPDATE(); // Register all supported hals if (REGISTER_ALL_HALS() != NV_OK) { RmDestroyRegistry(NULL); return NV_FALSE; } system_memory_size = NV_RM_PAGES_PER_OS_PAGE * os_get_num_phys_pages(); // if known, relay the number of system memory pages (in terms of RM page // size) to the RM; this is needed for e.g. TurboCache parts. if (system_memory_size != (NvU64)-1) pOS->SystemMemorySize = system_memory_size; // Setup any ThreadState defaults threadStateInitSetupFlags(THREAD_STATE_SETUP_FLAGS_ENABLED | THREAD_STATE_SETUP_FLAGS_TIMEOUT_ENABLED | THREAD_STATE_SETUP_FLAGS_SLI_LOGIC_ENABLED | THREAD_STATE_SETUP_FLAGS_DO_NOT_INCLUDE_SLEEP_TIME_ENABLED); return NV_TRUE; } void RmShutdownRm(void) { NV_PRINTF(LEVEL_INFO, "shutdown rm\n"); RmDestroyRegistry(NULL); // Free objects created with RmInitRm, including the system object RmDestroyRm(); } // // osAttachGpu // // This routine is used as a callback by the gpumgrAttachGpu // interface to allow os-dependent code to set up any state // before engine construction begins. // NV_STATUS osAttachGpu( OBJGPU *pGpu, void *pOsGpuInfo ) { nv_state_t *nv = (nv_state_t *)pOsGpuInfo; nv_priv_t *nvp; nvp = NV_GET_NV_PRIV(nv); nvp->pGpu = pGpu; NV_SET_NV_STATE(pGpu, (void *)nv); initUnixSpecificRegistry(pGpu); // Assign default values to Registry keys for VGX if (os_is_vgx_hyper()) { initVGXSpecificRegistry(pGpu); } return NV_OK; } NV_STATUS osDpcAttachGpu( OBJGPU *pGpu, void *pOsGpuInfo ) { return NV_OK; // Nothing to do for unix } void osDpcDetachGpu( OBJGPU *pGpu ) { return; // Nothing to do for unix } NV_STATUS osHandleGpuLost ( OBJGPU *pGpu ) { nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NvU32 pmc_boot_0; // Determine if we've already run the handler if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_CONNECTED)) { return NV_OK; } pmc_boot_0 = NV_PRIV_REG_RD32(nv->regs->map_u, NV_PMC_BOOT_0); if (pmc_boot_0 != nvp->pmc_boot_0) { // // This doesn't support PEX Reset and Recovery yet. // This will help to prevent accessing registers of a GPU // which has fallen off the bus. // nvErrorLog_va((void *)pGpu, ROBUST_CHANNEL_GPU_HAS_FALLEN_OFF_THE_BUS, "GPU has fallen off the bus."); NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU has fallen off the bus.\n"); if (pGpu->boardInfo != NULL && pGpu->boardInfo->serialNumber[0] != '\0') { NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU serial number is %s.\n", pGpu->boardInfo->serialNumber); } gpuSetDisconnectedProperties(pGpu); // Trigger the OS's PCI recovery mechanism if (nv_pci_trigger_recovery(nv) != NV_OK) { // // Initiate a crash dump immediately, since the OS doesn't appear // to have a mechanism wired up for attempted recovery. // (void) RmLogGpuCrash(pGpu); } else { // // Make the SW state stick around until the recovery can start, but // don't change the PDB property: this is only used to report to // clients whether or not persistence mode is enabled, and we'll // need it after the recovery callbacks to restore the correct // persistence mode for the GPU. // osModifyGpuSwStatePersistence(pGpu->pOsGpuInfo, NV_TRUE); } OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); // Set SURPRISE_REMOVAL flag for eGPU to help in device removal. if ((pCl != NULL) && pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU)) { nv->flags |= NV_FLAG_IN_SURPRISE_REMOVAL; } DBG_BREAKPOINT(); } return NV_OK; } /* * Initialize the required GPU information by doing RMAPI control calls * and store the same in the UNIX specific data structures. */ static NV_STATUS RmInitGpuInfoWithRmApi ( OBJGPU *pGpu ) { RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pGpuInfoParams = { 0 }; NV_STATUS status; // LOCK: acquire GPUs lock status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status != NV_OK) { return status; } pGpuInfoParams = portMemAllocNonPaged(sizeof(*pGpuInfoParams)); if (pGpuInfoParams == NULL) { // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); return NV_ERR_NO_MEMORY; } portMemSet(pGpuInfoParams, 0, sizeof(*pGpuInfoParams)); pGpuInfoParams->gpuInfoListSize = 3; pGpuInfoParams->gpuInfoList[0].index = NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED; pGpuInfoParams->gpuInfoList[1].index = NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED; pGpuInfoParams->gpuInfoList[2].index = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY; status = pRmApi->Control(pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, NV2080_CTRL_CMD_GPU_GET_INFO_V2, pGpuInfoParams, sizeof(*pGpuInfoParams)); if (status == NV_OK) { nvp->b_4k_page_isolation_required = (pGpuInfoParams->gpuInfoList[0].data == NV2080_CTRL_GPU_INFO_INDEX_4K_PAGE_ISOLATION_REQUIRED_YES); nvp->b_mobile_config_enabled = (pGpuInfoParams->gpuInfoList[1].data == NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES); nv->dma_buf_supported = (pGpuInfoParams->gpuInfoList[2].data == NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES); } portMemFree(pGpuInfoParams); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); return status; } static void RmSetSocDispDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvAddr = (GPUHWREG*) nv->regs->map; gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvPAddr = nv->regs->cpu_address; gpuAttachArg->socDeviceArgs.deviceMapping[SOC_DEV_MAPPING_DISP].gpuNvLength = (NvU32) nv->regs->size; } static void RmSetSocDpauxDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void RmSetSocHdacodecDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void RmSetSocMipiCalDeviceMappings( GPUATTACHARG *gpuAttachArg, nv_state_t *nv ) { } static void osInitNvMapping( nv_state_t *nv, NvU32 *pDeviceReference, UNIX_STATUS *status ) { OBJGPU *pGpu; OBJSYS *pSys = SYS_GET_INSTANCE(); GPUATTACHARG *gpuAttachArg; nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NvU32 deviceInstance; NvU32 data = 0; NV_PRINTF(LEVEL_INFO, "osInitNvMapping:\n"); // allocate the next available gpu device number status->rmStatus = gpumgrAllocGpuInstance(pDeviceReference); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot get valid gpu instance\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); return; } // RM_BASIC_LOCK_MODEL: allocate GPU lock status->rmStatus = rmGpuLockAlloc(*pDeviceReference); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** cannot allocate GPU lock\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } // attach default single-entry broadcast device for this gpu status->rmStatus = gpumgrCreateDevice(&deviceInstance, NVBIT(*pDeviceReference), NULL); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot attach bc gpu\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_CREATE_DEV_FAILED); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } // init attach state gpuAttachArg = portMemAllocNonPaged(sizeof(GPUATTACHARG)); if (gpuAttachArg == NULL) { NV_PRINTF(LEVEL_ERROR, "*** Cannot allocate gpuAttachArg\n"); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ALLOC_GPU_FAILED); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } portMemSet(gpuAttachArg, 0, sizeof(GPUATTACHARG)); if (NV_IS_SOC_DISPLAY_DEVICE(nv)) { gpuAttachArg->socDeviceArgs.specified = NV_TRUE; RmSetSocDispDeviceMappings(gpuAttachArg, nv); RmSetSocDpauxDeviceMappings(gpuAttachArg, nv); RmSetSocHdacodecDeviceMappings(gpuAttachArg, nv); RmSetSocMipiCalDeviceMappings(gpuAttachArg, nv); gpuAttachArg->socDeviceArgs.socChipId0 = nv->disp_sw_soc_chip_id; gpuAttachArg->socDeviceArgs.iovaspaceId = nv->iovaspace_id; } else { gpuAttachArg->fbPhysAddr = nv->fb->cpu_address; gpuAttachArg->fbBaseAddr = (GPUHWREG*) 0; // not mapped gpuAttachArg->devPhysAddr = nv->regs->cpu_address; gpuAttachArg->regBaseAddr = (GPUHWREG*) nv->regs->map; gpuAttachArg->intLine = 0; // don't know yet gpuAttachArg->instPhysAddr = nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address; gpuAttachArg->instBaseAddr = (GPUHWREG*) 0; // not mapped gpuAttachArg->regLength = nv->regs->size; gpuAttachArg->fbLength = nv->fb->size; gpuAttachArg->instLength = nv->bars[NV_GPU_BAR_INDEX_IMEM].size; gpuAttachArg->iovaspaceId = nv->iovaspace_id; } // // we need this to check if we are running on virtual GPU // in gpuBindHal function later. // gpuAttachArg->nvDomainBusDeviceFunc = nv_encode_pci_info(&nv->pci_info); gpuAttachArg->bRequestFwClientRm = nv->request_fw_client_rm; gpuAttachArg->pOsAttachArg = (void *)nv; // use gpu manager to attach gpu status->rmStatus = gpumgrAttachGpu(*pDeviceReference, gpuAttachArg); portMemFree(gpuAttachArg); if (status->rmStatus != NV_OK) { gpumgrDestroyDevice(deviceInstance); RM_SET_ERROR(*status, RM_INIT_GPU_GPUMGR_ATTACH_GPU_FAILED); NV_PRINTF(LEVEL_ERROR, "*** Cannot attach gpu\n"); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(*pDeviceReference); return; } nvp->flags |= NV_INIT_FLAG_GPUMGR_ATTACH; pGpu = gpumgrGetGpu(*pDeviceReference); sysInitRegistryOverrides(pSys); sysApplyLockingPolicy(pSys); pGpu->busInfo.IntLine = nv->interrupt_line; // // Set the DMA address size as soon as we have the HAL to call to // determine the precise number of physical address bits supported // by the architecture. DMA allocations should not be made before // this point. // nv_set_dma_address_size(nv, gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM)); pGpu->dmaStartAddress = (RmPhysAddr)nv_get_dma_start_address(nv); if (nv->fb != NULL) { pGpu->registerAccess.gpuFbAddr = (GPUHWREG*) nv->fb->map; pGpu->busInfo.gpuPhysFbAddr = nv->fb->cpu_address; } // set default parent gpu gpumgrSetParentGPU(pGpu, pGpu); NV_PRINTF(LEVEL_INFO, "device instance : %d\n", *pDeviceReference); NV_PRINTF(LEVEL_INFO, "NV regs using linear address : 0x%p\n", pGpu->deviceMappings[SOC_DEV_MAPPING_DISP].gpuNvAddr); NV_PRINTF(LEVEL_INFO, "NV fb using linear address : 0x%p\n", pGpu->registerAccess.gpuFbAddr); pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED, NV_TRUE); pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_FALSE); if (osReadRegistryDword(pGpu, NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR, &data) == NV_OK) { if (data == NV_REG_PROCESS_NONSTALL_INTR_IN_LOCKLESS_ISR_ENABLE) { pGpu->setProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS, NV_TRUE); } } if (!os_is_vgx_hyper()) { pGpu->setProperty(pGpu, PDB_PROP_GPU_ALLOW_PAGE_RETIREMENT, NV_TRUE); } if ((osReadRegistryDword(NULL, NV_REG_PRESERVE_VIDEO_MEMORY_ALLOCATIONS, &data) == NV_OK) && data) { KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); memmgrSetPmaForcePersistence(pMemoryManager, NV_TRUE); pKernelMemorySystem->bPreserveComptagBackingStoreOnSuspend = NV_TRUE; nv->preserve_vidmem_allocations = NV_TRUE; } } void osInitScalabilityOptions ( OBJGPU *pGpu, void *pDeviceExtension ) { } static NV_STATUS osInitScalability( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); osInitScalabilityOptions(pGpu, NULL); // We need PCI-E chipset information very early. if (pCl != NULL) { return clInitPcie(pGpu, pCl); } else { return NV_ERR_NOT_SUPPORTED; } } static NV_STATUS osTeardownScalability( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); return clTeardownPcie(pGpu, pCl); } static void populateDeviceAttributes( OBJGPU *pGpu, nv_state_t *nv ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); if ((pCl != NULL) && pCl->getProperty(pCl, PDB_PROP_CL_IS_EXTERNAL_GPU)) { nv->is_external_gpu = NV_TRUE; } } static void RmSetConsolePreservationParams(OBJGPU *pGpu) { nv_state_t *nv = NV_GET_NV_STATE(pGpu); NvU64 fbBaseAddress = 0; NvU64 fbConsoleSize = 0; MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); // // PDB_PROP_GPU_PRIMARY_DEVICE should be NV_FALSE for vGPU configuration so // return early // if (os_is_vgx_hyper() || IS_VIRTUAL(pGpu)) return; // // Check the OS layer for any video memory used by a console // driver that should be reserved. // // If the console driver is using some amount of memory, its // mapping is presumably either pointing at the start of BAR1 // (this is the mapping the EFI GOP driver provides for // efifb), or the upper 16MB of BAR2 (this is the mapping the // VBIOS provides for, e.g., vesafb). // // In the BAR2 case, there is nothing more to do: the upper // 16MB of BAR2 will remain in physical mode // (see the documentation for NV_PBUS_BAR2_BLOCK in dev_bus.ref), // and the console driver can continue to access it. // // In the BAR1 case, RM will put all of BAR1 into virtual // mode, so we need to create a BAR1 mapping for the console // driver to use. // // If the console driver is not using _either_ of those // mappings, then the console driver will not be able to // access the memory we reserve. This happens on some // UEFI systems with multiple GPUs, because the firmware // initializes the GOP driver on more than one GPU. In // that case, just skip reserving anything for GPUs where // neither BAR1 nor BAR2 match the console's base // address. // fbConsoleSize = rm_get_uefi_console_size(nv, &fbBaseAddress); if (fbConsoleSize == 0) { if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_UEFI)) { fbConsoleSize = 0x40000; } else { NV_PRINTF(LEVEL_INFO, "No Frame Buffer Present\n"); } } if ((fbConsoleSize > 0) && (fbBaseAddress != 0)) { KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); pKernelBus->bPreserveBar1ConsoleEnabled = (fbBaseAddress == nv->fb->cpu_address); } pMemoryManager->Ram.ReservedConsoleDispMemSize = NV_ALIGN_UP(fbConsoleSize, 0x10000); } static NV_STATUS RmInitDeviceDma( nv_state_t *nv ) { if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) { OBJSYS *pSys = SYS_GET_INSTANCE(); POBJVMM pVmm = SYS_GET_VMM(pSys); POBJVASPACE pIOVAS; NV_STATUS status = vmmCreateVaspace(pVmm, IO_VASPACE_A, nv->iovaspace_id, 0, 0ULL, ~0ULL, 0ULL, 0ULL, NULL, VASPACE_FLAGS_ENABLE_VMM, &pIOVAS); if (status != NV_OK) { return status; } } return NV_OK; } static void RmTeardownDeviceDma( nv_state_t *nv ) { if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE) { OBJSYS *pSys = SYS_GET_INSTANCE(); POBJVMM pVmm = SYS_GET_VMM(pSys); POBJVASPACE pIOVAS; if (NV_OK == vmmGetVaspaceFromId(pVmm, nv->iovaspace_id, IO_VASPACE_A, &pIOVAS)) { vmmDestroyVaspace(pVmm, pIOVAS); } } } static void RmInitNvDevice( NvU32 deviceReference, UNIX_STATUS *status ) { // set the device context OBJGPU *pGpu = gpumgrGetGpu(deviceReference); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV_PRINTF(LEVEL_INFO, "RmInitNvDevice:\n"); NV_PRINTF(LEVEL_INFO, "device instance : 0x%08x\n", deviceReference); // initialize all engines -- calls back osInitMapping() status->rmStatus = gpumgrStatePreInitGpu(pGpu); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot pre-initialize the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_PRE_INIT_FAILED); return; } os_disable_console_access(); status->rmStatus = gpumgrStateInitGpu(pGpu); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot initialize the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_STATE_INIT_FAILED); os_enable_console_access(); return; } nvp->flags |= NV_INIT_FLAG_GPU_STATE; KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); // // Make sure bifIsMSIEnabled() gets a chance to look up the OS PCI // handle for this GPU. This must happen after GPU state // initialization. // if (pKernelBif != NULL) { kbifCheckAndRearmMSI(pGpu, pKernelBif); } // Set RM's interrupt enable state to zero here so that interrupts // won't be enabled during loading Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); } status->rmStatus = gpumgrStateLoadGpu(pGpu, GPU_STATE_DEFAULT); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Cannot load state into the device\n"); RM_SET_ERROR(*status, RM_INIT_GPU_LOAD_FAILED); os_enable_console_access(); return; } nvp->flags |= NV_INIT_FLAG_GPU_STATE_LOAD; os_enable_console_access(); status->rmStatus = gpuPerformUniversalValidation_HAL(pGpu); if (status->rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "*** Failed universal validation\n"); RM_SET_ERROR(*status, RM_INIT_GPU_UNIVERSAL_VALIDATION_FAILED); return; } return; } NV_STATUS osInitMapping( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); nv_state_t *nv = NV_GET_NV_STATE(pGpu); nv_priv_t *nvp = NV_GET_NV_PRIV(nv); NV_PRINTF(LEVEL_INFO, "osInitMapping:\n"); // Some PCI BIOSs leave the ROM mapped. This causes problems if it overlays system RAM. Just disable it. GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_12, 0); // make sure our PCI latency timer is sufficient (max it out) GPU_BUS_CFG_WR32(pGpu, NV_CONFIG_PCI_NV_3, DRF_DEF(_CONFIG, _PCI_NV_3, _LATENCY_TIMER, _248_CLOCKS)); if (pCl != NULL) { initCoreLogic(pGpu); } nvp->flags |= NV_INIT_FLAG_CORE_LOGIC; return NV_OK; } // end of osInitMapping() static NV_STATUS initCoreLogic( OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); #if defined(NVCPU_X86_64) OBJOS *pOS = SYS_GET_OS(pSys); if (!os_pat_supported()) pOS->setProperty(pOS, PDB_PROP_OS_PAT_UNSUPPORTED, NV_TRUE); #endif return clInit(pGpu, pCl); } static NV_STATUS teardownCoreLogic( OBJOS *pOS, OBJGPU *pGpu ) { OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); return clTeardown(pGpu, pCl); } static void RmTeardownDpauxRegisters( nv_state_t *nv ) { } static void RmTeardownHdacodecRegisters( nv_state_t *nv ) { } static void RmTeardownMipiCalRegisters( nv_state_t *nv ) { } static NV_STATUS RmTeardownRegisters( nv_state_t *nv ) { NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Tearing down registers\n"); if (nv->regs && nv->regs->map) { osUnmapKernelSpace(nv->regs->map, nv->regs->size); nv->regs->map = 0; nv->regs->map_u = NULL; } RmTeardownDpauxRegisters(nv); RmTeardownHdacodecRegisters(nv); RmTeardownMipiCalRegisters(nv); return NV_OK; } static NV_STATUS RmSetupDpauxRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static NV_STATUS RmSetupHdacodecRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static NV_STATUS RmSetupMipiCalRegisters( nv_state_t *nv, UNIX_STATUS *status ) { return NV_OK; } static void RmSetupRegisters( nv_state_t *nv, UNIX_STATUS *status ) { NV_STATUS ret; NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmSetupRegisters for 0x%x:0x%x\n", nv->pci_info.vendor_id, nv->pci_info.device_id); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "pci config info:\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, " registers look like: " NvP64_fmt " " NvP64_fmt, nv->regs->cpu_address, nv->regs->size); if (nv->fb != NULL) { NV_DEV_PRINTF(NV_DBG_SETUP, nv, " fb looks like: " NvP64_fmt " " NvP64_fmt, nv->fb->cpu_address, nv->fb->size); } { nv_os_map_kernel_space(nv, nv->regs); } if (nv->regs->map == NULL) { NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "Failed to map regs registers!!\n"); RM_SET_ERROR(*status, RM_INIT_REG_SETUP_FAILED); status->rmStatus = NV_ERR_OPERATING_SYSTEM; return; } NV_DEV_PRINTF(NV_DBG_SETUP, nv, "Successfully mapped framebuffer and registers\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "final mappings:\n"); NV_DEV_PRINTF(NV_DBG_SETUP, nv, " regs: " NvP64_fmt " " NvP64_fmt " 0x%p\n", nv->regs->cpu_address, nv->regs->size, nv->regs->map); ret = RmSetupDpauxRegisters(nv, status); if (ret != NV_OK) goto err_unmap_disp_regs; ret = RmSetupHdacodecRegisters(nv, status); if (ret != NV_OK) { RmTeardownDpauxRegisters(nv); goto err_unmap_disp_regs; } ret = RmSetupMipiCalRegisters(nv, status); if (ret != NV_OK) { RmTeardownHdacodecRegisters(nv); RmTeardownDpauxRegisters(nv); goto err_unmap_disp_regs; } return; err_unmap_disp_regs: if (nv->regs && nv->regs->map) { osUnmapKernelSpace(nv->regs->map, nv->regs->size); nv->regs->map = 0; } return; } NvBool RmInitPrivateState( nv_state_t *pNv ) { nv_priv_t *nvp; NvU32 gpuId; NvU32 pmc_boot_0 = 0; NvU32 pmc_boot_42 = 0; NV_SET_NV_PRIV(pNv, NULL); if (!NV_IS_SOC_DISPLAY_DEVICE(pNv) && !NV_IS_SOC_IGPU_DEVICE(pNv)) { pNv->regs->map_u = os_map_kernel_space(pNv->regs->cpu_address, os_page_size, NV_MEMORY_UNCACHED); if (pNv->regs->map_u == NULL) { NV_PRINTF(LEVEL_ERROR, "failed to map GPU registers (DISABLE_INTERRUPTS).\n"); return NV_FALSE; } pmc_boot_0 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_0); pmc_boot_42 = NV_PRIV_REG_RD32(pNv->regs->map_u, NV_PMC_BOOT_42); os_unmap_kernel_space(pNv->regs->map_u, os_page_size); pNv->regs->map_u = NULL; } if (os_alloc_mem((void **)&nvp, sizeof(*nvp)) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to allocate private device state.\n"); return NV_FALSE; } gpuId = nv_generate_id_from_pci_info(&pNv->pci_info); if (gpumgrRegisterGpuId(gpuId, nv_encode_pci_info(&pNv->pci_info)) != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to register GPU with GPU manager.\n"); os_free_mem(nvp); return NV_FALSE; } pNv->gpu_id = gpuId; pNv->iovaspace_id = nv_requires_dma_remap(pNv) ? gpuId : NV_IOVA_DOMAIN_NONE; kvgpumgrAttachGpu(pNv->gpu_id); // // Set up a reasonable default DMA address size, based on the minimum // possible on currently supported GPUs. // nv_set_dma_address_size(pNv, NV_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH); os_mem_set(nvp, 0, sizeof(*nvp)); nvp->status = NV_ERR_INVALID_STATE; nvp->pmc_boot_0 = pmc_boot_0; nvp->pmc_boot_42 = pmc_boot_42; NV_SET_NV_PRIV(pNv, nvp); return NV_TRUE; } void RmClearPrivateState( nv_state_t *pNv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); NvU32 status; void *pVbiosCopy = NULL; void *pRegistryCopy = NULL; NvU32 vbiosSize; NvU32 *pEfiDisplayCache; NvU32 efiDisplayCacheSize; nv_i2c_adapter_entry_t i2c_adapters[MAX_I2C_ADAPTERS]; nv_dynamic_power_t dynamicPowerCopy; NvU32 x = 0; NvU32 pmc_boot_0, pmc_boot_42; // // Do not clear private state after GPU resets, it is used while // recovering the GPU. Only clear the pGpu pointer, which is // restored during next initialization cycle. // if (pNv->flags & NV_FLAG_IN_RECOVERY) { nvp->pGpu = NULL; } status = nvp->status; pVbiosCopy = nvp->pVbiosCopy; vbiosSize = nvp->vbiosSize; pRegistryCopy = nvp->pRegistry; pEfiDisplayCache = nvp->efi.display.pCache; efiDisplayCacheSize = nvp->efi.display.cacheSize; dynamicPowerCopy = nvp->dynamic_power; pmc_boot_0 = nvp->pmc_boot_0; pmc_boot_42 = nvp->pmc_boot_42; for (x = 0; x < MAX_I2C_ADAPTERS; x++) { i2c_adapters[x] = nvp->i2c_adapters[x]; } portMemSet(nvp, 0, sizeof(nv_priv_t)); nvp->status = status; nvp->pVbiosCopy = pVbiosCopy; nvp->vbiosSize = vbiosSize; nvp->pRegistry = pRegistryCopy; nvp->efi.display.pCache = pEfiDisplayCache; nvp->efi.display.cacheSize = efiDisplayCacheSize; nvp->dynamic_power = dynamicPowerCopy; nvp->pmc_boot_0 = pmc_boot_0; nvp->pmc_boot_42 = pmc_boot_42; for (x = 0; x < MAX_I2C_ADAPTERS; x++) { nvp->i2c_adapters[x] = i2c_adapters[x]; } nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; } void RmFreePrivateState( nv_state_t *pNv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(pNv); gpumgrUnregisterGpuId(pNv->gpu_id); kvgpumgrDetachGpu(pNv->gpu_id); RmDestroyRegistry(pNv); if (nvp != NULL) { portMemFree(nvp->pVbiosCopy); portMemFree(nvp->efi.display.pCache); os_free_mem(nvp); } NV_SET_NV_PRIV(pNv, NULL); } NvBool RmPartiallyInitAdapter( nv_state_t *nv ) { NV_PRINTF(LEVEL_INFO, "%s: %04x:%02x:%02x.0\n", __FUNCTION__, nv->pci_info.domain, nv->pci_info.bus, nv->pci_info.slot); nv_start_rc_timer(nv); return NV_TRUE; } static NV_STATUS RmInitX86Emu( OBJGPU *pGpu ) { NV_STATUS status = NV_OK; nv_state_t *nv = NV_GET_NV_STATE(pGpu); PORT_UNREFERENCED_VARIABLE(nv); #if NVCPU_IS_X86_64 status = RmInitX86EmuState(pGpu); #else // We don't expect a "primary VGA" adapter on non-amd64 platforms NV_ASSERT(!NV_PRIMARY_VGA(nv)); #endif return status; } static NV_STATUS RmRegisterGpudb( OBJGPU *pGpu ) { NV_STATUS rmStatus; const NvU8 *pGid; nv_state_t *pNv = NV_GET_NV_STATE(pGpu); pGid = RmGetGpuUuidRaw(pNv); if (pGid == NULL) { NV_PRINTF(LEVEL_ERROR, "Failed to get UUID\n"); return NV_ERR_OPERATING_SYSTEM; } rmStatus = gpudbRegisterGpu(pGid, &pGpu->gpuClData.upstreamPort.addr, pGpu->busInfo.nvDomainBusDeviceFunc); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Failed to register GPU with GPU data base\n"); } return rmStatus; } static void RmUnixFreeRmApi( nv_state_t *nv ) { RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); if (nv->rmapi.hClient != 0) { pRmApi->Free(pRmApi, nv->rmapi.hClient, nv->rmapi.hClient); } portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); } static NvBool RmUnixAllocRmApi( nv_state_t *nv, NvU32 deviceId ) { NV0080_ALLOC_PARAMETERS deviceParams = { 0 }; NV2080_ALLOC_PARAMETERS subDeviceParams = { 0 }; RM_API *pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL); portMemSet(&nv->rmapi, 0, sizeof(nv->rmapi)); if (pRmApi->AllocWithHandle( pRmApi, NV01_NULL_OBJECT, NV01_NULL_OBJECT, NV01_NULL_OBJECT, NV01_ROOT, &nv->rmapi.hClient) != NV_OK) { goto fail; } // // Any call to rmapiDelPendingDevices() will internally delete the UNIX OS // layer RMAPI handles. Set this flag to preserve these handles. These // handles will be freed explicitly by RmUnixFreeRmApi(). // if (!rmclientSetClientFlagsByHandle(nv->rmapi.hClient, RMAPI_CLIENT_FLAG_RM_INTERNAL_CLIENT)) { goto fail; } deviceParams.deviceId = deviceId; if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hClient, &nv->rmapi.hDevice, NV01_DEVICE_0, &deviceParams) != NV_OK) { goto fail; } subDeviceParams.subDeviceId = 0; if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hDevice, &nv->rmapi.hSubDevice, NV20_SUBDEVICE_0, &subDeviceParams) != NV_OK) { goto fail; } // // The NV40_I2C allocation expected to fail, if it is disabled // with RM config. // if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hSubDevice, &nv->rmapi.hI2C, NV40_I2C, NULL) != NV_OK) { nv->rmapi.hI2C = 0; } // // The NV04_DISPLAY_COMMON allocation expected to fail for displayless // system. nv->rmapi.hDisp value needs to be checked before doing display // related control calls. // if (pRmApi->Alloc( pRmApi, nv->rmapi.hClient, nv->rmapi.hDevice, &nv->rmapi.hDisp, NV04_DISPLAY_COMMON, NULL) != NV_OK) { nv->rmapi.hDisp = 0; } return NV_TRUE; fail: RmUnixFreeRmApi(nv); return NV_FALSE; } NvBool RmInitAdapter( nv_state_t *nv ) { NvU32 devicereference = 0; UNIX_STATUS status = INIT_UNIX_STATUS; nv_priv_t *nvp; NvBool retVal = NV_FALSE; OBJSYS *pSys; OBJGPU *pGpu = NULL; OBJOS *pOS; KernelDisplay *pKernelDisplay; const void *gspFwHandle = NULL; const void *gspFwLogHandle = NULL; GSP_FIRMWARE gspFw = {0}; PORT_UNREFERENCED_VARIABLE(gspFw); NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter\n"); nv->flags &= ~NV_FLAG_PASSTHRU; RmSetupRegisters(nv, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) goto failed; nvp = NV_GET_NV_PRIV(nv); nvp->status = NV_ERR_OPERATING_SYSTEM; status.rmStatus = RmInitDeviceDma(nv); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Cannot configure the device for DMA\n"); RM_SET_ERROR(status, RM_INIT_GPU_DMA_CONFIGURATION_FAILED); goto shutdown; } nvp->flags |= NV_INIT_FLAG_DMA; pSys = SYS_GET_INSTANCE(); // // WAR: If the below UEFI property is set, display RM will attempt to read // the state cache during RM init in order to retrieve a snapshot of the // display state that the UEFI driver has already programmed. On Orin // (T234D), the UEFI boot flow is being enabled on Linux, but our UEFI // driver doesn't have any display support right now. As such, our UEFI // driver won't allocate any of the display channels, which means that RM // will attempt to read the state cache for uninitialized channels. WAR this // issue by un-setting the below UEFI property for now. // // JIRA task TDS-5094 tracks adding display support to the UEFI driver. // if (NV_IS_SOC_DISPLAY_DEVICE(nv)) { pSys->setProperty(pSys, PDB_PROP_SYS_IS_UEFI, NV_FALSE); } // // Get firmware from the OS, if requested, and decide if RM will run as a // firmware client. // if (nv->request_firmware) { NvU32 gpuArch = (DRF_VAL(_PMC, _BOOT_42, _ARCHITECTURE, nvp->pmc_boot_42) << GPU_ARCH_SHIFT); NvU32 gpuImpl = DRF_VAL(_PMC, _BOOT_42, _IMPLEMENTATION, nvp->pmc_boot_42); nv_firmware_chip_family_t chipFamily = nv_firmware_get_chip_family(gpuArch, gpuImpl); nv_set_dma_address_size(nv, NV_GSP_GPU_MIN_SUPPORTED_DMA_ADDR_WIDTH); gspFwHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP, chipFamily, &gspFw.pBuf, &gspFw.size); if (gspFwHandle == NULL && !nv->allow_fallback_to_monolithic_rm) { RM_SET_ERROR(status, RM_INIT_FIRMWARE_FETCH_FAILED); goto shutdown; } else if (gspFwHandle != NULL) { #if LIBOS_LOG_DECODE_ENABLE if (nv->enable_firmware_logs) { gspFwLogHandle = nv_get_firmware(nv, NV_FIRMWARE_TYPE_GSP_LOG, chipFamily, &gspFw.pLogElf, &gspFw.logElfSize); if (gspFwLogHandle == NULL) { NV_PRINTF(LEVEL_ERROR, "Failed to load gsp_log_*.bin, no GSP-RM logs will be printed (non-fatal)\n"); } } #endif nv->request_fw_client_rm = NV_TRUE; } else { nv->request_fw_client_rm = NV_FALSE; } } // initialize the RM device register mapping osInitNvMapping(nv, &devicereference, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) { switch (status.rmStatus) { case NV_ERR_NOT_SUPPORTED: nvp->status = NV_ERR_NOT_SUPPORTED; break; } NV_PRINTF(LEVEL_ERROR, "osInitNvMapping failed, bailing out of RmInitAdapter\n"); goto shutdown; } // // now we can have a pdev for the first time... // pGpu = gpumgrGetGpu(devicereference); pOS = SYS_GET_OS(pSys); status.rmStatus = osInitScalability(pGpu); if (status.rmStatus == NV_OK) { nvp->flags |= NV_INIT_FLAG_SCALABILITY; } else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) { RM_SET_ERROR(status, RM_INIT_SCALABILITY_FAILED); goto shutdown; } KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu); if ((pKernelFsp != NULL) && !IS_GSP_CLIENT(pGpu) && !IS_VIRTUAL(pGpu)) { status.rmStatus = kfspSendBootCommands_HAL(pGpu, pKernelFsp); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "FSP boot command failed.\n"); goto shutdown; } } RmSetConsolePreservationParams(pGpu); RmInitAcpiMethods(pOS, pSys, pGpu); // // If GSP fw RM support is enabled then start the GSP microcode // (including the task running the full instance of the RM) and // exchange the necessary initial RPC messages before continuing // with GPU initialization here. // if (IS_GSP_CLIENT(pGpu)) { status.rmStatus = kgspInitRm(pGpu, GPU_GET_KERNEL_GSP(pGpu), &gspFw); if (status.rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "Cannot initialize GSP firmware RM\n"); RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); goto shutdown; } } else if (nv->request_fw_client_rm) { // We were expecting to enable GSP-RM but something went wrong. if (!nv->allow_fallback_to_monolithic_rm) { RM_SET_ERROR(status, RM_INIT_FIRMWARE_POLICY_FAILED); goto shutdown; } else { NV_PRINTF(LEVEL_NOTICE, "Falling back to monolithic RM\n"); } } pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); if (pKernelDisplay != NULL) { kdispSetWarPurgeSatellitesOnCoreFree(pKernelDisplay, NV_TRUE); } if (IS_PASSTHRU(pGpu)) nv->flags |= NV_FLAG_PASSTHRU; populateDeviceAttributes(pGpu, nv); initVendorSpecificRegistry(pGpu, nv->pci_info.device_id); if (!IS_VIRTUAL(pGpu) && !IS_GSP_CLIENT(pGpu)) { initNbsiTable(pGpu); } // finally, initialize the device RmInitNvDevice(devicereference, &status); if (! RM_INIT_SUCCESS(status.initStatus) ) { NV_PRINTF(LEVEL_ERROR, "RmInitNvDevice failed, bailing out of RmInitAdapter\n"); switch (status.rmStatus) { case NV_ERR_INSUFFICIENT_POWER: nvp->status = NV_ERR_INSUFFICIENT_POWER; NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "GPU does not have the necessary power cables connected.\n"); break; } goto shutdown; } // LOCK: acquire GPUs lock status.rmStatus = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_INIT); if (status.rmStatus != NV_OK) { goto shutdown; } status.rmStatus = osVerifySystemEnvironment(pGpu); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_SYS_ENVIRONMENT_FAILED); switch (status.rmStatus) { case NV_ERR_IRQ_NOT_FIRING: nvp->status = NV_ERR_IRQ_NOT_FIRING; break; } NV_PRINTF(LEVEL_ERROR, "RmVerifySystemEnvironment failed, bailing!\n"); goto shutdown; } Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_HARDWARE); } KernelRc *pKernelRc = GPU_GET_KERNEL_RC(pGpu); // initialize the watchdog (disabled by default) status.rmStatus = pKernelRc != NULL ? krcWatchdogInit_HAL(pGpu, pKernelRc) : NV_ERR_NOT_SUPPORTED; if (status.rmStatus == NV_OK) { krcWatchdogDisable(pKernelRc); nvp->flags |= NV_INIT_FLAG_FIFO_WATCHDOG; } else if (status.rmStatus == NV_ERR_NOT_SUPPORTED) { NV_PRINTF(LEVEL_INFO, "krcWatchdogInit returned _NOT_SUPPORTED. For Kepler GPUs in PGOB mode, this is normal\n"); } else { RM_SET_ERROR(status, RM_INIT_WATCHDOG_FAILED); NV_PRINTF(LEVEL_ERROR, "krcWatchdogInit failed, bailing out of RmInitAdapter\n"); goto shutdown; } nv_start_rc_timer(nv); nvp->status = NV_OK; if (!RmUnixAllocRmApi(nv, devicereference)) { RM_SET_ERROR(status, RM_INIT_ALLOC_RMAPI_FAILED); status.rmStatus = NV_ERR_GENERIC; goto shutdown; } status.rmStatus = RmInitGpuInfoWithRmApi(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_GPUINFO_WITH_RMAPI_FAILED); goto shutdown; } status.rmStatus = RmInitX86Emu(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_INIT_VBIOS_X86EMU_FAILED); NV_PRINTF(LEVEL_ERROR, "RmInitX86Emu failed, bailing out of RmInitAdapter\n"); goto shutdown; } // i2c only on master device?? RmI2cAddGpuPorts(nv); nvp->flags |= NV_INIT_FLAG_PUBLIC_I2C; // This fifo hal call will fail on pre-fermi gpus. In that case, userd // info will remain 0 and skipped by fb mmap code KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); NvU64 udaddr = 0; NvU32 udsize = 0; if (pKernelFifo != NULL) { status.rmStatus = kfifoGetUserdBar1MapInfo_HAL(pGpu, pKernelFifo, &udaddr, &udsize); if (status.rmStatus == NV_OK) { nv->ud.cpu_address = (nv->fb->cpu_address + udaddr); nv->ud.size = udsize; } else if (status.rmStatus != NV_ERR_NOT_SUPPORTED) { RM_SET_ERROR(status, RM_FIFO_GET_UD_BAR1_MAP_INFO_FAILED); NV_PRINTF(LEVEL_ERROR, "kfifoGetUserdBar1MapInfo failed, bailing out of RmInitAdapter\n"); goto shutdown; } } nv->flags &= ~NV_FLAG_IN_RECOVERY; pOS->setProperty(pOS, PDB_PROP_OS_SYSTEM_EVENTS_SUPPORTED, NV_TRUE); RmInitS0ixPowerManagement(nv); RmInitDeferredDynamicPowerManagement(nv); if (!NV_IS_SOC_DISPLAY_DEVICE(nv) && !NV_IS_SOC_IGPU_DEVICE(nv)) { status.rmStatus = RmRegisterGpudb(pGpu); if (status.rmStatus != NV_OK) { RM_SET_ERROR(status, RM_GPUDB_REGISTER_FAILED); goto shutdown; } } if (nvp->b_mobile_config_enabled) { NvU32 ac_plugged = 0; if (nv_acpi_get_powersource(&ac_plugged) == NV_OK) { // LOCK: acquire GPU lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_NONE) == NV_OK) { // // As we have already acquired the API Lock here, we are calling // RmPowerSourceChangeEvent directly instead of rm_power_source_change_event. // RmPowerSourceChangeEvent(nv, !ac_plugged); // UNLOCK: release GPU lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); } } } { // OpenRM support for features beyond what is used on Data Center GPUs // is still fairly immature, so for now require users to opt into use of // OpenRM with a special registry key, if not on a Data Center GPU. const GspStaticConfigInfo *pSCI = GPU_GET_GSP_STATIC_INFO(pGpu); if (pSCI->computeBranding != COMPUTE_BRANDING_TYPE_TESLA && ((pGpu->idInfo.PCIDeviceID >> 16) & 0xffff) != NV_PCI_DEVID_DEVICE_PG189_SKU600) { NvU32 data = NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DEFAULT; RmReadRegistryDword(nv, NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS, &data); if (data == NV_REG_OPENRM_ENABLE_UNSUPPORTED_GPUS_DISABLE) { if (!nv->printed_openrm_enable_unsupported_gpus_error) { nv_printf(NV_DBG_ERRORS, "NVRM: Open nvidia.ko is only ready for use on Data Center GPUs.\n"); nv_printf(NV_DBG_ERRORS, "NVRM: To force use of Open nvidia.ko on other GPUs, see the\n"); nv_printf(NV_DBG_ERRORS, "NVRM: 'OpenRmEnableUnsupportedGpus' kernel module parameter described\n"); nv_printf(NV_DBG_ERRORS, "NVRM: in the README.\n"); nv->printed_openrm_enable_unsupported_gpus_error = NV_TRUE; } RM_SET_ERROR(status, RM_INIT_FIRMWARE_INIT_FAILED); goto shutdown; } } } NV_DEV_PRINTF(NV_DBG_SETUP, nv, "RmInitAdapter succeeded!\n"); retVal = NV_TRUE; goto done; shutdown: nv->flags &= ~NV_FLAG_IN_RECOVERY; // call ShutdownAdapter to undo anything we've done above RmShutdownAdapter(nv); failed: NV_DEV_PRINTF(NV_DBG_ERRORS, nv, "RmInitAdapter failed! (0x%x:0x%x:%d)\n", status.initStatus, status.rmStatus, status.line); done: nv_put_firmware(gspFwHandle); nv_put_firmware(gspFwLogHandle); return retVal; } void RmShutdownAdapter( nv_state_t *nv ) { nv_priv_t *nvp = NV_GET_NV_PRIV(nv); OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); NV_STATUS rmStatus; if ((pGpu != NULL) && (nvp->flags & NV_INIT_FLAG_GPUMGR_ATTACH)) { NvU32 gpuInstance = gpuGetInstance(pGpu); NvU32 deviceInstance = gpuGetDeviceInstance(pGpu); OBJSYS *pSys = SYS_GET_INSTANCE(); RmUnixFreeRmApi(nv); nv->ud.cpu_address = 0; nv->ud.size = 0; // LOCK: acquire GPUs lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) { RmDestroyDeferredDynamicPowerManagement(nv); freeNbsiTable(pGpu); gpuFreeEventHandle(pGpu); OBJCL *pCl = SYS_GET_CL(pSys); OBJOS *pOS = GPU_GET_OS(pGpu); if (pCl != NULL) { if (nvp->flags & NV_INIT_FLAG_CORE_LOGIC) { teardownCoreLogic(pOS, pGpu); } else if (nvp->flags & NV_INIT_FLAG_SCALABILITY) { osTeardownScalability(pGpu); } } rmapiSetDelPendingClientResourcesFromGpuMask(NVBIT(gpuInstance)); rmapiDelPendingDevices(NVBIT(gpuInstance)); os_disable_console_access(); if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) { rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); NV_ASSERT(rmStatus == NV_OK); } if (nvp->flags & NV_INIT_FLAG_GPU_STATE) { rmStatus = gpuStateDestroy(pGpu); NV_ASSERT(rmStatus == NV_OK); } os_enable_console_access(); //if (nvp->flags & NV_INIT_FLAG_HAL) // destroyHal(pDev); #if NVCPU_IS_X86_64 RmFreeX86EmuState(pGpu); #endif gpumgrDetachGpu(gpuInstance); gpumgrDestroyDevice(deviceInstance); if (nvp->flags & NV_INIT_FLAG_DMA) { RmTeardownDeviceDma(nv); } RmClearPrivateState(nv); RmUnInitAcpiMethods(pSys); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); // RM_BASIC_LOCK_MODEL: free GPU lock rmGpuLockFree(deviceInstance); } } else { RmClearPrivateState(nv); } RmTeardownRegisters(nv); } void RmPartiallyDisableAdapter( nv_state_t *nv ) { NV_PRINTF(LEVEL_INFO, "%s: RM is in SW Persistence mode\n", __FUNCTION__); nv_stop_rc_timer(nv); } void RmDisableAdapter( nv_state_t *nv ) { NV_STATUS rmStatus; OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(nv); NvU32 gpuMask; nv_priv_t *nvp = NV_GET_NV_PRIV(nv); // // Normally, we re-enable interrupts when we release the lock // after RmDisableAdapter(), before disabling the bottom-half ISR. // If an interrupt came in in that time window, but the bottom half // wasn't scheduled to run in time, the semaphore acquired by the // top-half would never be released afterwards (since the bottom-half // would never run). This call to intrSetIntrEn() ensures that this won't // happen, by making sure interrupts remain disabled on the GPU in question after // RmDisableAdapter(). // Intr *pIntr = GPU_GET_INTR(pGpu); if (pIntr != NULL) { intrSetIntrEn(pIntr, INTERRUPT_TYPE_DISABLED); } if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SECONDARY_BUS_RESET) || pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_FULLCHIP_RESET)) { pGpu->setProperty(pGpu, PDB_PROP_GPU_IN_TIMEOUT_RECOVERY, NV_TRUE); nv->flags |= NV_FLAG_IN_RECOVERY; } // // Free the client allocated resources. // // This needs to happen prior to tearing down SLI state when SLI is enabled. // // Note this doesn't free RM internal resource allocations. Those are // freed during (gpumgrUpdateSLIConfig->...->)gpuStateUnload. // // We need to free resources for all GPUs linked in a group as // gpumgrUpdateSLIConfig will teardown GPU state for the entire set. // gpuMask = gpumgrGetGpuMask(pGpu); rmapiSetDelPendingClientResourcesFromGpuMask(gpuMask); rmapiDelPendingDevices(gpuMask); // LOCK: acquire GPUs lock if (rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DESTROY) == NV_OK) { nv_stop_rc_timer(nv); OBJSYS *pSys = SYS_GET_INSTANCE(); OBJCL *pCl = SYS_GET_CL(pSys); OBJOS *pOS = SYS_GET_OS(pSys); if (pCl != NULL) { teardownCoreLogic(pOS, pGpu); nvp->flags &= ~NV_INIT_FLAG_CORE_LOGIC; } if (nvp->flags & NV_INIT_FLAG_FIFO_WATCHDOG) { krcWatchdogShutdown(pGpu, GPU_GET_KERNEL_RC(pGpu)); nvp->flags &= ~NV_INIT_FLAG_FIFO_WATCHDOG; } os_disable_console_access(); if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) { rmStatus = gpuStateUnload(pGpu, GPU_STATE_DEFAULT); NV_ASSERT(rmStatus == NV_OK); nvp->flags &= ~NV_INIT_FLAG_GPU_STATE_LOAD; } os_enable_console_access(); // UNLOCK: release GPUs lock rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL); } } NV_STATUS RmGetAdapterStatus( nv_state_t *pNv, NvU32 *pStatus ) { // // This status is determined in RmInitAdapter(); the glue layer // requests it when the adapter failed to initialize to learn // more about the error condition. This is currently limited to // osVerifySystemEnvironment() failures. // nv_priv_t *nvp; nvp = NV_GET_NV_PRIV(pNv); if (nvp == NULL) { return NV_ERR_INVALID_STATE; } *pStatus = nvp->status; return NV_OK; } static void initVendorSpecificRegistry( OBJGPU *pGpu, NvU16 device_id ) { NV_STATUS rmStatus; NvU32 i; NvU32 subsystem_id; NvU32 subsystem_vendor_id; NvU32 subsystem_device_id; NvU32 vendor_id = 0; if (!pGpu) return; rmStatus = GPU_BUS_CFG_RD32(pGpu, NV_CONFIG_PCI_NV_11, &subsystem_id); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "%s: Cannot read NV_CONFIG_PCI_NV_11\n", __FUNCTION__); return; } subsystem_vendor_id = (subsystem_id & 0xffff); subsystem_device_id = (subsystem_id >> 16); for (i = 0; (nb_id_table[i].subsystem_vendor_id) != 0; i++) { if ((nb_id_table[i].subsystem_vendor_id == subsystem_vendor_id) && (nb_id_table[i].subsystem_device_id == subsystem_device_id) && (nb_id_table[i].gpu_device_id == device_id)) { vendor_id = subsystem_vendor_id; break; } } if (vendor_id != 0) { for (i = 0; nb_reg_table[i].vendor_id != 0; i++) { if (nb_reg_table[i].vendor_id == vendor_id) { osWriteRegistryDword(pGpu, nb_reg_table[i].name, nb_reg_table[i].data); } } } } static void initUnixSpecificRegistry( OBJGPU *pGpu ) { // By default, enable GPU reset on Unix osWriteRegistryDword(pGpu, "RMSecBusResetEnable", 1); osWriteRegistryDword(pGpu, "RMForcePcieConfigSave", 1); } void osRemoveGpu( NvU32 domain, NvU8 bus, NvU8 device ) { void *handle; handle = os_pci_init_handle(domain, bus, device, 0, NULL, NULL); if (handle != NULL) { os_pci_remove(handle); } } /* * Check to see if this board supports GPU exclusion. */ static NvBool RmIsExcludingAllowed( nv_state_t *pNv ) { NvU32 feature = 0; NvU32 brand; // DGX-2/HGX-2 systems pre-date the PBI call if (pNv->pci_info.device_id == 0x1db8) return NV_TRUE; if (pciPbiGetFeature(pNv->handle, &feature) != NV_OK) return NV_FALSE; brand = DRF_VAL(_PBI, _EXECUTE_ROUTINE, _GET_FEATURE_EXCLUSION, feature); return (brand == NV_PBI_EXECUTE_ROUTINE_GET_FEATURE_EXCLUSION_ALLOWED); } NV_STATUS RmExcludeAdapter( nv_state_t *nv ) { NV_STATUS rmStatus; if (!RmIsExcludingAllowed(nv)) return NV_ERR_NOT_SUPPORTED; rmStatus = gpumgrExcludeGpuId(nv->gpu_id); if (rmStatus != NV_OK) { NV_PRINTF(LEVEL_ERROR, "failed to exclude GPU: 0x%x\n", rmStatus); return rmStatus; } /* * GPU exclusion only happens during initialization, and therefore there * is no need to stop activity on the GPU */ return rmStatus; }