1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2013--2024 Intel Corporation 4 */ 5 6#include <linux/bitfield.h> 7#include <linux/bits.h> 8#include <linux/dma-mapping.h> 9#include <linux/err.h> 10#include <linux/firmware.h> 11#include <linux/kernel.h> 12#include <linux/interrupt.h> 13#include <linux/io.h> 14#include <linux/list.h> 15#include <linux/module.h> 16#include <linux/pci-ats.h> 17#include <linux/pm_runtime.h> 18#include <linux/property.h> 19#include <linux/scatterlist.h> 20#include <linux/slab.h> 21#include <linux/types.h> 22 23#include <media/ipu-bridge.h> 24#include <media/ipu6-pci-table.h> 25 26#include "ipu6.h" 27#include "ipu6-bus.h" 28#include "ipu6-buttress.h" 29#include "ipu6-cpd.h" 30#include "ipu6-isys.h" 31#include "ipu6-mmu.h" 32#include "ipu6-platform-buttress-regs.h" 33#include "ipu6-platform-isys-csi2-reg.h" 34#include "ipu6-platform-regs.h" 35 36#define IPU6_PCI_BAR 0 37 38struct ipu6_cell_program { 39 u32 magic_number; 40 41 u32 blob_offset; 42 u32 blob_size; 43 44 u32 start[3]; 45 46 u32 icache_source; 47 u32 icache_target; 48 u32 icache_size; 49 50 u32 pmem_source; 51 u32 pmem_target; 52 u32 pmem_size; 53 54 u32 data_source; 55 u32 data_target; 56 u32 data_size; 57 58 u32 bss_target; 59 u32 bss_size; 60 61 u32 cell_id; 62 u32 regs_addr; 63 64 u32 cell_pmem_data_bus_address; 65 u32 cell_dmem_data_bus_address; 66 u32 cell_pmem_control_bus_address; 67 u32 cell_dmem_control_bus_address; 68 69 u32 next; 70 u32 dummy[2]; 71}; 72 73static struct ipu6_isys_internal_pdata isys_ipdata = { 74 .hw_variant = { 75 .offset = IPU6_UNIFIED_OFFSET, 76 .nr_mmus = 3, 77 .mmu_hw = { 78 { 79 .offset = IPU6_ISYS_IOMMU0_OFFSET, 80 .info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF, 81 .nr_l1streams = 16, 82 .l1_block_sz = { 83 3, 8, 2, 2, 2, 2, 2, 2, 1, 1, 84 1, 1, 1, 1, 1, 1 85 }, 86 .nr_l2streams = 16, 87 .l2_block_sz = { 88 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 89 2, 2, 2, 2, 2, 2 90 }, 91 .insert_read_before_invalidate = false, 92 .l1_stream_id_reg_offset = 93 IPU6_MMU_L1_STREAM_ID_REG_OFFSET, 94 .l2_stream_id_reg_offset = 95 IPU6_MMU_L2_STREAM_ID_REG_OFFSET, 96 }, 97 { 98 .offset = IPU6_ISYS_IOMMU1_OFFSET, 99 .info_bits = 0, 100 .nr_l1streams = 16, 101 .l1_block_sz = { 102 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 103 2, 2, 2, 1, 1, 4 104 }, 105 .nr_l2streams = 16, 106 .l2_block_sz = { 107 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 108 2, 2, 2, 2, 2, 2 109 }, 110 .insert_read_before_invalidate = false, 111 .l1_stream_id_reg_offset = 112 IPU6_MMU_L1_STREAM_ID_REG_OFFSET, 113 .l2_stream_id_reg_offset = 114 IPU6_MMU_L2_STREAM_ID_REG_OFFSET, 115 }, 116 { 117 .offset = IPU6_ISYS_IOMMUI_OFFSET, 118 .info_bits = 0, 119 .nr_l1streams = 0, 120 .nr_l2streams = 0, 121 .insert_read_before_invalidate = false, 122 }, 123 }, 124 .cdc_fifos = 3, 125 .cdc_fifo_threshold = {6, 8, 2}, 126 .dmem_offset = IPU6_ISYS_DMEM_OFFSET, 127 .spc_offset = IPU6_ISYS_SPC_OFFSET, 128 }, 129 .isys_dma_overshoot = IPU6_ISYS_OVERALLOC_MIN, 130}; 131 132static struct ipu6_psys_internal_pdata psys_ipdata = { 133 .hw_variant = { 134 .offset = IPU6_UNIFIED_OFFSET, 135 .nr_mmus = 4, 136 .mmu_hw = { 137 { 138 .offset = IPU6_PSYS_IOMMU0_OFFSET, 139 .info_bits = 140 IPU6_INFO_REQUEST_DESTINATION_IOSF, 141 .nr_l1streams = 16, 142 .l1_block_sz = { 143 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 144 2, 2, 2, 2, 2, 2 145 }, 146 .nr_l2streams = 16, 147 .l2_block_sz = { 148 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 149 2, 2, 2, 2, 2, 2 150 }, 151 .insert_read_before_invalidate = false, 152 .l1_stream_id_reg_offset = 153 IPU6_MMU_L1_STREAM_ID_REG_OFFSET, 154 .l2_stream_id_reg_offset = 155 IPU6_MMU_L2_STREAM_ID_REG_OFFSET, 156 }, 157 { 158 .offset = IPU6_PSYS_IOMMU1_OFFSET, 159 .info_bits = 0, 160 .nr_l1streams = 32, 161 .l1_block_sz = { 162 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 163 2, 2, 2, 2, 2, 10, 164 5, 4, 14, 6, 4, 14, 6, 4, 8, 165 4, 2, 1, 1, 1, 1, 14 166 }, 167 .nr_l2streams = 32, 168 .l2_block_sz = { 169 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 170 2, 2, 2, 2, 2, 2, 171 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 172 2, 2, 2, 2, 2, 2 173 }, 174 .insert_read_before_invalidate = false, 175 .l1_stream_id_reg_offset = 176 IPU6_MMU_L1_STREAM_ID_REG_OFFSET, 177 .l2_stream_id_reg_offset = 178 IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET, 179 }, 180 { 181 .offset = IPU6_PSYS_IOMMU1R_OFFSET, 182 .info_bits = 0, 183 .nr_l1streams = 16, 184 .l1_block_sz = { 185 1, 4, 4, 4, 4, 16, 8, 4, 32, 186 16, 16, 2, 2, 2, 1, 12 187 }, 188 .nr_l2streams = 16, 189 .l2_block_sz = { 190 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 191 2, 2, 2, 2, 2, 2 192 }, 193 .insert_read_before_invalidate = false, 194 .l1_stream_id_reg_offset = 195 IPU6_MMU_L1_STREAM_ID_REG_OFFSET, 196 .l2_stream_id_reg_offset = 197 IPU6_MMU_L2_STREAM_ID_REG_OFFSET, 198 }, 199 { 200 .offset = IPU6_PSYS_IOMMUI_OFFSET, 201 .info_bits = 0, 202 .nr_l1streams = 0, 203 .nr_l2streams = 0, 204 .insert_read_before_invalidate = false, 205 }, 206 }, 207 .dmem_offset = IPU6_PSYS_DMEM_OFFSET, 208 }, 209}; 210 211static const struct ipu6_buttress_ctrl isys_buttress_ctrl = { 212 .ratio = IPU6_IS_FREQ_CTL_DEFAULT_RATIO, 213 .qos_floor = IPU6_IS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO, 214 .freq_ctl = IPU6_BUTTRESS_REG_IS_FREQ_CTL, 215 .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_IS_PWR_SHIFT, 216 .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_IS_PWR_MASK, 217 .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE, 218 .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE, 219}; 220 221static const struct ipu6_buttress_ctrl psys_buttress_ctrl = { 222 .ratio = IPU6_PS_FREQ_CTL_DEFAULT_RATIO, 223 .qos_floor = IPU6_PS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO, 224 .freq_ctl = IPU6_BUTTRESS_REG_PS_FREQ_CTL, 225 .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_PS_PWR_SHIFT, 226 .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_PS_PWR_MASK, 227 .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE, 228 .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE, 229}; 230 231static void 232ipu6_pkg_dir_configure_spc(struct ipu6_device *isp, 233 const struct ipu6_hw_variants *hw_variant, 234 int pkg_dir_idx, void __iomem *base, 235 u64 *pkg_dir, dma_addr_t pkg_dir_vied_address) 236{ 237 struct ipu6_cell_program *prog; 238 void __iomem *spc_base; 239 u32 server_fw_addr; 240 dma_addr_t dma_addr; 241 u32 pg_offset; 242 243 server_fw_addr = lower_32_bits(*(pkg_dir + (pkg_dir_idx + 1) * 2)); 244 if (pkg_dir_idx == IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX) 245 dma_addr = sg_dma_address(isp->isys->fw_sgt.sgl); 246 else 247 dma_addr = sg_dma_address(isp->psys->fw_sgt.sgl); 248 249 pg_offset = server_fw_addr - dma_addr; 250 prog = (struct ipu6_cell_program *)((u64)isp->cpd_fw->data + pg_offset); 251 spc_base = base + prog->regs_addr; 252 if (spc_base != (base + hw_variant->spc_offset)) 253 dev_warn(&isp->pdev->dev, 254 "SPC reg addr %p not matching value from CPD %p\n", 255 base + hw_variant->spc_offset, spc_base); 256 writel(server_fw_addr + prog->blob_offset + 257 prog->icache_source, spc_base + IPU6_PSYS_REG_SPC_ICACHE_BASE); 258 writel(IPU6_INFO_REQUEST_DESTINATION_IOSF, 259 spc_base + IPU6_REG_PSYS_INFO_SEG_0_CONFIG_ICACHE_MASTER); 260 writel(prog->start[1], spc_base + IPU6_PSYS_REG_SPC_START_PC); 261 writel(pkg_dir_vied_address, base + hw_variant->dmem_offset); 262} 263 264void ipu6_configure_spc(struct ipu6_device *isp, 265 const struct ipu6_hw_variants *hw_variant, 266 int pkg_dir_idx, void __iomem *base, u64 *pkg_dir, 267 dma_addr_t pkg_dir_dma_addr) 268{ 269 void __iomem *dmem_base = base + hw_variant->dmem_offset; 270 void __iomem *spc_regs_base = base + hw_variant->spc_offset; 271 u32 val; 272 273 val = readl(spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL); 274 val |= IPU6_PSYS_SPC_STATUS_CTRL_ICACHE_INVALIDATE; 275 writel(val, spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL); 276 277 if (isp->secure_mode) 278 writel(IPU6_PKG_DIR_IMR_OFFSET, dmem_base); 279 else 280 ipu6_pkg_dir_configure_spc(isp, hw_variant, pkg_dir_idx, base, 281 pkg_dir, pkg_dir_dma_addr); 282} 283EXPORT_SYMBOL_NS_GPL(ipu6_configure_spc, INTEL_IPU6); 284 285#define IPU6_ISYS_CSI2_NPORTS 4 286#define IPU6SE_ISYS_CSI2_NPORTS 4 287#define IPU6_TGL_ISYS_CSI2_NPORTS 8 288#define IPU6EP_MTL_ISYS_CSI2_NPORTS 6 289 290static void ipu6_internal_pdata_init(struct ipu6_device *isp) 291{ 292 u8 hw_ver = isp->hw_ver; 293 294 isys_ipdata.num_parallel_streams = IPU6_ISYS_NUM_STREAMS; 295 isys_ipdata.sram_gran_shift = IPU6_SRAM_GRANULARITY_SHIFT; 296 isys_ipdata.sram_gran_size = IPU6_SRAM_GRANULARITY_SIZE; 297 isys_ipdata.max_sram_size = IPU6_MAX_SRAM_SIZE; 298 isys_ipdata.sensor_type_start = IPU6_FW_ISYS_SENSOR_TYPE_START; 299 isys_ipdata.sensor_type_end = IPU6_FW_ISYS_SENSOR_TYPE_END; 300 isys_ipdata.max_streams = IPU6_ISYS_NUM_STREAMS; 301 isys_ipdata.max_send_queues = IPU6_N_MAX_SEND_QUEUES; 302 isys_ipdata.max_sram_blocks = IPU6_NOF_SRAM_BLOCKS_MAX; 303 isys_ipdata.max_devq_size = IPU6_DEV_SEND_QUEUE_SIZE; 304 isys_ipdata.csi2.nports = IPU6_ISYS_CSI2_NPORTS; 305 isys_ipdata.csi2.irq_mask = IPU6_CSI_RX_ERROR_IRQ_MASK; 306 isys_ipdata.csi2.ctrl0_irq_edge = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE; 307 isys_ipdata.csi2.ctrl0_irq_clear = 308 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR; 309 isys_ipdata.csi2.ctrl0_irq_mask = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK; 310 isys_ipdata.csi2.ctrl0_irq_enable = 311 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE; 312 isys_ipdata.csi2.ctrl0_irq_status = 313 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS; 314 isys_ipdata.csi2.ctrl0_irq_lnp = 315 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE; 316 isys_ipdata.enhanced_iwake = is_ipu6ep_mtl(hw_ver) || is_ipu6ep(hw_ver); 317 psys_ipdata.hw_variant.spc_offset = IPU6_PSYS_SPC_OFFSET; 318 isys_ipdata.csi2.fw_access_port_ofs = CSI_REG_HUB_FW_ACCESS_PORT_OFS; 319 320 if (is_ipu6ep(hw_ver)) { 321 isys_ipdata.ltr = IPU6EP_LTR_VALUE; 322 isys_ipdata.memopen_threshold = IPU6EP_MIN_MEMOPEN_TH; 323 } 324 325 if (is_ipu6_tgl(hw_ver)) 326 isys_ipdata.csi2.nports = IPU6_TGL_ISYS_CSI2_NPORTS; 327 328 if (is_ipu6ep_mtl(hw_ver)) { 329 isys_ipdata.csi2.nports = IPU6EP_MTL_ISYS_CSI2_NPORTS; 330 331 isys_ipdata.csi2.ctrl0_irq_edge = 332 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE; 333 isys_ipdata.csi2.ctrl0_irq_clear = 334 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR; 335 isys_ipdata.csi2.ctrl0_irq_mask = 336 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK; 337 isys_ipdata.csi2.ctrl0_irq_enable = 338 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE; 339 isys_ipdata.csi2.ctrl0_irq_lnp = 340 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE; 341 isys_ipdata.csi2.ctrl0_irq_status = 342 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS; 343 isys_ipdata.csi2.fw_access_port_ofs = 344 CSI_REG_HUB_FW_ACCESS_PORT_V6OFS; 345 isys_ipdata.ltr = IPU6EP_MTL_LTR_VALUE; 346 isys_ipdata.memopen_threshold = IPU6EP_MTL_MIN_MEMOPEN_TH; 347 } 348 349 if (is_ipu6se(hw_ver)) { 350 isys_ipdata.csi2.nports = IPU6SE_ISYS_CSI2_NPORTS; 351 isys_ipdata.csi2.irq_mask = IPU6SE_CSI_RX_ERROR_IRQ_MASK; 352 isys_ipdata.num_parallel_streams = IPU6SE_ISYS_NUM_STREAMS; 353 isys_ipdata.sram_gran_shift = IPU6SE_SRAM_GRANULARITY_SHIFT; 354 isys_ipdata.sram_gran_size = IPU6SE_SRAM_GRANULARITY_SIZE; 355 isys_ipdata.max_sram_size = IPU6SE_MAX_SRAM_SIZE; 356 isys_ipdata.sensor_type_start = 357 IPU6SE_FW_ISYS_SENSOR_TYPE_START; 358 isys_ipdata.sensor_type_end = IPU6SE_FW_ISYS_SENSOR_TYPE_END; 359 isys_ipdata.max_streams = IPU6SE_ISYS_NUM_STREAMS; 360 isys_ipdata.max_send_queues = IPU6SE_N_MAX_SEND_QUEUES; 361 isys_ipdata.max_sram_blocks = IPU6SE_NOF_SRAM_BLOCKS_MAX; 362 isys_ipdata.max_devq_size = IPU6SE_DEV_SEND_QUEUE_SIZE; 363 psys_ipdata.hw_variant.spc_offset = IPU6SE_PSYS_SPC_OFFSET; 364 } 365} 366 367static struct ipu6_bus_device * 368ipu6_isys_init(struct pci_dev *pdev, struct device *parent, 369 struct ipu6_buttress_ctrl *ctrl, void __iomem *base, 370 const struct ipu6_isys_internal_pdata *ipdata) 371{ 372 struct device *dev = &pdev->dev; 373 struct ipu6_bus_device *isys_adev; 374 struct ipu6_isys_pdata *pdata; 375 int ret; 376 377 ret = ipu_bridge_init(dev, ipu_bridge_parse_ssdb); 378 if (ret) { 379 dev_err_probe(dev, ret, "IPU6 bridge init failed\n"); 380 return ERR_PTR(ret); 381 } 382 383 pdata = kzalloc(sizeof(*pdata), GFP_KERNEL); 384 if (!pdata) 385 return ERR_PTR(-ENOMEM); 386 387 pdata->base = base; 388 pdata->ipdata = ipdata; 389 390 isys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl, 391 IPU6_ISYS_NAME); 392 if (IS_ERR(isys_adev)) { 393 dev_err_probe(dev, PTR_ERR(isys_adev), 394 "ipu6_bus_initialize_device isys failed\n"); 395 kfree(pdata); 396 return ERR_CAST(isys_adev); 397 } 398 399 isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID, 400 &ipdata->hw_variant); 401 if (IS_ERR(isys_adev->mmu)) { 402 dev_err_probe(dev, PTR_ERR(isys_adev->mmu), 403 "ipu6_mmu_init(isys_adev->mmu) failed\n"); 404 put_device(&isys_adev->auxdev.dev); 405 kfree(pdata); 406 return ERR_CAST(isys_adev->mmu); 407 } 408 409 isys_adev->mmu->dev = &isys_adev->auxdev.dev; 410 411 ret = ipu6_bus_add_device(isys_adev); 412 if (ret) { 413 kfree(pdata); 414 return ERR_PTR(ret); 415 } 416 417 return isys_adev; 418} 419 420static struct ipu6_bus_device * 421ipu6_psys_init(struct pci_dev *pdev, struct device *parent, 422 struct ipu6_buttress_ctrl *ctrl, void __iomem *base, 423 const struct ipu6_psys_internal_pdata *ipdata) 424{ 425 struct ipu6_bus_device *psys_adev; 426 struct ipu6_psys_pdata *pdata; 427 int ret; 428 429 pdata = kzalloc(sizeof(*pdata), GFP_KERNEL); 430 if (!pdata) 431 return ERR_PTR(-ENOMEM); 432 433 pdata->base = base; 434 pdata->ipdata = ipdata; 435 436 psys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl, 437 IPU6_PSYS_NAME); 438 if (IS_ERR(psys_adev)) { 439 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev), 440 "ipu6_bus_initialize_device psys failed\n"); 441 kfree(pdata); 442 return ERR_CAST(psys_adev); 443 } 444 445 psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID, 446 &ipdata->hw_variant); 447 if (IS_ERR(psys_adev->mmu)) { 448 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu), 449 "ipu6_mmu_init(psys_adev->mmu) failed\n"); 450 put_device(&psys_adev->auxdev.dev); 451 kfree(pdata); 452 return ERR_CAST(psys_adev->mmu); 453 } 454 455 psys_adev->mmu->dev = &psys_adev->auxdev.dev; 456 457 ret = ipu6_bus_add_device(psys_adev); 458 if (ret) { 459 kfree(pdata); 460 return ERR_PTR(ret); 461 } 462 463 return psys_adev; 464} 465 466static int ipu6_pci_config_setup(struct pci_dev *dev, u8 hw_ver) 467{ 468 int ret; 469 470 /* disable IPU6 PCI ATS on mtl ES2 */ 471 if (is_ipu6ep_mtl(hw_ver) && boot_cpu_data.x86_stepping == 0x2 && 472 pci_ats_supported(dev)) 473 pci_disable_ats(dev); 474 475 /* No PCI msi capability for IPU6EP */ 476 if (is_ipu6ep(hw_ver) || is_ipu6ep_mtl(hw_ver)) { 477 /* likely do nothing as msi not enabled by default */ 478 pci_disable_msi(dev); 479 return 0; 480 } 481 482 ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_MSI); 483 if (ret < 0) 484 return dev_err_probe(&dev->dev, ret, "Request msi failed"); 485 486 return 0; 487} 488 489static void ipu6_configure_vc_mechanism(struct ipu6_device *isp) 490{ 491 u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL); 492 493 if (IPU6_BTRS_ARB_STALL_MODE_VC0 == IPU6_BTRS_ARB_MODE_TYPE_STALL) 494 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0; 495 else 496 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0; 497 498 if (IPU6_BTRS_ARB_STALL_MODE_VC1 == IPU6_BTRS_ARB_MODE_TYPE_STALL) 499 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1; 500 else 501 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1; 502 503 writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL); 504} 505 506static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 507{ 508 struct ipu6_buttress_ctrl *isys_ctrl = NULL, *psys_ctrl = NULL; 509 struct device *dev = &pdev->dev; 510 void __iomem *isys_base = NULL; 511 void __iomem *psys_base = NULL; 512 struct ipu6_device *isp; 513 phys_addr_t phys; 514 u32 val, version, sku_id; 515 int ret; 516 517 isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); 518 if (!isp) 519 return -ENOMEM; 520 521 isp->pdev = pdev; 522 INIT_LIST_HEAD(&isp->devices); 523 524 ret = pcim_enable_device(pdev); 525 if (ret) 526 return dev_err_probe(dev, ret, "Enable PCI device failed\n"); 527 528 phys = pci_resource_start(pdev, IPU6_PCI_BAR); 529 dev_dbg(dev, "IPU6 PCI bar[%u] = %pa\n", IPU6_PCI_BAR, &phys); 530 531 ret = pcim_iomap_regions(pdev, 1 << IPU6_PCI_BAR, pci_name(pdev)); 532 if (ret) 533 return dev_err_probe(dev, ret, "Failed to I/O mem remapping\n"); 534 535 isp->base = pcim_iomap_table(pdev)[IPU6_PCI_BAR]; 536 pci_set_drvdata(pdev, isp); 537 pci_set_master(pdev); 538 539 isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt); 540 switch (id->device) { 541 case PCI_DEVICE_ID_INTEL_IPU6: 542 isp->hw_ver = IPU6_VER_6; 543 isp->cpd_fw_name = IPU6_FIRMWARE_NAME; 544 break; 545 case PCI_DEVICE_ID_INTEL_IPU6SE: 546 isp->hw_ver = IPU6_VER_6SE; 547 isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME; 548 isp->cpd_metadata_cmpnt_size = 549 sizeof(struct ipu6se_cpd_metadata_cmpnt); 550 break; 551 case PCI_DEVICE_ID_INTEL_IPU6EP_ADLP: 552 case PCI_DEVICE_ID_INTEL_IPU6EP_RPLP: 553 isp->hw_ver = IPU6_VER_6EP; 554 isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME; 555 break; 556 case PCI_DEVICE_ID_INTEL_IPU6EP_ADLN: 557 isp->hw_ver = IPU6_VER_6EP; 558 isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME; 559 break; 560 case PCI_DEVICE_ID_INTEL_IPU6EP_MTL: 561 isp->hw_ver = IPU6_VER_6EP_MTL; 562 isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME; 563 break; 564 default: 565 return dev_err_probe(dev, -ENODEV, 566 "Unsupported IPU6 device %x\n", 567 id->device); 568 } 569 570 ipu6_internal_pdata_init(isp); 571 572 isys_base = isp->base + isys_ipdata.hw_variant.offset; 573 psys_base = isp->base + psys_ipdata.hw_variant.offset; 574 575 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39)); 576 if (ret) 577 return dev_err_probe(dev, ret, "Failed to set DMA mask\n"); 578 579 ret = dma_set_max_seg_size(dev, UINT_MAX); 580 if (ret) 581 return dev_err_probe(dev, ret, "Failed to set max_seg_size\n"); 582 583 ret = ipu6_pci_config_setup(pdev, isp->hw_ver); 584 if (ret) 585 return ret; 586 587 ret = ipu6_buttress_init(isp); 588 if (ret) 589 return ret; 590 591 ret = request_firmware(&isp->cpd_fw, isp->cpd_fw_name, dev); 592 if (ret) { 593 dev_err_probe(&isp->pdev->dev, ret, 594 "Requesting signed firmware %s failed\n", 595 isp->cpd_fw_name); 596 goto buttress_exit; 597 } 598 599 ret = ipu6_cpd_validate_cpd_file(isp, isp->cpd_fw->data, 600 isp->cpd_fw->size); 601 if (ret) { 602 dev_err_probe(&isp->pdev->dev, ret, 603 "Failed to validate cpd\n"); 604 goto out_ipu6_bus_del_devices; 605 } 606 607 isys_ctrl = devm_kmemdup(dev, &isys_buttress_ctrl, 608 sizeof(isys_buttress_ctrl), GFP_KERNEL); 609 if (!isys_ctrl) { 610 ret = -ENOMEM; 611 goto out_ipu6_bus_del_devices; 612 } 613 614 isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base, 615 &isys_ipdata); 616 if (IS_ERR(isp->isys)) { 617 ret = PTR_ERR(isp->isys); 618 goto out_ipu6_bus_del_devices; 619 } 620 621 psys_ctrl = devm_kmemdup(dev, &psys_buttress_ctrl, 622 sizeof(psys_buttress_ctrl), GFP_KERNEL); 623 if (!psys_ctrl) { 624 ret = -ENOMEM; 625 goto out_ipu6_bus_del_devices; 626 } 627 628 isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl, 629 psys_base, &psys_ipdata); 630 if (IS_ERR(isp->psys)) { 631 ret = PTR_ERR(isp->psys); 632 goto out_ipu6_bus_del_devices; 633 } 634 635 ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev); 636 if (ret < 0) 637 goto out_ipu6_bus_del_devices; 638 639 ret = ipu6_mmu_hw_init(isp->psys->mmu); 640 if (ret) { 641 dev_err_probe(&isp->pdev->dev, ret, 642 "Failed to set MMU hardware\n"); 643 goto out_ipu6_bus_del_devices; 644 } 645 646 ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw, 647 &isp->psys->fw_sgt); 648 if (ret) { 649 dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n"); 650 goto out_ipu6_bus_del_devices; 651 } 652 653 ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data); 654 if (ret) { 655 dev_err_probe(&isp->pdev->dev, ret, 656 "failed to create pkg dir\n"); 657 goto out_ipu6_bus_del_devices; 658 } 659 660 ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr, 661 ipu6_buttress_isr_threaded, 662 IRQF_SHARED, IPU6_NAME, isp); 663 if (ret) { 664 dev_err_probe(dev, ret, "Requesting irq failed\n"); 665 goto out_ipu6_bus_del_devices; 666 } 667 668 ret = ipu6_buttress_authenticate(isp); 669 if (ret) { 670 dev_err_probe(&isp->pdev->dev, ret, 671 "FW authentication failed\n"); 672 goto out_free_irq; 673 } 674 675 ipu6_mmu_hw_cleanup(isp->psys->mmu); 676 pm_runtime_put(&isp->psys->auxdev.dev); 677 678 /* Configure the arbitration mechanisms for VC requests */ 679 ipu6_configure_vc_mechanism(isp); 680 681 val = readl(isp->base + BUTTRESS_REG_SKU); 682 sku_id = FIELD_GET(GENMASK(6, 4), val); 683 version = FIELD_GET(GENMASK(3, 0), val); 684 dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id, 685 pdev->device, isp->hw_ver); 686 687 pm_runtime_put_noidle(dev); 688 pm_runtime_allow(dev); 689 690 isp->bus_ready_to_probe = true; 691 692 return 0; 693 694out_free_irq: 695 devm_free_irq(dev, pdev->irq, isp); 696out_ipu6_bus_del_devices: 697 if (isp->psys) { 698 ipu6_cpd_free_pkg_dir(isp->psys); 699 ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt); 700 } 701 if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu)) 702 ipu6_mmu_cleanup(isp->psys->mmu); 703 if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu)) 704 ipu6_mmu_cleanup(isp->isys->mmu); 705 ipu6_bus_del_devices(pdev); 706 release_firmware(isp->cpd_fw); 707buttress_exit: 708 ipu6_buttress_exit(isp); 709 710 return ret; 711} 712 713static void ipu6_pci_remove(struct pci_dev *pdev) 714{ 715 struct ipu6_device *isp = pci_get_drvdata(pdev); 716 struct ipu6_mmu *isys_mmu = isp->isys->mmu; 717 struct ipu6_mmu *psys_mmu = isp->psys->mmu; 718 719 devm_free_irq(&pdev->dev, pdev->irq, isp); 720 ipu6_cpd_free_pkg_dir(isp->psys); 721 722 ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt); 723 ipu6_buttress_exit(isp); 724 725 ipu6_bus_del_devices(pdev); 726 727 pm_runtime_forbid(&pdev->dev); 728 pm_runtime_get_noresume(&pdev->dev); 729 730 release_firmware(isp->cpd_fw); 731 732 ipu6_mmu_cleanup(psys_mmu); 733 ipu6_mmu_cleanup(isys_mmu); 734} 735 736static void ipu6_pci_reset_prepare(struct pci_dev *pdev) 737{ 738 struct ipu6_device *isp = pci_get_drvdata(pdev); 739 740 pm_runtime_forbid(&isp->pdev->dev); 741} 742 743static void ipu6_pci_reset_done(struct pci_dev *pdev) 744{ 745 struct ipu6_device *isp = pci_get_drvdata(pdev); 746 747 ipu6_buttress_restore(isp); 748 if (isp->secure_mode) 749 ipu6_buttress_reset_authentication(isp); 750 751 isp->need_ipc_reset = true; 752 pm_runtime_allow(&isp->pdev->dev); 753} 754 755/* 756 * PCI base driver code requires driver to provide these to enable 757 * PCI device level PM state transitions (D0<->D3) 758 */ 759static int ipu6_suspend(struct device *dev) 760{ 761 return 0; 762} 763 764static int ipu6_resume(struct device *dev) 765{ 766 struct pci_dev *pdev = to_pci_dev(dev); 767 struct ipu6_device *isp = pci_get_drvdata(pdev); 768 struct ipu6_buttress *b = &isp->buttress; 769 int ret; 770 771 /* Configure the arbitration mechanisms for VC requests */ 772 ipu6_configure_vc_mechanism(isp); 773 774 isp->secure_mode = ipu6_buttress_get_secure_mode(isp); 775 dev_info(dev, "IPU6 in %s mode\n", 776 isp->secure_mode ? "secure" : "non-secure"); 777 778 ipu6_buttress_restore(isp); 779 780 ret = ipu6_buttress_ipc_reset(isp, &b->cse); 781 if (ret) 782 dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n"); 783 784 ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev); 785 if (ret < 0) { 786 dev_err(&isp->psys->auxdev.dev, "Failed to get runtime PM\n"); 787 return 0; 788 } 789 790 ret = ipu6_buttress_authenticate(isp); 791 if (ret) 792 dev_err(&isp->pdev->dev, "FW authentication failed(%d)\n", ret); 793 794 pm_runtime_put(&isp->psys->auxdev.dev); 795 796 return 0; 797} 798 799static int ipu6_runtime_resume(struct device *dev) 800{ 801 struct pci_dev *pdev = to_pci_dev(dev); 802 struct ipu6_device *isp = pci_get_drvdata(pdev); 803 int ret; 804 805 ipu6_configure_vc_mechanism(isp); 806 ipu6_buttress_restore(isp); 807 808 if (isp->need_ipc_reset) { 809 struct ipu6_buttress *b = &isp->buttress; 810 811 isp->need_ipc_reset = false; 812 ret = ipu6_buttress_ipc_reset(isp, &b->cse); 813 if (ret) 814 dev_err(&isp->pdev->dev, "IPC reset protocol failed\n"); 815 } 816 817 return 0; 818} 819 820static const struct dev_pm_ops ipu6_pm_ops = { 821 SYSTEM_SLEEP_PM_OPS(&ipu6_suspend, &ipu6_resume) 822 RUNTIME_PM_OPS(&ipu6_suspend, &ipu6_runtime_resume, NULL) 823}; 824 825MODULE_DEVICE_TABLE(pci, ipu6_pci_tbl); 826 827static const struct pci_error_handlers pci_err_handlers = { 828 .reset_prepare = ipu6_pci_reset_prepare, 829 .reset_done = ipu6_pci_reset_done, 830}; 831 832static struct pci_driver ipu6_pci_driver = { 833 .name = IPU6_NAME, 834 .id_table = ipu6_pci_tbl, 835 .probe = ipu6_pci_probe, 836 .remove = ipu6_pci_remove, 837 .driver = { 838 .pm = pm_ptr(&ipu6_pm_ops), 839 }, 840 .err_handler = &pci_err_handlers, 841}; 842 843module_pci_driver(ipu6_pci_driver); 844 845MODULE_IMPORT_NS(INTEL_IPU_BRIDGE); 846MODULE_AUTHOR("Sakari Ailus <sakari.ailus@linux.intel.com>"); 847MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>"); 848MODULE_AUTHOR("Bingbu Cao <bingbu.cao@intel.com>"); 849MODULE_AUTHOR("Qingwu Zhang <qingwu.zhang@intel.com>"); 850MODULE_AUTHOR("Yunliang Ding <yunliang.ding@intel.com>"); 851MODULE_AUTHOR("Hongju Wang <hongju.wang@intel.com>"); 852MODULE_LICENSE("GPL"); 853MODULE_DESCRIPTION("Intel IPU6 PCI driver"); 854