1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI Message Signaled Interrupt (MSI) - irqdomain support 4 */ 5#include <linux/acpi_iort.h> 6#include <linux/irqdomain.h> 7#include <linux/of_irq.h> 8 9#include "msi.h" 10 11int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 12{ 13 struct irq_domain *domain; 14 15 domain = dev_get_msi_domain(&dev->dev); 16 if (domain && irq_domain_is_hierarchy(domain)) 17 return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); 18 19 return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); 20} 21 22void pci_msi_teardown_msi_irqs(struct pci_dev *dev) 23{ 24 struct irq_domain *domain; 25 26 domain = dev_get_msi_domain(&dev->dev); 27 if (domain && irq_domain_is_hierarchy(domain)) { 28 msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); 29 } else { 30 pci_msi_legacy_teardown_msi_irqs(dev); 31 msi_free_msi_descs(&dev->dev); 32 } 33} 34 35/** 36 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space 37 * @irq_data: Pointer to interrupt data of the MSI interrupt 38 * @msg: Pointer to the message 39 */ 40static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) 41{ 42 struct msi_desc *desc = irq_data_get_msi_desc(irq_data); 43 44 /* 45 * For MSI-X desc->irq is always equal to irq_data->irq. For 46 * MSI only the first interrupt of MULTI MSI passes the test. 47 */ 48 if (desc->irq == irq_data->irq) 49 __pci_write_msi_msg(desc, msg); 50} 51 52/** 53 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source 54 * @desc: Pointer to the MSI descriptor 55 * 56 * The ID number is only used within the irqdomain. 57 */ 58static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) 59{ 60 struct pci_dev *dev = msi_desc_to_pci_dev(desc); 61 62 return (irq_hw_number_t)desc->msi_index | 63 pci_dev_id(dev) << 11 | 64 ((irq_hw_number_t)(pci_domain_nr(dev->bus) & 0xFFFFFFFF)) << 27; 65} 66 67static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, 68 struct msi_desc *desc) 69{ 70 arg->desc = desc; 71 arg->hwirq = pci_msi_domain_calc_hwirq(desc); 72} 73 74static struct msi_domain_ops pci_msi_domain_ops_default = { 75 .set_desc = pci_msi_domain_set_desc, 76}; 77 78static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) 79{ 80 struct msi_domain_ops *ops = info->ops; 81 82 if (ops == NULL) { 83 info->ops = &pci_msi_domain_ops_default; 84 } else { 85 if (ops->set_desc == NULL) 86 ops->set_desc = pci_msi_domain_set_desc; 87 } 88} 89 90static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) 91{ 92 struct irq_chip *chip = info->chip; 93 94 BUG_ON(!chip); 95 if (!chip->irq_write_msi_msg) 96 chip->irq_write_msi_msg = pci_msi_domain_write_msg; 97 if (!chip->irq_mask) 98 chip->irq_mask = pci_msi_mask_irq; 99 if (!chip->irq_unmask) 100 chip->irq_unmask = pci_msi_unmask_irq; 101} 102 103/** 104 * pci_msi_create_irq_domain - Create a MSI interrupt domain 105 * @fwnode: Optional fwnode of the interrupt controller 106 * @info: MSI domain info 107 * @parent: Parent irq domain 108 * 109 * Updates the domain and chip ops and creates a MSI interrupt domain. 110 * 111 * Returns: 112 * A domain pointer or NULL in case of failure. 113 */ 114struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, 115 struct msi_domain_info *info, 116 struct irq_domain *parent) 117{ 118 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) 119 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; 120 121 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) 122 pci_msi_domain_update_dom_ops(info); 123 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 124 pci_msi_domain_update_chip_ops(info); 125 126 /* Let the core code free MSI descriptors when freeing interrupts */ 127 info->flags |= MSI_FLAG_FREE_MSI_DESCS; 128 129 info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; 130 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) 131 info->flags |= MSI_FLAG_MUST_REACTIVATE; 132 133 /* PCI-MSI is oneshot-safe */ 134 info->chip->flags |= IRQCHIP_ONESHOT_SAFE; 135 /* Let the core update the bus token */ 136 info->bus_token = DOMAIN_BUS_PCI_MSI; 137 138 return msi_create_irq_domain(fwnode, info, parent); 139} 140EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); 141 142/* 143 * Per device MSI[-X] domain functionality 144 */ 145static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) 146{ 147 arg->desc = desc; 148 arg->hwirq = desc->msi_index; 149} 150 151static void pci_irq_mask_msi(struct irq_data *data) 152{ 153 struct msi_desc *desc = irq_data_get_msi_desc(data); 154 155 pci_msi_mask(desc, BIT(data->irq - desc->irq)); 156} 157 158static void pci_irq_unmask_msi(struct irq_data *data) 159{ 160 struct msi_desc *desc = irq_data_get_msi_desc(data); 161 162 pci_msi_unmask(desc, BIT(data->irq - desc->irq)); 163} 164 165#ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE 166# define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE 167#else 168# define MSI_REACTIVATE 0 169#endif 170 171#define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \ 172 MSI_FLAG_ACTIVATE_EARLY | \ 173 MSI_FLAG_DEV_SYSFS | \ 174 MSI_REACTIVATE) 175 176static const struct msi_domain_template pci_msi_template = { 177 .chip = { 178 .name = "PCI-MSI", 179 .irq_mask = pci_irq_mask_msi, 180 .irq_unmask = pci_irq_unmask_msi, 181 .irq_write_msi_msg = pci_msi_domain_write_msg, 182 .flags = IRQCHIP_ONESHOT_SAFE, 183 }, 184 185 .ops = { 186 .set_desc = pci_device_domain_set_desc, 187 }, 188 189 .info = { 190 .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI, 191 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI, 192 }, 193}; 194 195static void pci_irq_mask_msix(struct irq_data *data) 196{ 197 pci_msix_mask(irq_data_get_msi_desc(data)); 198} 199 200static void pci_irq_unmask_msix(struct irq_data *data) 201{ 202 pci_msix_unmask(irq_data_get_msi_desc(data)); 203} 204 205static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg, 206 struct msi_desc *desc) 207{ 208 /* Don't fiddle with preallocated MSI descriptors */ 209 if (!desc->pci.mask_base) 210 msix_prepare_msi_desc(to_pci_dev(desc->dev), desc); 211} 212 213static const struct msi_domain_template pci_msix_template = { 214 .chip = { 215 .name = "PCI-MSIX", 216 .irq_mask = pci_irq_mask_msix, 217 .irq_unmask = pci_irq_unmask_msix, 218 .irq_write_msi_msg = pci_msi_domain_write_msg, 219 .flags = IRQCHIP_ONESHOT_SAFE, 220 }, 221 222 .ops = { 223 .prepare_desc = pci_msix_prepare_desc, 224 .set_desc = pci_device_domain_set_desc, 225 }, 226 227 .info = { 228 .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX | 229 MSI_FLAG_PCI_MSIX_ALLOC_DYN, 230 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX, 231 }, 232}; 233 234static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token) 235{ 236 return msi_match_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, bus_token); 237} 238 239static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl, 240 unsigned int hwsize) 241{ 242 struct irq_domain *domain = dev_get_msi_domain(&pdev->dev); 243 244 if (!domain || !irq_domain_is_msi_parent(domain)) 245 return true; 246 247 return msi_create_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, tmpl, 248 hwsize, NULL, NULL); 249} 250 251/** 252 * pci_setup_msi_device_domain - Setup a device MSI interrupt domain 253 * @pdev: The PCI device to create the domain on 254 * 255 * Return: 256 * True when: 257 * - The device does not have a MSI parent irq domain associated, 258 * which keeps the legacy architecture specific and the global 259 * PCI/MSI domain models working 260 * - The MSI domain exists already 261 * - The MSI domain was successfully allocated 262 * False when: 263 * - MSI-X is enabled 264 * - The domain creation fails. 265 * 266 * The created MSI domain is preserved until: 267 * - The device is removed 268 * - MSI is disabled and a MSI-X domain is created 269 */ 270bool pci_setup_msi_device_domain(struct pci_dev *pdev) 271{ 272 if (WARN_ON_ONCE(pdev->msix_enabled)) 273 return false; 274 275 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 276 return true; 277 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 278 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 279 280 return pci_create_device_domain(pdev, &pci_msi_template, 1); 281} 282 283/** 284 * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain 285 * @pdev: The PCI device to create the domain on 286 * @hwsize: The size of the MSI-X vector table 287 * 288 * Return: 289 * True when: 290 * - The device does not have a MSI parent irq domain associated, 291 * which keeps the legacy architecture specific and the global 292 * PCI/MSI domain models working 293 * - The MSI-X domain exists already 294 * - The MSI-X domain was successfully allocated 295 * False when: 296 * - MSI is enabled 297 * - The domain creation fails. 298 * 299 * The created MSI-X domain is preserved until: 300 * - The device is removed 301 * - MSI-X is disabled and a MSI domain is created 302 */ 303bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize) 304{ 305 if (WARN_ON_ONCE(pdev->msi_enabled)) 306 return false; 307 308 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 309 return true; 310 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 311 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 312 313 return pci_create_device_domain(pdev, &pci_msix_template, hwsize); 314} 315 316/** 317 * pci_msi_domain_supports - Check for support of a particular feature flag 318 * @pdev: The PCI device to operate on 319 * @feature_mask: The feature mask to check for (full match) 320 * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain 321 * associated to the device. If DENY_LEGACY the lack of an irq domain 322 * makes the feature unsupported 323 */ 324bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask, 325 enum support_mode mode) 326{ 327 struct msi_domain_info *info; 328 struct irq_domain *domain; 329 unsigned int supported; 330 331 domain = dev_get_msi_domain(&pdev->dev); 332 333 if (!domain || !irq_domain_is_hierarchy(domain)) 334 return mode == ALLOW_LEGACY; 335 336 if (!irq_domain_is_msi_parent(domain)) { 337 /* 338 * For "global" PCI/MSI interrupt domains the associated 339 * msi_domain_info::flags is the authoritative source of 340 * information. 341 */ 342 info = domain->host_data; 343 supported = info->flags; 344 } else { 345 /* 346 * For MSI parent domains the supported feature set 347 * is available in the parent ops. This makes checks 348 * possible before actually instantiating the 349 * per device domain because the parent is never 350 * expanding the PCI/MSI functionality. 351 */ 352 supported = domain->msi_parent_ops->supported_flags; 353 } 354 355 return (supported & feature_mask) == feature_mask; 356} 357 358/** 359 * pci_create_ims_domain - Create a secondary IMS domain for a PCI device 360 * @pdev: The PCI device to operate on 361 * @template: The MSI info template which describes the domain 362 * @hwsize: The size of the hardware entry table or 0 if the domain 363 * is purely software managed 364 * @data: Optional pointer to domain specific data to be stored 365 * in msi_domain_info::data 366 * 367 * Return: True on success, false otherwise 368 * 369 * An IMS domain is expected to have the following constraints: 370 * - The index space is managed by the core code 371 * 372 * - There is no requirement for consecutive index ranges 373 * 374 * - The interrupt chip must provide the following callbacks: 375 * - irq_mask() 376 * - irq_unmask() 377 * - irq_write_msi_msg() 378 * 379 * - The interrupt chip must provide the following optional callbacks 380 * when the irq_mask(), irq_unmask() and irq_write_msi_msg() callbacks 381 * cannot operate directly on hardware, e.g. in the case that the 382 * interrupt message store is in queue memory: 383 * - irq_bus_lock() 384 * - irq_bus_unlock() 385 * 386 * These callbacks are invoked from preemptible task context and are 387 * allowed to sleep. In this case the mandatory callbacks above just 388 * store the information. The irq_bus_unlock() callback is supposed 389 * to make the change effective before returning. 390 * 391 * - Interrupt affinity setting is handled by the underlying parent 392 * interrupt domain and communicated to the IMS domain via 393 * irq_write_msi_msg(). 394 * 395 * The domain is automatically destroyed when the PCI device is removed. 396 */ 397bool pci_create_ims_domain(struct pci_dev *pdev, const struct msi_domain_template *template, 398 unsigned int hwsize, void *data) 399{ 400 struct irq_domain *domain = dev_get_msi_domain(&pdev->dev); 401 402 if (!domain || !irq_domain_is_msi_parent(domain)) 403 return false; 404 405 if (template->info.bus_token != DOMAIN_BUS_PCI_DEVICE_IMS || 406 !(template->info.flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS) || 407 !(template->info.flags & MSI_FLAG_FREE_MSI_DESCS) || 408 !template->chip.irq_mask || !template->chip.irq_unmask || 409 !template->chip.irq_write_msi_msg || template->chip.irq_set_affinity) 410 return false; 411 412 return msi_create_device_irq_domain(&pdev->dev, MSI_SECONDARY_DOMAIN, template, 413 hwsize, data, NULL); 414} 415EXPORT_SYMBOL_GPL(pci_create_ims_domain); 416 417/* 418 * Users of the generic MSI infrastructure expect a device to have a single ID, 419 * so with DMA aliases we have to pick the least-worst compromise. Devices with 420 * DMA phantom functions tend to still emit MSIs from the real function number, 421 * so we ignore those and only consider topological aliases where either the 422 * alias device or RID appears on a different bus number. We also make the 423 * reasonable assumption that bridges are walked in an upstream direction (so 424 * the last one seen wins), and the much braver assumption that the most likely 425 * case is that of PCI->PCIe so we should always use the alias RID. This echoes 426 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works 427 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions 428 * for taking ownership all we can really do is close our eyes and hope... 429 */ 430static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) 431{ 432 u32 *pa = data; 433 u8 bus = PCI_BUS_NUM(*pa); 434 435 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) 436 *pa = alias; 437 438 return 0; 439} 440 441/** 442 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) 443 * @domain: The interrupt domain 444 * @pdev: The PCI device. 445 * 446 * The RID for a device is formed from the alias, with a firmware 447 * supplied mapping applied 448 * 449 * Returns: The RID. 450 */ 451u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) 452{ 453 struct device_node *of_node; 454 u32 rid = pci_dev_id(pdev); 455 456 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 457 458 of_node = irq_domain_get_of_node(domain); 459 rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : 460 iort_msi_map_id(&pdev->dev, rid); 461 462 return rid; 463} 464 465/** 466 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device 467 * @pdev: The PCI device 468 * 469 * Use the firmware data to find a device-specific MSI domain 470 * (i.e. not one that is set as a default). 471 * 472 * Returns: The corresponding MSI domain or NULL if none has been found. 473 */ 474struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) 475{ 476 struct irq_domain *dom; 477 u32 rid = pci_dev_id(pdev); 478 479 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 480 dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); 481 if (!dom) 482 dom = iort_get_device_domain(&pdev->dev, rid, 483 DOMAIN_BUS_PCI_MSI); 484 return dom; 485} 486