1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI MSI/MSI-X ��� Exported APIs for device drivers 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 * Copyright (C) 2016 Christoph Hellwig. 8 * Copyright (C) 2022 Linutronix GmbH 9 */ 10 11#include <linux/export.h> 12#include <linux/irq.h> 13 14#include "msi.h" 15 16/** 17 * pci_enable_msi() - Enable MSI interrupt mode on device 18 * @dev: the PCI device to operate on 19 * 20 * Legacy device driver API to enable MSI interrupts mode on device and 21 * allocate a single interrupt vector. On success, the allocated vector 22 * Linux IRQ will be saved at @dev->irq. The driver must invoke 23 * pci_disable_msi() on cleanup. 24 * 25 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 26 * pair should, in general, be used instead. 27 * 28 * Return: 0 on success, errno otherwise 29 */ 30int pci_enable_msi(struct pci_dev *dev) 31{ 32 int rc = __pci_enable_msi_range(dev, 1, 1, NULL); 33 if (rc < 0) 34 return rc; 35 return 0; 36} 37EXPORT_SYMBOL(pci_enable_msi); 38 39/** 40 * pci_disable_msi() - Disable MSI interrupt mode on device 41 * @dev: the PCI device to operate on 42 * 43 * Legacy device driver API to disable MSI interrupt mode on device, 44 * free earlier allocated interrupt vectors, and restore INTx emulation. 45 * The PCI device Linux IRQ (@dev->irq) is restored to its default 46 * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi(). 47 * 48 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 49 * pair should, in general, be used instead. 50 */ 51void pci_disable_msi(struct pci_dev *dev) 52{ 53 if (!pci_msi_enabled() || !dev || !dev->msi_enabled) 54 return; 55 56 msi_lock_descs(&dev->dev); 57 pci_msi_shutdown(dev); 58 pci_free_msi_irqs(dev); 59 msi_unlock_descs(&dev->dev); 60} 61EXPORT_SYMBOL(pci_disable_msi); 62 63/** 64 * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device 65 * @dev: the PCI device to operate on 66 * 67 * Return: number of MSI-X interrupt vectors available on this device 68 * (i.e., the device's MSI-X capability structure "table size"), -EINVAL 69 * if the device is not MSI-X capable, other errnos otherwise. 70 */ 71int pci_msix_vec_count(struct pci_dev *dev) 72{ 73 u16 control; 74 75 if (!dev->msix_cap) 76 return -EINVAL; 77 78 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); 79 return msix_table_size(control); 80} 81EXPORT_SYMBOL(pci_msix_vec_count); 82 83/** 84 * pci_enable_msix_range() - Enable MSI-X interrupt mode on device 85 * @dev: the PCI device to operate on 86 * @entries: input/output parameter, array of MSI-X configuration entries 87 * @minvec: minimum required number of MSI-X vectors 88 * @maxvec: maximum desired number of MSI-X vectors 89 * 90 * Legacy device driver API to enable MSI-X interrupt mode on device and 91 * configure its MSI-X capability structure as appropriate. The passed 92 * @entries array must have each of its members "entry" field set to a 93 * desired (valid) MSI-X vector number, where the range of valid MSI-X 94 * vector numbers can be queried through pci_msix_vec_count(). If 95 * successful, the driver must invoke pci_disable_msix() on cleanup. 96 * 97 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 98 * pair should, in general, be used instead. 99 * 100 * Return: number of MSI-X vectors allocated (which might be smaller 101 * than @maxvecs), where Linux IRQ numbers for such allocated vectors 102 * are saved back in the @entries array elements' "vector" field. Return 103 * -ENOSPC if less than @minvecs interrupt vectors are available. 104 * Return -EINVAL if one of the passed @entries members "entry" field 105 * was invalid or a duplicate, or if plain MSI interrupts mode was 106 * earlier enabled on device. Return other errnos otherwise. 107 */ 108int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, 109 int minvec, int maxvec) 110{ 111 return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0); 112} 113EXPORT_SYMBOL(pci_enable_msix_range); 114 115/** 116 * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling 117 * MSI-X is supported 118 * 119 * @dev: PCI device to operate on 120 * 121 * Return: True if supported, false otherwise 122 */ 123bool pci_msix_can_alloc_dyn(struct pci_dev *dev) 124{ 125 if (!dev->msix_cap) 126 return false; 127 128 return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); 129} 130EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn); 131 132/** 133 * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X 134 * at a given MSI-X vector index or any free vector index 135 * 136 * @dev: PCI device to operate on 137 * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates 138 * the next free index in the MSI-X table 139 * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise 140 * 141 * Return: A struct msi_map 142 * 143 * On success msi_map::index contains the allocated index (>= 0) and 144 * msi_map::virq contains the allocated Linux interrupt number (> 0). 145 * 146 * On fail msi_map::index contains the error code and msi_map::virq 147 * is set to 0. 148 */ 149struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index, 150 const struct irq_affinity_desc *affdesc) 151{ 152 struct msi_map map = { .index = -ENOTSUPP }; 153 154 if (!dev->msix_enabled) 155 return map; 156 157 if (!pci_msix_can_alloc_dyn(dev)) 158 return map; 159 160 return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL); 161} 162EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at); 163 164/** 165 * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain 166 * 167 * @dev: The PCI device to operate on 168 * @map: A struct msi_map describing the interrupt to free 169 * 170 * Undo an interrupt vector allocation. Does not disable MSI-X. 171 */ 172void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map) 173{ 174 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) 175 return; 176 if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev))) 177 return; 178 msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index); 179} 180EXPORT_SYMBOL_GPL(pci_msix_free_irq); 181 182/** 183 * pci_disable_msix() - Disable MSI-X interrupt mode on device 184 * @dev: the PCI device to operate on 185 * 186 * Legacy device driver API to disable MSI-X interrupt mode on device, 187 * free earlier-allocated interrupt vectors, and restore INTx. 188 * The PCI device Linux IRQ (@dev->irq) is restored to its default pin 189 * assertion IRQ. This is the cleanup pair of pci_enable_msix_range(). 190 * 191 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 192 * pair should, in general, be used instead. 193 */ 194void pci_disable_msix(struct pci_dev *dev) 195{ 196 if (!pci_msi_enabled() || !dev || !dev->msix_enabled) 197 return; 198 199 msi_lock_descs(&dev->dev); 200 pci_msix_shutdown(dev); 201 pci_free_msi_irqs(dev); 202 msi_unlock_descs(&dev->dev); 203} 204EXPORT_SYMBOL(pci_disable_msix); 205 206/** 207 * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors 208 * @dev: the PCI device to operate on 209 * @min_vecs: minimum required number of vectors (must be >= 1) 210 * @max_vecs: maximum desired number of vectors 211 * @flags: One or more of: 212 * 213 * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations 214 * * %PCI_IRQ_MSI Allow trying MSI vector allocations 215 * 216 * * %PCI_IRQ_LEGACY Allow trying legacy INTx interrupts, if 217 * and only if @min_vecs == 1 218 * 219 * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading 220 * the vectors around available CPUs 221 * 222 * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq 223 * vector allocation has a higher precedence over plain MSI, which has a 224 * higher precedence over legacy INTx emulation. 225 * 226 * Upon a successful allocation, the caller should use pci_irq_vector() 227 * to get the Linux IRQ number to be passed to request_threaded_irq(). 228 * The driver must call pci_free_irq_vectors() on cleanup. 229 * 230 * Return: number of allocated vectors (which might be smaller than 231 * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are 232 * available, other errnos otherwise. 233 */ 234int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, 235 unsigned int max_vecs, unsigned int flags) 236{ 237 return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs, 238 flags, NULL); 239} 240EXPORT_SYMBOL(pci_alloc_irq_vectors); 241 242/** 243 * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt 244 * vectors with affinity requirements 245 * @dev: the PCI device to operate on 246 * @min_vecs: minimum required number of vectors (must be >= 1) 247 * @max_vecs: maximum desired number of vectors 248 * @flags: allocation flags, as in pci_alloc_irq_vectors() 249 * @affd: affinity requirements (can be %NULL). 250 * 251 * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter. 252 * Check that function docs, and &struct irq_affinity, for more details. 253 */ 254int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs, 255 unsigned int max_vecs, unsigned int flags, 256 struct irq_affinity *affd) 257{ 258 struct irq_affinity msi_default_affd = {0}; 259 int nvecs = -ENOSPC; 260 261 if (flags & PCI_IRQ_AFFINITY) { 262 if (!affd) 263 affd = &msi_default_affd; 264 } else { 265 if (WARN_ON(affd)) 266 affd = NULL; 267 } 268 269 if (flags & PCI_IRQ_MSIX) { 270 nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs, 271 affd, flags); 272 if (nvecs > 0) 273 return nvecs; 274 } 275 276 if (flags & PCI_IRQ_MSI) { 277 nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd); 278 if (nvecs > 0) 279 return nvecs; 280 } 281 282 /* use legacy IRQ if allowed */ 283 if (flags & PCI_IRQ_LEGACY) { 284 if (min_vecs == 1 && dev->irq) { 285 /* 286 * Invoke the affinity spreading logic to ensure that 287 * the device driver can adjust queue configuration 288 * for the single interrupt case. 289 */ 290 if (affd) 291 irq_create_affinity_masks(1, affd); 292 pci_intx(dev, 1); 293 return 1; 294 } 295 } 296 297 return nvecs; 298} 299EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity); 300 301/** 302 * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector 303 * @dev: the PCI device to operate on 304 * @nr: device-relative interrupt vector index (0-based); has different 305 * meanings, depending on interrupt mode: 306 * 307 * * MSI-X the index in the MSI-X vector table 308 * * MSI the index of the enabled MSI vectors 309 * * INTx must be 0 310 * 311 * Return: the Linux IRQ number, or -EINVAL if @nr is out of range 312 */ 313int pci_irq_vector(struct pci_dev *dev, unsigned int nr) 314{ 315 unsigned int irq; 316 317 if (!dev->msi_enabled && !dev->msix_enabled) 318 return !nr ? dev->irq : -EINVAL; 319 320 irq = msi_get_virq(&dev->dev, nr); 321 return irq ? irq : -EINVAL; 322} 323EXPORT_SYMBOL(pci_irq_vector); 324 325/** 326 * pci_irq_get_affinity() - Get a device interrupt vector affinity 327 * @dev: the PCI device to operate on 328 * @nr: device-relative interrupt vector index (0-based); has different 329 * meanings, depending on interrupt mode: 330 * 331 * * MSI-X the index in the MSI-X vector table 332 * * MSI the index of the enabled MSI vectors 333 * * INTx must be 0 334 * 335 * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if 336 * the MSI(-X) vector was allocated without explicit affinity 337 * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or 338 * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a 339 * generic set of CPU IDs representing all possible CPUs available 340 * during system boot if the device is in legacy INTx mode. 341 */ 342const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) 343{ 344 int idx, irq = pci_irq_vector(dev, nr); 345 struct msi_desc *desc; 346 347 if (WARN_ON_ONCE(irq <= 0)) 348 return NULL; 349 350 desc = irq_get_msi_desc(irq); 351 /* Non-MSI does not have the information handy */ 352 if (!desc) 353 return cpu_possible_mask; 354 355 /* MSI[X] interrupts can be allocated without affinity descriptor */ 356 if (!desc->affinity) 357 return NULL; 358 359 /* 360 * MSI has a mask array in the descriptor. 361 * MSI-X has a single mask. 362 */ 363 idx = dev->msi_enabled ? nr : 0; 364 return &desc->affinity[idx].mask; 365} 366EXPORT_SYMBOL(pci_irq_get_affinity); 367 368/** 369 * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain 370 * @dev: The PCI device to operate on 371 * @icookie: Pointer to an IMS implementation specific cookie for this 372 * IMS instance (PASID, queue ID, pointer...). 373 * The cookie content is copied into the MSI descriptor for the 374 * interrupt chip callbacks or domain specific setup functions. 375 * @affdesc: Optional pointer to an interrupt affinity descriptor 376 * 377 * There is no index for IMS allocations as IMS is an implementation 378 * specific storage and does not have any direct associations between 379 * index, which might be a pure software construct, and device 380 * functionality. This association is established by the driver either via 381 * the index - if there is a hardware table - or in case of purely software 382 * managed IMS implementation the association happens via the 383 * irq_write_msi_msg() callback of the implementation specific interrupt 384 * chip, which utilizes the provided @icookie to store the MSI message in 385 * the appropriate place. 386 * 387 * Return: A struct msi_map 388 * 389 * On success msi_map::index contains the allocated index (>= 0) and 390 * msi_map::virq the allocated Linux interrupt number (> 0). 391 * 392 * On fail msi_map::index contains the error code and msi_map::virq 393 * is set to 0. 394 */ 395struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie, 396 const struct irq_affinity_desc *affdesc) 397{ 398 return msi_domain_alloc_irq_at(&dev->dev, MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX, 399 affdesc, icookie); 400} 401EXPORT_SYMBOL_GPL(pci_ims_alloc_irq); 402 403/** 404 * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain 405 * which was allocated via pci_ims_alloc_irq() 406 * @dev: The PCI device to operate on 407 * @map: A struct msi_map describing the interrupt to free as 408 * returned from pci_ims_alloc_irq() 409 */ 410void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map) 411{ 412 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) 413 return; 414 msi_domain_free_irqs_range(&dev->dev, MSI_SECONDARY_DOMAIN, map.index, map.index); 415} 416EXPORT_SYMBOL_GPL(pci_ims_free_irq); 417 418/** 419 * pci_free_irq_vectors() - Free previously allocated IRQs for a device 420 * @dev: the PCI device to operate on 421 * 422 * Undo the interrupt vector allocations and possible device MSI/MSI-X 423 * enablement earlier done through pci_alloc_irq_vectors_affinity() or 424 * pci_alloc_irq_vectors(). 425 */ 426void pci_free_irq_vectors(struct pci_dev *dev) 427{ 428 pci_disable_msix(dev); 429 pci_disable_msi(dev); 430} 431EXPORT_SYMBOL(pci_free_irq_vectors); 432 433/** 434 * pci_restore_msi_state() - Restore cached MSI(-X) state on device 435 * @dev: the PCI device to operate on 436 * 437 * Write the Linux-cached MSI(-X) state back on device. This is 438 * typically useful upon system resume, or after an error-recovery PCI 439 * adapter reset. 440 */ 441void pci_restore_msi_state(struct pci_dev *dev) 442{ 443 __pci_restore_msi_state(dev); 444 __pci_restore_msix_state(dev); 445} 446EXPORT_SYMBOL_GPL(pci_restore_msi_state); 447 448/** 449 * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide? 450 * 451 * Return: true if MSI has not been globally disabled through ACPI FADT, 452 * PCI bridge quirks, or the "pci=nomsi" kernel command-line option. 453 */ 454int pci_msi_enabled(void) 455{ 456 return pci_msi_enable; 457} 458EXPORT_SYMBOL(pci_msi_enabled); 459