1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 2006 Jake Moilanen <moilanen@austin.ibm.com>, IBM Corp. 4 * Copyright 2006-2007 Michael Ellerman, IBM Corp. 5 */ 6 7#include <linux/crash_dump.h> 8#include <linux/device.h> 9#include <linux/irq.h> 10#include <linux/irqdomain.h> 11#include <linux/msi.h> 12 13#include <asm/rtas.h> 14#include <asm/hw_irq.h> 15#include <asm/ppc-pci.h> 16#include <asm/machdep.h> 17#include <asm/xive.h> 18 19#include "pseries.h" 20 21static int query_token, change_token; 22 23#define RTAS_QUERY_FN 0 24#define RTAS_CHANGE_FN 1 25#define RTAS_RESET_FN 2 26#define RTAS_CHANGE_MSI_FN 3 27#define RTAS_CHANGE_MSIX_FN 4 28#define RTAS_CHANGE_32MSI_FN 5 29#define RTAS_CHANGE_32MSIX_FN 6 30 31/* RTAS Helpers */ 32 33static int rtas_change_msi(struct pci_dn *pdn, u32 func, u32 num_irqs) 34{ 35 u32 addr, seq_num, rtas_ret[3]; 36 unsigned long buid; 37 int rc; 38 39 addr = rtas_config_addr(pdn->busno, pdn->devfn, 0); 40 buid = pdn->phb->buid; 41 42 seq_num = 1; 43 do { 44 if (func == RTAS_CHANGE_MSI_FN || func == RTAS_CHANGE_MSIX_FN || 45 func == RTAS_CHANGE_32MSI_FN || func == RTAS_CHANGE_32MSIX_FN) 46 rc = rtas_call(change_token, 6, 4, rtas_ret, addr, 47 BUID_HI(buid), BUID_LO(buid), 48 func, num_irqs, seq_num); 49 else 50 rc = rtas_call(change_token, 6, 3, rtas_ret, addr, 51 BUID_HI(buid), BUID_LO(buid), 52 func, num_irqs, seq_num); 53 54 seq_num = rtas_ret[1]; 55 } while (rtas_busy_delay(rc)); 56 57 /* 58 * If the RTAS call succeeded, return the number of irqs allocated. 59 * If not, make sure we return a negative error code. 60 */ 61 if (rc == 0) 62 rc = rtas_ret[0]; 63 else if (rc > 0) 64 rc = -rc; 65 66 pr_debug("rtas_msi: ibm,change_msi(func=%d,num=%d), got %d rc = %d\n", 67 func, num_irqs, rtas_ret[0], rc); 68 69 return rc; 70} 71 72static void rtas_disable_msi(struct pci_dev *pdev) 73{ 74 struct pci_dn *pdn; 75 76 pdn = pci_get_pdn(pdev); 77 if (!pdn) 78 return; 79 80 /* 81 * disabling MSI with the explicit interface also disables MSI-X 82 */ 83 if (rtas_change_msi(pdn, RTAS_CHANGE_MSI_FN, 0) != 0) { 84 /* 85 * may have failed because explicit interface is not 86 * present 87 */ 88 if (rtas_change_msi(pdn, RTAS_CHANGE_FN, 0) != 0) { 89 pr_debug("rtas_msi: Setting MSIs to 0 failed!\n"); 90 } 91 } 92} 93 94static int rtas_query_irq_number(struct pci_dn *pdn, int offset) 95{ 96 u32 addr, rtas_ret[2]; 97 unsigned long buid; 98 int rc; 99 100 addr = rtas_config_addr(pdn->busno, pdn->devfn, 0); 101 buid = pdn->phb->buid; 102 103 do { 104 rc = rtas_call(query_token, 4, 3, rtas_ret, addr, 105 BUID_HI(buid), BUID_LO(buid), offset); 106 } while (rtas_busy_delay(rc)); 107 108 if (rc) { 109 pr_debug("rtas_msi: error (%d) querying source number\n", rc); 110 return rc; 111 } 112 113 return rtas_ret[0]; 114} 115 116static int check_req(struct pci_dev *pdev, int nvec, char *prop_name) 117{ 118 struct device_node *dn; 119 const __be32 *p; 120 u32 req_msi; 121 122 dn = pci_device_to_OF_node(pdev); 123 124 p = of_get_property(dn, prop_name, NULL); 125 if (!p) { 126 pr_debug("rtas_msi: No %s on %pOF\n", prop_name, dn); 127 return -ENOENT; 128 } 129 130 req_msi = be32_to_cpup(p); 131 if (req_msi < nvec) { 132 pr_debug("rtas_msi: %s requests < %d MSIs\n", prop_name, nvec); 133 134 if (req_msi == 0) /* Be paranoid */ 135 return -ENOSPC; 136 137 return req_msi; 138 } 139 140 return 0; 141} 142 143static int check_req_msi(struct pci_dev *pdev, int nvec) 144{ 145 return check_req(pdev, nvec, "ibm,req#msi"); 146} 147 148static int check_req_msix(struct pci_dev *pdev, int nvec) 149{ 150 return check_req(pdev, nvec, "ibm,req#msi-x"); 151} 152 153/* Quota calculation */ 154 155static struct device_node *__find_pe_total_msi(struct device_node *node, int *total) 156{ 157 struct device_node *dn; 158 const __be32 *p; 159 160 dn = of_node_get(node); 161 while (dn) { 162 p = of_get_property(dn, "ibm,pe-total-#msi", NULL); 163 if (p) { 164 pr_debug("rtas_msi: found prop on dn %pOF\n", 165 dn); 166 *total = be32_to_cpup(p); 167 return dn; 168 } 169 170 dn = of_get_next_parent(dn); 171 } 172 173 return NULL; 174} 175 176static struct device_node *find_pe_total_msi(struct pci_dev *dev, int *total) 177{ 178 return __find_pe_total_msi(pci_device_to_OF_node(dev), total); 179} 180 181static struct device_node *find_pe_dn(struct pci_dev *dev, int *total) 182{ 183 struct device_node *dn; 184 struct eeh_dev *edev; 185 186 /* Found our PE and assume 8 at that point. */ 187 188 dn = pci_device_to_OF_node(dev); 189 if (!dn) 190 return NULL; 191 192 /* Get the top level device in the PE */ 193 edev = pdn_to_eeh_dev(PCI_DN(dn)); 194 if (edev->pe) 195 edev = list_first_entry(&edev->pe->edevs, struct eeh_dev, 196 entry); 197 dn = pci_device_to_OF_node(edev->pdev); 198 if (!dn) 199 return NULL; 200 201 /* We actually want the parent */ 202 dn = of_get_parent(dn); 203 if (!dn) 204 return NULL; 205 206 /* Hardcode of 8 for old firmwares */ 207 *total = 8; 208 pr_debug("rtas_msi: using PE dn %pOF\n", dn); 209 210 return dn; 211} 212 213struct msi_counts { 214 struct device_node *requestor; 215 int num_devices; 216 int request; 217 int quota; 218 int spare; 219 int over_quota; 220}; 221 222static void *count_non_bridge_devices(struct device_node *dn, void *data) 223{ 224 struct msi_counts *counts = data; 225 const __be32 *p; 226 u32 class; 227 228 pr_debug("rtas_msi: counting %pOF\n", dn); 229 230 p = of_get_property(dn, "class-code", NULL); 231 class = p ? be32_to_cpup(p) : 0; 232 233 if ((class >> 8) != PCI_CLASS_BRIDGE_PCI) 234 counts->num_devices++; 235 236 return NULL; 237} 238 239static void *count_spare_msis(struct device_node *dn, void *data) 240{ 241 struct msi_counts *counts = data; 242 const __be32 *p; 243 int req; 244 245 if (dn == counts->requestor) 246 req = counts->request; 247 else { 248 /* We don't know if a driver will try to use MSI or MSI-X, 249 * so we just have to punt and use the larger of the two. */ 250 req = 0; 251 p = of_get_property(dn, "ibm,req#msi", NULL); 252 if (p) 253 req = be32_to_cpup(p); 254 255 p = of_get_property(dn, "ibm,req#msi-x", NULL); 256 if (p) 257 req = max(req, (int)be32_to_cpup(p)); 258 } 259 260 if (req < counts->quota) 261 counts->spare += counts->quota - req; 262 else if (req > counts->quota) 263 counts->over_quota++; 264 265 return NULL; 266} 267 268static int msi_quota_for_device(struct pci_dev *dev, int request) 269{ 270 struct device_node *pe_dn; 271 struct msi_counts counts; 272 int total; 273 274 pr_debug("rtas_msi: calc quota for %s, request %d\n", pci_name(dev), 275 request); 276 277 pe_dn = find_pe_total_msi(dev, &total); 278 if (!pe_dn) 279 pe_dn = find_pe_dn(dev, &total); 280 281 if (!pe_dn) { 282 pr_err("rtas_msi: couldn't find PE for %s\n", pci_name(dev)); 283 goto out; 284 } 285 286 pr_debug("rtas_msi: found PE %pOF\n", pe_dn); 287 288 memset(&counts, 0, sizeof(struct msi_counts)); 289 290 /* Work out how many devices we have below this PE */ 291 pci_traverse_device_nodes(pe_dn, count_non_bridge_devices, &counts); 292 293 if (counts.num_devices == 0) { 294 pr_err("rtas_msi: found 0 devices under PE for %s\n", 295 pci_name(dev)); 296 goto out; 297 } 298 299 counts.quota = total / counts.num_devices; 300 if (request <= counts.quota) 301 goto out; 302 303 /* else, we have some more calculating to do */ 304 counts.requestor = pci_device_to_OF_node(dev); 305 counts.request = request; 306 pci_traverse_device_nodes(pe_dn, count_spare_msis, &counts); 307 308 /* If the quota isn't an integer multiple of the total, we can 309 * use the remainder as spare MSIs for anyone that wants them. */ 310 counts.spare += total % counts.num_devices; 311 312 /* Divide any spare by the number of over-quota requestors */ 313 if (counts.over_quota) 314 counts.quota += counts.spare / counts.over_quota; 315 316 /* And finally clamp the request to the possibly adjusted quota */ 317 request = min(counts.quota, request); 318 319 pr_debug("rtas_msi: request clamped to quota %d\n", request); 320out: 321 of_node_put(pe_dn); 322 323 return request; 324} 325 326static void rtas_hack_32bit_msi_gen2(struct pci_dev *pdev) 327{ 328 u32 addr_hi, addr_lo; 329 330 /* 331 * We should only get in here for IODA1 configs. This is based on the 332 * fact that we using RTAS for MSIs, we don't have the 32 bit MSI RTAS 333 * support, and we are in a PCIe Gen2 slot. 334 */ 335 dev_info(&pdev->dev, 336 "rtas_msi: No 32 bit MSI firmware support, forcing 32 bit MSI\n"); 337 pci_read_config_dword(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_HI, &addr_hi); 338 addr_lo = 0xffff0000 | ((addr_hi >> (48 - 32)) << 4); 339 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_LO, addr_lo); 340 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_HI, 0); 341} 342 343static int rtas_prepare_msi_irqs(struct pci_dev *pdev, int nvec_in, int type, 344 msi_alloc_info_t *arg) 345{ 346 struct pci_dn *pdn; 347 int quota, rc; 348 int nvec = nvec_in; 349 int use_32bit_msi_hack = 0; 350 351 if (type == PCI_CAP_ID_MSIX) 352 rc = check_req_msix(pdev, nvec); 353 else 354 rc = check_req_msi(pdev, nvec); 355 356 if (rc) 357 return rc; 358 359 quota = msi_quota_for_device(pdev, nvec); 360 361 if (quota && quota < nvec) 362 return quota; 363 364 /* 365 * Firmware currently refuse any non power of two allocation 366 * so we round up if the quota will allow it. 367 */ 368 if (type == PCI_CAP_ID_MSIX) { 369 int m = roundup_pow_of_two(nvec); 370 quota = msi_quota_for_device(pdev, m); 371 372 if (quota >= m) 373 nvec = m; 374 } 375 376 pdn = pci_get_pdn(pdev); 377 378 /* 379 * Try the new more explicit firmware interface, if that fails fall 380 * back to the old interface. The old interface is known to never 381 * return MSI-Xs. 382 */ 383again: 384 if (type == PCI_CAP_ID_MSI) { 385 if (pdev->no_64bit_msi) { 386 rc = rtas_change_msi(pdn, RTAS_CHANGE_32MSI_FN, nvec); 387 if (rc < 0) { 388 /* 389 * We only want to run the 32 bit MSI hack below if 390 * the max bus speed is Gen2 speed 391 */ 392 if (pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) 393 return rc; 394 395 use_32bit_msi_hack = 1; 396 } 397 } else 398 rc = -1; 399 400 if (rc < 0) 401 rc = rtas_change_msi(pdn, RTAS_CHANGE_MSI_FN, nvec); 402 403 if (rc < 0) { 404 pr_debug("rtas_msi: trying the old firmware call.\n"); 405 rc = rtas_change_msi(pdn, RTAS_CHANGE_FN, nvec); 406 } 407 408 if (use_32bit_msi_hack && rc > 0) 409 rtas_hack_32bit_msi_gen2(pdev); 410 } else { 411 if (pdev->no_64bit_msi) 412 rc = rtas_change_msi(pdn, RTAS_CHANGE_32MSIX_FN, nvec); 413 else 414 rc = rtas_change_msi(pdn, RTAS_CHANGE_MSIX_FN, nvec); 415 } 416 417 if (rc != nvec) { 418 if (nvec != nvec_in) { 419 nvec = nvec_in; 420 goto again; 421 } 422 pr_debug("rtas_msi: rtas_change_msi() failed\n"); 423 return rc; 424 } 425 426 return 0; 427} 428 429static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev, 430 int nvec, msi_alloc_info_t *arg) 431{ 432 struct pci_dev *pdev = to_pci_dev(dev); 433 int type = pdev->msix_enabled ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; 434 435 return rtas_prepare_msi_irqs(pdev, nvec, type, arg); 436} 437 438/* 439 * ->msi_free() is called before irq_domain_free_irqs_top() when the 440 * handler data is still available. Use that to clear the XIVE 441 * controller data. 442 */ 443static void pseries_msi_ops_msi_free(struct irq_domain *domain, 444 struct msi_domain_info *info, 445 unsigned int irq) 446{ 447 if (xive_enabled()) 448 xive_irq_free_data(irq); 449} 450 451/* 452 * RTAS can not disable one MSI at a time. It's all or nothing. Do it 453 * at the end after all IRQs have been freed. 454 */ 455static void pseries_msi_post_free(struct irq_domain *domain, struct device *dev) 456{ 457 if (WARN_ON_ONCE(!dev_is_pci(dev))) 458 return; 459 460 rtas_disable_msi(to_pci_dev(dev)); 461} 462 463static struct msi_domain_ops pseries_pci_msi_domain_ops = { 464 .msi_prepare = pseries_msi_ops_prepare, 465 .msi_free = pseries_msi_ops_msi_free, 466 .msi_post_free = pseries_msi_post_free, 467}; 468 469static void pseries_msi_shutdown(struct irq_data *d) 470{ 471 d = d->parent_data; 472 if (d->chip->irq_shutdown) 473 d->chip->irq_shutdown(d); 474} 475 476static void pseries_msi_mask(struct irq_data *d) 477{ 478 pci_msi_mask_irq(d); 479 irq_chip_mask_parent(d); 480} 481 482static void pseries_msi_unmask(struct irq_data *d) 483{ 484 pci_msi_unmask_irq(d); 485 irq_chip_unmask_parent(d); 486} 487 488static void pseries_msi_write_msg(struct irq_data *data, struct msi_msg *msg) 489{ 490 struct msi_desc *entry = irq_data_get_msi_desc(data); 491 492 /* 493 * Do not update the MSIx vector table. It's not strictly necessary 494 * because the table is initialized by the underlying hypervisor, PowerVM 495 * or QEMU/KVM. However, if the MSIx vector entry is cleared, any further 496 * activation will fail. This can happen in some drivers (eg. IPR) which 497 * deactivate an IRQ used for testing MSI support. 498 */ 499 entry->msg = *msg; 500} 501 502static struct irq_chip pseries_pci_msi_irq_chip = { 503 .name = "pSeries-PCI-MSI", 504 .irq_shutdown = pseries_msi_shutdown, 505 .irq_mask = pseries_msi_mask, 506 .irq_unmask = pseries_msi_unmask, 507 .irq_eoi = irq_chip_eoi_parent, 508 .irq_write_msi_msg = pseries_msi_write_msg, 509}; 510 511 512/* 513 * Set MSI_FLAG_MSIX_CONTIGUOUS as there is no way to express to 514 * firmware to request a discontiguous or non-zero based range of 515 * MSI-X entries. Core code will reject such setup attempts. 516 */ 517static struct msi_domain_info pseries_msi_domain_info = { 518 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | 519 MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX | 520 MSI_FLAG_MSIX_CONTIGUOUS), 521 .ops = &pseries_pci_msi_domain_ops, 522 .chip = &pseries_pci_msi_irq_chip, 523}; 524 525static void pseries_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) 526{ 527 __pci_read_msi_msg(irq_data_get_msi_desc(data), msg); 528} 529 530static struct irq_chip pseries_msi_irq_chip = { 531 .name = "pSeries-MSI", 532 .irq_shutdown = pseries_msi_shutdown, 533 .irq_mask = irq_chip_mask_parent, 534 .irq_unmask = irq_chip_unmask_parent, 535 .irq_eoi = irq_chip_eoi_parent, 536 .irq_set_affinity = irq_chip_set_affinity_parent, 537 .irq_compose_msi_msg = pseries_msi_compose_msg, 538}; 539 540static int pseries_irq_parent_domain_alloc(struct irq_domain *domain, unsigned int virq, 541 irq_hw_number_t hwirq) 542{ 543 struct irq_fwspec parent_fwspec; 544 int ret; 545 546 parent_fwspec.fwnode = domain->parent->fwnode; 547 parent_fwspec.param_count = 2; 548 parent_fwspec.param[0] = hwirq; 549 parent_fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 550 551 ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec); 552 if (ret) 553 return ret; 554 555 return 0; 556} 557 558static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 559 unsigned int nr_irqs, void *arg) 560{ 561 struct pci_controller *phb = domain->host_data; 562 msi_alloc_info_t *info = arg; 563 struct msi_desc *desc = info->desc; 564 struct pci_dev *pdev = msi_desc_to_pci_dev(desc); 565 int hwirq; 566 int i, ret; 567 568 hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); 569 if (hwirq < 0) { 570 dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); 571 return hwirq; 572 } 573 574 dev_dbg(&pdev->dev, "%s bridge %pOF %d/%x #%d\n", __func__, 575 phb->dn, virq, hwirq, nr_irqs); 576 577 for (i = 0; i < nr_irqs; i++) { 578 ret = pseries_irq_parent_domain_alloc(domain, virq + i, hwirq + i); 579 if (ret) 580 goto out; 581 582 irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, 583 &pseries_msi_irq_chip, domain->host_data); 584 } 585 586 return 0; 587 588out: 589 /* TODO: handle RTAS cleanup in ->msi_finish() ? */ 590 irq_domain_free_irqs_parent(domain, virq, i - 1); 591 return ret; 592} 593 594static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virq, 595 unsigned int nr_irqs) 596{ 597 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 598 struct pci_controller *phb = irq_data_get_irq_chip_data(d); 599 600 pr_debug("%s bridge %pOF %d #%d\n", __func__, phb->dn, virq, nr_irqs); 601 602 /* XIVE domain data is cleared through ->msi_free() */ 603} 604 605static const struct irq_domain_ops pseries_irq_domain_ops = { 606 .alloc = pseries_irq_domain_alloc, 607 .free = pseries_irq_domain_free, 608}; 609 610static int __pseries_msi_allocate_domains(struct pci_controller *phb, 611 unsigned int count) 612{ 613 struct irq_domain *parent = irq_get_default_host(); 614 615 phb->fwnode = irq_domain_alloc_named_id_fwnode("pSeries-MSI", 616 phb->global_number); 617 if (!phb->fwnode) 618 return -ENOMEM; 619 620 phb->dev_domain = irq_domain_create_hierarchy(parent, 0, count, 621 phb->fwnode, 622 &pseries_irq_domain_ops, phb); 623 if (!phb->dev_domain) { 624 pr_err("PCI: failed to create IRQ domain bridge %pOF (domain %d)\n", 625 phb->dn, phb->global_number); 626 irq_domain_free_fwnode(phb->fwnode); 627 return -ENOMEM; 628 } 629 630 phb->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(phb->dn), 631 &pseries_msi_domain_info, 632 phb->dev_domain); 633 if (!phb->msi_domain) { 634 pr_err("PCI: failed to create MSI IRQ domain bridge %pOF (domain %d)\n", 635 phb->dn, phb->global_number); 636 irq_domain_free_fwnode(phb->fwnode); 637 irq_domain_remove(phb->dev_domain); 638 return -ENOMEM; 639 } 640 641 return 0; 642} 643 644int pseries_msi_allocate_domains(struct pci_controller *phb) 645{ 646 int count; 647 648 if (!__find_pe_total_msi(phb->dn, &count)) { 649 pr_err("PCI: failed to find MSIs for bridge %pOF (domain %d)\n", 650 phb->dn, phb->global_number); 651 return -ENOSPC; 652 } 653 654 return __pseries_msi_allocate_domains(phb, count); 655} 656 657void pseries_msi_free_domains(struct pci_controller *phb) 658{ 659 if (phb->msi_domain) 660 irq_domain_remove(phb->msi_domain); 661 if (phb->dev_domain) 662 irq_domain_remove(phb->dev_domain); 663 if (phb->fwnode) 664 irq_domain_free_fwnode(phb->fwnode); 665} 666 667static void rtas_msi_pci_irq_fixup(struct pci_dev *pdev) 668{ 669 /* No LSI -> leave MSIs (if any) configured */ 670 if (!pdev->irq) { 671 dev_dbg(&pdev->dev, "rtas_msi: no LSI, nothing to do.\n"); 672 return; 673 } 674 675 /* No MSI -> MSIs can't have been assigned by fw, leave LSI */ 676 if (check_req_msi(pdev, 1) && check_req_msix(pdev, 1)) { 677 dev_dbg(&pdev->dev, "rtas_msi: no req#msi/x, nothing to do.\n"); 678 return; 679 } 680 681 dev_dbg(&pdev->dev, "rtas_msi: disabling existing MSI.\n"); 682 rtas_disable_msi(pdev); 683} 684 685static int rtas_msi_init(void) 686{ 687 query_token = rtas_function_token(RTAS_FN_IBM_QUERY_INTERRUPT_SOURCE_NUMBER); 688 change_token = rtas_function_token(RTAS_FN_IBM_CHANGE_MSI); 689 690 if ((query_token == RTAS_UNKNOWN_SERVICE) || 691 (change_token == RTAS_UNKNOWN_SERVICE)) { 692 pr_debug("rtas_msi: no RTAS tokens, no MSI support.\n"); 693 return -1; 694 } 695 696 pr_debug("rtas_msi: Registering RTAS MSI callbacks.\n"); 697 698 WARN_ON(ppc_md.pci_irq_fixup); 699 ppc_md.pci_irq_fixup = rtas_msi_pci_irq_fixup; 700 701 return 0; 702} 703machine_arch_initcall(pseries, rtas_msi_init); 704