1// Copyright 2016 The Fuchsia Authors 2// Copyright (c) 2016, Google, Inc. All rights reserved 3// 4// Use of this source code is governed by a MIT-style 5// license that can be found in the LICENSE file or at 6// https://opensource.org/licenses/MIT 7// 8#include <assert.h> 9#include <debug.h> 10#include <dev/interrupt.h> 11#include <dev/pcie_bus_driver.h> 12#include <dev/pcie_bridge.h> 13#include <dev/pcie_root.h> 14#include <err.h> 15#include <kernel/spinlock.h> 16#include <list.h> 17#include <pow2.h> 18#include <string.h> 19#include <trace.h> 20#include <vm/vm.h> 21 22#include <dev/pci_config.h> 23#include <dev/pcie_device.h> 24 25#include <fbl/alloc_checker.h> 26#include <fbl/auto_lock.h> 27 28using fbl::AutoLock; 29 30#define LOCAL_TRACE 0 31 32/****************************************************************************** 33 * 34 * Helper routines common to all IRQ modes. 35 * 36 ******************************************************************************/ 37void PcieDevice::ResetCommonIrqBookkeeping() { 38 if (irq_.handler_count > 1) { 39 DEBUG_ASSERT(irq_.handlers != &irq_.singleton_handler); 40 delete[] irq_.handlers; 41 } 42 43 irq_.singleton_handler.handler = nullptr; 44 irq_.singleton_handler.ctx = nullptr; 45 irq_.singleton_handler.dev = nullptr; 46 irq_.mode = PCIE_IRQ_MODE_DISABLED; 47 irq_.handlers = nullptr; 48 irq_.handler_count = 0; 49} 50 51zx_status_t PcieDevice::AllocIrqHandlers(uint requested_irqs, bool is_masked) { 52 DEBUG_ASSERT(requested_irqs); 53 DEBUG_ASSERT(!irq_.handlers); 54 DEBUG_ASSERT(!irq_.handler_count); 55 56 if (requested_irqs == 1) { 57 irq_.handlers = &irq_.singleton_handler; 58 irq_.handler_count = 1; 59 } else { 60 fbl::AllocChecker ac; 61 irq_.handlers = new (&ac) pcie_irq_handler_state_t[requested_irqs]; 62 63 if (!ac.check()) 64 return ZX_ERR_NO_MEMORY; 65 66 irq_.handler_count = requested_irqs; 67 } 68 69 for (uint i = 0; i < irq_.handler_count; ++i) { 70 DEBUG_ASSERT(irq_.handlers[i].handler == nullptr); 71 DEBUG_ASSERT(irq_.handlers[i].dev == nullptr); 72 DEBUG_ASSERT(irq_.handlers[i].ctx == nullptr); 73 irq_.handlers[i].dev = this; 74 irq_.handlers[i].pci_irq_id = i; 75 irq_.handlers[i].masked = is_masked; 76 } 77 78 return ZX_OK; 79} 80 81/****************************************************************************** 82 * 83 * Legacy IRQ mode routines. 84 * 85 ******************************************************************************/ 86fbl::RefPtr<SharedLegacyIrqHandler> SharedLegacyIrqHandler::Create(uint irq_id) { 87 fbl::AllocChecker ac; 88 89 SharedLegacyIrqHandler* handler = new (&ac) SharedLegacyIrqHandler(irq_id); 90 if (!ac.check()) { 91 TRACEF("Failed to create shared legacry IRQ handler for system IRQ ID %u\n", irq_id); 92 return nullptr; 93 } 94 95 return fbl::AdoptRef(handler); 96} 97 98SharedLegacyIrqHandler::SharedLegacyIrqHandler(uint irq_id) 99 : irq_id_(irq_id) { 100 list_initialize(&device_handler_list_); 101 mask_interrupt(irq_id_); // This should not be needed, but just in case. 102 zx_status_t status = register_int_handler(irq_id_, HandlerThunk, this); 103 DEBUG_ASSERT(status == ZX_OK); 104} 105 106SharedLegacyIrqHandler::~SharedLegacyIrqHandler() { 107 DEBUG_ASSERT(list_is_empty(&device_handler_list_)); 108 mask_interrupt(irq_id_); 109 zx_status_t status = register_int_handler(irq_id_, nullptr, nullptr); 110 DEBUG_ASSERT(status == ZX_OK); 111} 112 113void SharedLegacyIrqHandler::Handler() { 114 /* Go over the list of device's which share this legacy IRQ and give them a 115 * chance to handle any interrupts which may be pending in their device. 116 * Keep track of whether or not any device has requested a re-schedule event 117 * at the end of this IRQ. */ 118 AutoSpinLockNoIrqSave list_lock(&device_handler_list_lock_); 119 120 if (list_is_empty(&device_handler_list_)) { 121 TRACEF("Received legacy PCI INT (system IRQ %u), but there are no devices registered to " 122 "handle this interrupt. This is Very Bad. Disabling the interrupt at the system " 123 "IRQ level to prevent meltdown.\n", 124 irq_id_); 125 mask_interrupt(irq_id_); 126 return; 127 } 128 129 PcieDevice* dev; 130 list_for_every_entry(&device_handler_list_, 131 dev, 132 PcieDevice, 133 irq_.legacy.shared_handler_node) { 134 uint16_t command, status; 135 auto cfg = dev->config(); 136 137 { 138 AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_); 139 command = cfg->Read(PciConfig::kCommand); 140 status = cfg->Read(PciConfig::kStatus); 141 } 142 143 if ((status & PCIE_CFG_STATUS_INT_STS) && !(command & PCIE_CFG_COMMAND_INT_DISABLE)) { 144 DEBUG_ASSERT(dev); 145 pcie_irq_handler_state_t* hstate = &dev->irq_.handlers[0]; 146 147 if (hstate) { 148 pcie_irq_handler_retval_t irq_ret = PCIE_IRQRET_MASK; 149 AutoSpinLockNoIrqSave device_handler_lock(&hstate->lock); 150 151 if (hstate->handler) { 152 if (!hstate->masked) 153 irq_ret = hstate->handler(*dev, 0, hstate->ctx); 154 } else { 155 TRACEF("Received legacy PCI INT (system IRQ %u) for %02x:%02x.%02x, but no irq_ " 156 "handler has been registered by the driver. Force disabling the " 157 "interrupt.\n", 158 irq_id_, dev->bus_id_, dev->dev_id_, dev->func_id_); 159 } 160 161 if (irq_ret & PCIE_IRQRET_MASK) { 162 hstate->masked = true; 163 { 164 AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_); 165 command = cfg->Read(PciConfig::kCommand); 166 cfg->Write(PciConfig::kCommand, command | PCIE_CFG_COMMAND_INT_DISABLE); 167 } 168 } 169 } else { 170 TRACEF("Received legacy PCI INT (system IRQ %u) for %02x:%02x.%02x , but no irq_ " 171 "handlers have been allocated! Force disabling the interrupt.\n", 172 irq_id_, dev->bus_id_, dev->dev_id_, dev->func_id_); 173 174 { 175 AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_); 176 command = cfg->Read(PciConfig::kCommand); 177 cfg->Write(PciConfig::kCommand, command | PCIE_CFG_COMMAND_INT_DISABLE); 178 } 179 } 180 } 181 } 182} 183 184void SharedLegacyIrqHandler::AddDevice(PcieDevice& dev) { 185 DEBUG_ASSERT(dev.irq_.legacy.shared_handler.get() == this); 186 DEBUG_ASSERT(!list_in_list(&dev.irq_.legacy.shared_handler_node)); 187 188 /* Make certain that the device's legacy IRQ has been masked at the PCI 189 * device level. Then add this dev to the handler's list. If this was the 190 * first device added to the handler list, unmask the handler IRQ at the top 191 * level. */ 192 AutoSpinLock lock(&device_handler_list_lock_); 193 194 dev.cfg_->Write(PciConfig::kCommand, dev.cfg_->Read(PciConfig::kCommand) | 195 PCIE_CFG_COMMAND_INT_DISABLE); 196 197 bool first_device = list_is_empty(&device_handler_list_); 198 list_add_tail(&device_handler_list_, &dev.irq_.legacy.shared_handler_node); 199 200 if (first_device) 201 unmask_interrupt(irq_id_); 202} 203 204void SharedLegacyIrqHandler::RemoveDevice(PcieDevice& dev) { 205 DEBUG_ASSERT(dev.irq_.legacy.shared_handler.get() == this); 206 DEBUG_ASSERT(list_in_list(&dev.irq_.legacy.shared_handler_node)); 207 208 /* Make absolutely sure we have been masked at the PCIe config level, then 209 * remove the device from the shared handler list. If this was the last 210 * device on the list, mask the top level IRQ */ 211 AutoSpinLock lock(&device_handler_list_lock_); 212 213 dev.cfg_->Write(PciConfig::kCommand, dev.cfg_->Read(PciConfig::kCommand) | 214 PCIE_CFG_COMMAND_INT_DISABLE); 215 list_delete(&dev.irq_.legacy.shared_handler_node); 216 217 if (list_is_empty(&device_handler_list_)) 218 mask_interrupt(irq_id_); 219} 220 221zx_status_t PcieDevice::MaskUnmaskLegacyIrq(bool mask) { 222 if (!irq_.handlers || !irq_.handler_count) 223 return ZX_ERR_INVALID_ARGS; 224 225 pcie_irq_handler_state_t& hstate = irq_.handlers[0]; 226 227 { 228 AutoSpinLock lock(&hstate.lock); 229 230 if (mask) ModifyCmdLocked(0, PCIE_CFG_COMMAND_INT_DISABLE); 231 else ModifyCmdLocked(PCIE_CFG_COMMAND_INT_DISABLE, 0); 232 hstate.masked = mask; 233 } 234 235 return ZX_OK; 236} 237 238zx_status_t PcieDevice::EnterLegacyIrqMode(uint requested_irqs) { 239 DEBUG_ASSERT(requested_irqs); 240 241 if (!irq_.legacy.pin || (requested_irqs > 1)) 242 return ZX_ERR_NOT_SUPPORTED; 243 244 // Make absolutely certain we are masked. 245 ModifyCmdLocked(0, PCIE_CFG_COMMAND_INT_DISABLE); 246 247 // We can never fail to allocated a single handlers (since we are going to 248 // use the pre-allocated singleton) 249 __UNUSED zx_status_t res = AllocIrqHandlers(requested_irqs, true); 250 DEBUG_ASSERT(res == ZX_OK); 251 DEBUG_ASSERT(irq_.handlers == &irq_.singleton_handler); 252 253 irq_.mode = PCIE_IRQ_MODE_LEGACY; 254 irq_.legacy.shared_handler->AddDevice(*this); 255 256 return ZX_OK; 257} 258 259void PcieDevice::LeaveLegacyIrqMode() { 260 /* Disable legacy IRQs and unregister from the shared legacy handler */ 261 MaskUnmaskLegacyIrq(true); 262 irq_.legacy.shared_handler->RemoveDevice(*this); 263 264 /* Release any handler storage and reset all of our bookkeeping */ 265 ResetCommonIrqBookkeeping(); 266} 267 268/****************************************************************************** 269 * 270 * MSI IRQ mode routines. 271 * 272 ******************************************************************************/ 273bool PcieDevice::MaskUnmaskMsiIrqLocked(uint irq_id, bool mask) { 274 DEBUG_ASSERT(irq_.mode == PCIE_IRQ_MODE_MSI); 275 DEBUG_ASSERT(irq_id < irq_.handler_count); 276 DEBUG_ASSERT(irq_.handlers); 277 278 pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id]; 279 DEBUG_ASSERT(hstate.lock.IsHeld()); 280 281 /* Internal code should not be calling this function if they want to mask 282 * the interrupt, but it is not possible to do so. */ 283 DEBUG_ASSERT(!mask || 284 bus_drv_.platform().supports_msi_masking() || 285 irq_.msi->has_pvm()); 286 287 /* If we can mask at the PCI device level, do so. */ 288 if (irq_.msi->has_pvm()) { 289 DEBUG_ASSERT(irq_id < PCIE_MAX_MSI_IRQS); 290 uint32_t val = cfg_->Read(irq_.msi->mask_bits_reg()); 291 if (mask) val |= (static_cast<uint32_t>(1u) << irq_id); 292 else val &= ~(static_cast<uint32_t>(1u) << irq_id); 293 cfg_->Write(irq_.msi->mask_bits_reg(), val); 294 } 295 296 /* If we can mask at the platform interrupt controller level, do so. */ 297 DEBUG_ASSERT(irq_.msi->irq_block_.allocated); 298 DEBUG_ASSERT(irq_id < irq_.msi->irq_block_.num_irq); 299 if (bus_drv_.platform().supports_msi_masking()) 300 bus_drv_.platform().MaskUnmaskMsi(&irq_.msi->irq_block_, irq_id, mask); 301 302 bool ret = hstate.masked; 303 hstate.masked = mask; 304 return ret; 305} 306 307zx_status_t PcieDevice::MaskUnmaskMsiIrq(uint irq_id, bool mask) { 308 if (irq_id >= irq_.handler_count) 309 return ZX_ERR_INVALID_ARGS; 310 311 /* If a mask is being requested, and we cannot mask at either the platform 312 * interrupt controller or the PCI device level, tell the caller that the 313 * operation is unsupported. */ 314 if (mask && !bus_drv_.platform().supports_msi_masking() && !irq_.msi->has_pvm()) 315 return ZX_ERR_NOT_SUPPORTED; 316 317 DEBUG_ASSERT(irq_.handlers); 318 319 { 320 AutoSpinLock handler_lock(&irq_.handlers[irq_id].lock); 321 MaskUnmaskMsiIrqLocked(irq_id, mask); 322 } 323 324 return ZX_OK; 325} 326 327void PcieDevice::MaskAllMsiVectors() { 328 DEBUG_ASSERT(irq_.msi); 329 DEBUG_ASSERT(irq_.msi->is_valid()); 330 331 for (uint i = 0; i < irq_.handler_count; i++) 332 MaskUnmaskMsiIrq(i, true); 333 334 /* In theory, this should not be needed as all of the relevant bits should 335 * have already been masked during the calls to MaskUnmaskMsiIrq. Just to 336 * be careful, however, we explicitly mask all of the upper bits as well. */ 337 if (irq_.msi->has_pvm()) 338 cfg_->Write(irq_.msi->mask_bits_reg(), 0xFFFFFFFF); 339} 340 341void PcieDevice::SetMsiTarget(uint64_t tgt_addr, uint32_t tgt_data) { 342 DEBUG_ASSERT(irq_.msi); 343 DEBUG_ASSERT(irq_.msi->is_valid()); 344 DEBUG_ASSERT(irq_.msi->is64Bit() || !(tgt_addr >> 32)); 345 DEBUG_ASSERT(!(tgt_data >> 16)); 346 347 /* Make sure MSI is disabled_ and all vectors masked (if possible) before 348 * changing the target address and data. */ 349 SetMsiEnb(false); 350 MaskAllMsiVectors(); 351 352 /* lower bits of the address register are common to all forms of the MSI 353 * capability structure. Upper address bits and data position depend on 354 * whether this is a 64 bit or 32 bit version */ 355 cfg_->Write(irq_.msi->addr_reg(), static_cast<uint32_t>(tgt_addr & 0xFFFFFFFF)); 356 if (irq_.msi->is64Bit()) { 357 cfg_->Write(irq_.msi->addr_upper_reg(), static_cast<uint32_t>(tgt_addr >> 32)); 358 } 359 cfg_->Write(irq_.msi->data_reg(), static_cast<uint16_t>(tgt_data & 0xFFFF)); 360} 361 362void PcieDevice::FreeMsiBlock() { 363 /* If no block has been allocated, there is nothing to do */ 364 if (!irq_.msi->irq_block_.allocated) 365 return; 366 367 DEBUG_ASSERT(bus_drv_.platform().supports_msi()); 368 369 /* Mask the IRQ at the platform interrupt controller level if we can, and 370 * unregister any registered handler. */ 371 const msi_block_t* b = &irq_.msi->irq_block_; 372 for (uint i = 0; i < b->num_irq; i++) { 373 if (bus_drv_.platform().supports_msi_masking()) { 374 bus_drv_.platform().MaskUnmaskMsi(b, i, true); 375 } 376 bus_drv_.platform().RegisterMsiHandler(b, i, nullptr, nullptr); 377 } 378 379 /* Give the block of IRQs back to the plaform */ 380 bus_drv_.platform().FreeMsiBlock(&irq_.msi->irq_block_); 381 DEBUG_ASSERT(!irq_.msi->irq_block_.allocated); 382} 383 384void PcieDevice::SetMsiMultiMessageEnb(uint requested_irqs) { 385 DEBUG_ASSERT(irq_.msi); 386 DEBUG_ASSERT(irq_.msi->is_valid()); 387 DEBUG_ASSERT((requested_irqs >= 1) && (requested_irqs <= PCIE_MAX_MSI_IRQS)); 388 389 uint log2 = log2_uint_ceil(requested_irqs); 390 391 DEBUG_ASSERT(log2 <= 5); 392 DEBUG_ASSERT(!log2 || ((0x1u << (log2 - 1)) < requested_irqs)); 393 DEBUG_ASSERT((0x1u << log2) >= requested_irqs); 394 395 cfg_->Write(irq_.msi->ctrl_reg(), 396 PCIE_CAP_MSI_CTRL_SET_MME(log2, cfg_->Read(irq_.msi->ctrl_reg()))); 397} 398 399void PcieDevice::LeaveMsiIrqMode() { 400 /* Disable MSI, mask all vectors and zero out the target */ 401 SetMsiTarget(0x0, 0x0); 402 403 /* Return any allocated irq_ block to the platform, unregistering with 404 * the interrupt controller and synchronizing with the dispatchers in 405 * the process. */ 406 FreeMsiBlock(); 407 408 /* Reset our common state, free any allocated handlers */ 409 ResetCommonIrqBookkeeping(); 410} 411 412zx_status_t PcieDevice::EnterMsiIrqMode(uint requested_irqs) { 413 DEBUG_ASSERT(requested_irqs); 414 415 zx_status_t res = ZX_OK; 416 417 // We cannot go into MSI mode if we don't support MSI at all, or we don't 418 // support the number of IRQs requested 419 if (!irq_.msi || 420 !irq_.msi->is_valid() || 421 !bus_drv_.platform().supports_msi() || 422 (requested_irqs > irq_.msi->max_irqs())) 423 return ZX_ERR_NOT_SUPPORTED; 424 425 // If we support PVM, make sure that we are completely masked before 426 // attempting to allocte the block of IRQs. 427 bool initially_masked; 428 if (irq_.msi->has_pvm()) { 429 cfg_->Write(irq_.msi->mask_bits_reg(), 0xFFFFFFFF); 430 initially_masked = true; 431 } else { 432 // If we cannot mask at the PCI level, then our IRQs will be initially 433 // masked only if the platform supports masking at the interrupt 434 // controller level. 435 initially_masked = bus_drv_.platform().supports_msi_masking(); 436 } 437 438 /* Ask the platform for a chunk of MSI compatible IRQs */ 439 DEBUG_ASSERT(!irq_.msi->irq_block_.allocated); 440 res = bus_drv_.platform().AllocMsiBlock(requested_irqs, 441 irq_.msi->is64Bit(), 442 false, /* is_msix == false */ 443 &irq_.msi->irq_block_); 444 if (res != ZX_OK) { 445 LTRACEF("Failed to allocate a block of %u MSI IRQs for device " 446 "%02x:%02x.%01x (res %d)\n", 447 requested_irqs, bus_id_, dev_id_, func_id_, res); 448 goto bailout; 449 } 450 451 /* Allocate our handler table */ 452 res = AllocIrqHandlers(requested_irqs, initially_masked); 453 if (res != ZX_OK) 454 goto bailout; 455 456 /* Record our new IRQ mode */ 457 irq_.mode = PCIE_IRQ_MODE_MSI; 458 459 /* Program the target write transaction into the MSI registers. As a side 460 * effect, this will ensure that... 461 * 462 * 1) MSI mode has been disabled_ at the top level 463 * 2) Each IRQ has been masked at system level (if supported) 464 * 3) Each IRQ has been masked at the PCI PVM level (if supported) 465 */ 466 DEBUG_ASSERT(irq_.msi->irq_block_.allocated); 467 SetMsiTarget(irq_.msi->irq_block_.tgt_addr, irq_.msi->irq_block_.tgt_data); 468 469 /* Properly program the multi-message enable field in the control register */ 470 SetMsiMultiMessageEnb(requested_irqs); 471 472 /* Register each IRQ with the dispatcher */ 473 DEBUG_ASSERT(irq_.handler_count <= irq_.msi->irq_block_.num_irq); 474 for (uint i = 0; i < irq_.handler_count; ++i) { 475 bus_drv_.platform().RegisterMsiHandler(&irq_.msi->irq_block_, 476 i, 477 PcieDevice::MsiIrqHandlerThunk, 478 irq_.handlers + i); 479 } 480 481 /* Enable MSI at the top level */ 482 SetMsiEnb(true); 483 484bailout: 485 if (res != ZX_OK) 486 LeaveMsiIrqMode(); 487 488 return res; 489} 490 491void PcieDevice::MsiIrqHandler(pcie_irq_handler_state_t& hstate) { 492 DEBUG_ASSERT(irq_.msi); 493 /* No need to save IRQ state; we are in an IRQ handler at the moment. */ 494 AutoSpinLockNoIrqSave handler_lock(&hstate.lock); 495 496 /* Mask our IRQ if we can. */ 497 bool was_masked; 498 if (bus_drv_.platform().supports_msi_masking() || irq_.msi->has_pvm()) { 499 was_masked = MaskUnmaskMsiIrqLocked(hstate.pci_irq_id, true); 500 } else { 501 DEBUG_ASSERT(!hstate.masked); 502 was_masked = false; 503 } 504 505 /* If the IRQ was masked or the handler removed by the time we got here, 506 * leave the IRQ masked, unlock and get out. */ 507 if (was_masked || !hstate.handler) 508 return; 509 510 /* Dispatch */ 511 pcie_irq_handler_retval_t irq_ret = hstate.handler(*this, hstate.pci_irq_id, hstate.ctx); 512 513 /* Re-enable the IRQ if asked to do so */ 514 if (!(irq_ret & PCIE_IRQRET_MASK)) 515 MaskUnmaskMsiIrqLocked(hstate.pci_irq_id, false); 516} 517 518void PcieDevice::MsiIrqHandlerThunk(void *arg) { 519 DEBUG_ASSERT(arg); 520 auto& hstate = *(reinterpret_cast<pcie_irq_handler_state_t*>(arg)); 521 DEBUG_ASSERT(hstate.dev); 522 return hstate.dev->MsiIrqHandler(hstate); 523} 524 525/****************************************************************************** 526 * 527 * Internal implementation of the Kernel facing API. 528 * 529 ******************************************************************************/ 530zx_status_t PcieDevice::QueryIrqModeCapabilitiesLocked(pcie_irq_mode_t mode, 531 pcie_irq_mode_caps_t* out_caps) const { 532 DEBUG_ASSERT(plugged_in_); 533 DEBUG_ASSERT(dev_lock_.IsHeld()); 534 DEBUG_ASSERT(out_caps); 535 536 memset(out_caps, 0, sizeof(*out_caps)); 537 538 switch (mode) { 539 // All devices always support "DISABLED". No need to set the max_irqs to 540 // zero or the PVM supported flag to false, the memset has taken care of 541 // this for us already. 542 case PCIE_IRQ_MODE_DISABLED: 543 return ZX_OK; 544 545 case PCIE_IRQ_MODE_LEGACY: 546 if (!irq_.legacy.pin) 547 return ZX_ERR_NOT_SUPPORTED; 548 549 out_caps->max_irqs = 1; 550 out_caps->per_vector_masking_supported = true; 551 break; 552 553 case PCIE_IRQ_MODE_MSI: 554 /* If the platform does not support MSI, then we don't support MSI, 555 * even if the device does. */ 556 if (!bus_drv_.platform().supports_msi()) 557 return ZX_ERR_NOT_SUPPORTED; 558 559 /* If the device supports MSI, it will have a pointer to the control 560 * structure in config. */ 561 if (!irq_.msi || !irq_.msi->is_valid()) 562 return ZX_ERR_NOT_SUPPORTED; 563 564 /* We support PVM if either the device does, or if the platform is 565 * capable of masking and unmasking individual IRQs from an MSI block 566 * allocation. */ 567 out_caps->max_irqs = irq_.msi->max_irqs(); 568 out_caps->per_vector_masking_supported = irq_.msi->has_pvm() 569 || (bus_drv_.platform().supports_msi_masking()); 570 break; 571 572 case PCIE_IRQ_MODE_MSI_X: 573 /* If the platform does not support MSI, then we don't support MSI, 574 * even if the device does. */ 575 if (!bus_drv_.platform().supports_msi()) 576 return ZX_ERR_NOT_SUPPORTED; 577 578 /* TODO(johngro) : finish MSI-X implementation. */ 579 return ZX_ERR_NOT_SUPPORTED; 580 581 default: 582 return ZX_ERR_INVALID_ARGS; 583 } 584 585 return ZX_OK; 586} 587 588zx_status_t PcieDevice::GetIrqModeLocked(pcie_irq_mode_info_t* out_info) const { 589 DEBUG_ASSERT(plugged_in_); 590 DEBUG_ASSERT(dev_lock_.IsHeld()); 591 DEBUG_ASSERT(out_info); 592 593 out_info->mode = irq_.mode; 594 out_info->max_handlers = irq_.handler_count; 595 out_info->registered_handlers = irq_.registered_handler_count; 596 597 return ZX_OK; 598} 599 600zx_status_t PcieDevice::SetIrqModeLocked(pcie_irq_mode_t mode, uint requested_irqs) { 601 DEBUG_ASSERT(plugged_in_); 602 DEBUG_ASSERT(dev_lock_.IsHeld()); 603 604 // Ensure the mode selection is valid 605 if (mode >= PCIE_IRQ_MODE_COUNT) { 606 return ZX_ERR_INVALID_ARGS; 607 } 608 609 // We need a valid number of IRQs for any mode that isn't strictly 610 // disabling the device's IRQs. 611 if (mode != PCIE_IRQ_MODE_DISABLED && requested_irqs < 1) { 612 return ZX_ERR_INVALID_ARGS; 613 } 614 615 // Leave our present IRQ mode 616 switch (irq_.mode) { 617 case PCIE_IRQ_MODE_LEGACY: 618 DEBUG_ASSERT(list_in_list(&irq_.legacy.shared_handler_node)); 619 LeaveLegacyIrqMode(); 620 break; 621 622 case PCIE_IRQ_MODE_MSI: 623 DEBUG_ASSERT(irq_.msi); 624 DEBUG_ASSERT(irq_.msi->is_valid()); 625 DEBUG_ASSERT(irq_.msi->irq_block_.allocated); 626 LeaveMsiIrqMode(); 627 break; 628 629 // Right now, there should be no way to get into MSI-X mode 630 case PCIE_IRQ_MODE_MSI_X: 631 return ZX_ERR_NOT_SUPPORTED; 632 633 // If we're disabled we have no work to do besides some sanity checks 634 case PCIE_IRQ_MODE_DISABLED: 635 default: 636 DEBUG_ASSERT(!irq_.handlers); 637 DEBUG_ASSERT(!irq_.handler_count); 638 break; 639 } 640 641 // At this point we should be in the disabled state and can enable 642 // the requested IRQ mode. 643 switch (mode) { 644 case PCIE_IRQ_MODE_DISABLED: return ZX_OK; 645 case PCIE_IRQ_MODE_LEGACY: return EnterLegacyIrqMode(requested_irqs); 646 case PCIE_IRQ_MODE_MSI: return EnterMsiIrqMode (requested_irqs); 647 default: return ZX_ERR_NOT_SUPPORTED; 648 } 649} 650 651zx_status_t PcieDevice::RegisterIrqHandlerLocked(uint irq_id, 652 pcie_irq_handler_fn_t handler, 653 void* ctx) { 654 DEBUG_ASSERT(plugged_in_); 655 DEBUG_ASSERT(dev_lock_.IsHeld()); 656 657 /* Cannot register a handler if we are currently disabled_ */ 658 if (irq_.mode == PCIE_IRQ_MODE_DISABLED) 659 return ZX_ERR_BAD_STATE; 660 661 DEBUG_ASSERT(irq_.handlers); 662 DEBUG_ASSERT(irq_.handler_count); 663 664 /* Make sure that the IRQ ID is within range */ 665 if (irq_id >= irq_.handler_count) 666 return ZX_ERR_INVALID_ARGS; 667 668 /* Looks good, register (or unregister the handler) and we are done. */ 669 pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id]; 670 671 /* Update our registered handler bookkeeping. Perform some sanity checks as we do so */ 672 if (hstate.handler) { 673 DEBUG_ASSERT(irq_.registered_handler_count); 674 if (!handler) 675 irq_.registered_handler_count--; 676 } else { 677 if (handler) 678 irq_.registered_handler_count++; 679 } 680 DEBUG_ASSERT(irq_.registered_handler_count <= irq_.handler_count); 681 682 { 683 AutoSpinLock handler_lock(&hstate.lock); 684 hstate.handler = handler; 685 hstate.ctx = handler ? ctx : nullptr; 686 } 687 688 return ZX_OK; 689} 690 691zx_status_t PcieDevice::MaskUnmaskIrqLocked(uint irq_id, bool mask) { 692 DEBUG_ASSERT(plugged_in_); 693 DEBUG_ASSERT(dev_lock_.IsHeld()); 694 695 /* Cannot manipulate mask status while in the DISABLED state */ 696 if (irq_.mode == PCIE_IRQ_MODE_DISABLED) 697 return ZX_ERR_BAD_STATE; 698 699 DEBUG_ASSERT(irq_.handlers); 700 DEBUG_ASSERT(irq_.handler_count); 701 702 /* Make sure that the IRQ ID is within range */ 703 if (irq_id >= irq_.handler_count) 704 return ZX_ERR_INVALID_ARGS; 705 706 /* If we are unmasking (enabling), then we need to make sure that there is a 707 * handler in place for the IRQ we are enabling. */ 708 pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id]; 709 if (!mask && !hstate.handler) 710 return ZX_ERR_BAD_STATE; 711 712 /* OK, everything looks good. Go ahead and make the change based on the 713 * mode we are currently in. */ 714 switch (irq_.mode) { 715 case PCIE_IRQ_MODE_LEGACY: return MaskUnmaskLegacyIrq(mask); 716 case PCIE_IRQ_MODE_MSI: return MaskUnmaskMsiIrq(irq_id, mask); 717 case PCIE_IRQ_MODE_MSI_X: return ZX_ERR_NOT_SUPPORTED; 718 default: 719 DEBUG_ASSERT(false); /* This should be un-possible! */ 720 return ZX_ERR_INTERNAL; 721 } 722 723 return ZX_OK; 724} 725 726/****************************************************************************** 727 * 728 * Kernel API; prototypes in dev/pcie_irqs.h 729 * 730 ******************************************************************************/ 731zx_status_t PcieDevice::QueryIrqModeCapabilities(pcie_irq_mode_t mode, 732 pcie_irq_mode_caps_t* out_caps) const { 733 if (!out_caps) 734 return ZX_ERR_INVALID_ARGS; 735 736 AutoLock dev_lock(&dev_lock_); 737 738 return (plugged_in_ && !disabled_) 739 ? QueryIrqModeCapabilitiesLocked(mode, out_caps) 740 : ZX_ERR_BAD_STATE; 741} 742 743zx_status_t PcieDevice::GetIrqMode(pcie_irq_mode_info_t* out_info) const { 744 if (!out_info) 745 return ZX_ERR_INVALID_ARGS; 746 747 AutoLock dev_lock(&dev_lock_); 748 749 return (plugged_in_ && !disabled_) 750 ? GetIrqModeLocked(out_info) 751 : ZX_ERR_BAD_STATE; 752} 753 754zx_status_t PcieDevice::SetIrqMode(pcie_irq_mode_t mode, uint requested_irqs) { 755 AutoLock dev_lock(&dev_lock_); 756 757 return ((mode == PCIE_IRQ_MODE_DISABLED) || (plugged_in_ && !disabled_)) 758 ? SetIrqModeLocked(mode, requested_irqs) 759 : ZX_ERR_BAD_STATE; 760} 761 762zx_status_t PcieDevice::RegisterIrqHandler(uint irq_id, pcie_irq_handler_fn_t handler, void* ctx) { 763 AutoLock dev_lock(&dev_lock_); 764 765 return (plugged_in_ && !disabled_) 766 ? RegisterIrqHandlerLocked(irq_id, handler, ctx) 767 : ZX_ERR_BAD_STATE; 768} 769 770zx_status_t PcieDevice::MaskUnmaskIrq(uint irq_id, bool mask) { 771 AutoLock dev_lock(&dev_lock_); 772 773 return (mask || (plugged_in_ && !disabled_)) 774 ? MaskUnmaskIrqLocked(irq_id, mask) 775 : ZX_ERR_BAD_STATE; 776} 777 778 779// Map from a device's interrupt pin ID to the proper system IRQ ID. Follow the 780// PCIe graph up to the root, swizzling as we traverse PCIe switches, 781// PCIe-to-PCI bridges, and native PCI-to-PCI bridges. Once we hit the root, 782// perform the final remapping using the platform supplied remapping routine. 783// 784// Platform independent swizzling behavior is documented in the PCIe base 785// specification in section 2.2.8.1 and Table 2-20. 786// 787// Platform dependent remapping is an exercise for the reader. FWIW: PC 788// architectures use the _PRT tables in ACPI to perform the remapping. 789// 790zx_status_t PcieDevice::MapPinToIrqLocked(fbl::RefPtr<PcieUpstreamNode>&& upstream) { 791 DEBUG_ASSERT(dev_lock_.IsHeld()); 792 793 if (!legacy_irq_pin() || (legacy_irq_pin() > PCIE_MAX_LEGACY_IRQ_PINS)) 794 return ZX_ERR_BAD_STATE; 795 796 auto dev = fbl::WrapRefPtr(this); 797 uint pin = legacy_irq_pin() - 1; // Change to 0s indexing 798 799 // Walk up the PCI/PCIe tree, applying the swizzling rules as we go. Stop 800 // when we reach the device which is hanging off of the root bus/root 801 // complex. At this point, platform specific swizzling takes over. 802 while ((upstream != nullptr) && 803 (upstream->type() == PcieUpstreamNode::Type::BRIDGE)) { 804 // TODO(johngro) : Eliminate the null-check of bridge below. Currently, 805 // it is needed because we have gcc/g++'s "null-dereference" warning 806 // turned on, and because of the potentially offsetting nature of static 807 // casting, the compiler cannot be sure that bridge is non-null, just 808 // because upstream was non-null (check in the while predicate, above). 809 // Even adding explicit checks to the Downcast method in RefPtr<> does 810 // not seem to satisfy it. 811 // 812 // Some potential future options include... 813 // 1) Change this to DEBUG_ASSERT and turn off the null-dereference 814 // warning in release builds. 815 // 2) Wait until GCC becomes smart enough to figure this out. 816 // 3) Switch completely to clang (assuming that clang does not have 817 // similar problems). 818 auto bridge = fbl::RefPtr<PcieBridge>::Downcast(fbl::move(upstream)); 819 if (bridge == nullptr) 820 return ZX_ERR_INTERNAL; 821 822 // We need to swizzle every time we pass through... 823 // 1) A PCI-to-PCI bridge (real or virtual) 824 // 2) A PCIe-to-PCI bridge 825 // 3) The Upstream port of a switch. 826 // 827 // We do NOT swizzle when we pass through... 828 // 1) A root port hanging off the root complex. (any swizzling here is up 829 // to the platform implementation) 830 // 2) A Downstream switch port. Since downstream PCIe switch ports are 831 // only permitted to have a single device located at position 0 on 832 // their "bus", it does not really matter if we do the swizzle or 833 // not, since it would turn out to be an identity transformation 834 // anyway. 835 switch (bridge->pcie_device_type()) { 836 // UNKNOWN devices are devices which did not have a PCI Express 837 // Capabilities structure in their capabilities list. Since every 838 // device we pass through on the way up the tree should be a device 839 // with a Type 1 header, these should be PCI-to-PCI bridges (real or 840 // virtual) 841 case PCIE_DEVTYPE_UNKNOWN: 842 case PCIE_DEVTYPE_SWITCH_UPSTREAM_PORT: 843 case PCIE_DEVTYPE_PCIE_TO_PCI_BRIDGE: 844 case PCIE_DEVTYPE_PCI_TO_PCIE_BRIDGE: 845 pin = (pin + dev->dev_id()) % PCIE_MAX_LEGACY_IRQ_PINS; 846 break; 847 848 default: 849 break; 850 } 851 852 // Climb one branch higher up the tree 853 dev = fbl::move(bridge); 854 upstream = dev->GetUpstream(); 855 } 856 857 // If our upstream is ever null as we climb the tree, then something must 858 // have been unplugged as we were climbing. 859 if (upstream == nullptr) 860 return ZX_ERR_BAD_STATE; 861 862 // We have hit root of the tree. Something is very wrong if our 863 // UpstreamNode is not, in fact, a root. 864 if (upstream->type() != PcieUpstreamNode::Type::ROOT) { 865 TRACEF("Failed to map legacy pin to platform IRQ ID for dev " 866 "%02x:%02x.%01x (pin %u). Top of the device tree " 867 "(managed bus ID 0x%02x) does not appear to be either a root or a " 868 "bridge! (type %u)\n", 869 bus_id_, dev_id_, func_id_, irq_.legacy.pin, 870 upstream->managed_bus_id(), static_cast<uint>(upstream->type())); 871 return ZX_ERR_BAD_STATE; 872 } 873 874 // TODO(johngro) : Eliminate the null-check of root below. See the TODO for 875 // the downcast of upstream -> bridge above for details. 876 auto root = fbl::RefPtr<PcieRoot>::Downcast(fbl::move(upstream)); 877 if (root == nullptr) 878 return ZX_ERR_INTERNAL; 879 return root->Swizzle(dev->dev_id(), dev->func_id(), pin, &irq_.legacy.irq_id); 880} 881 882zx_status_t PcieDevice::InitLegacyIrqStateLocked(PcieUpstreamNode& upstream) { 883 DEBUG_ASSERT(dev_lock_.IsHeld()); 884 DEBUG_ASSERT(cfg_); 885 DEBUG_ASSERT(irq_.legacy.shared_handler == nullptr); 886 887 // Make certain that the device's legacy IRQ (if any) has been disabled. 888 ModifyCmdLocked(0u, PCIE_CFG_COMMAND_INT_DISABLE); 889 890 // Does config say that we have a legacy IRQ pin? If so use the bus driver 891 // to map it to the system IRQ ID, then grab a hold of the shared legacy IRQ 892 // handler. 893 irq_.legacy.pin = cfg_->Read(PciConfig::kInterruptPin); 894 if (irq_.legacy.pin) { 895 zx_status_t res = MapPinToIrqLocked(fbl::RefPtr<PcieUpstreamNode>(&upstream)); 896 if (res != ZX_OK) { 897 TRACEF("Failed to map legacy pin to platform IRQ ID for " 898 "dev %02x:%02x.%01x (pin %u)\n", 899 bus_id_, dev_id_, func_id_, 900 irq_.legacy.pin); 901 return res; 902 } 903 904 irq_.legacy.shared_handler = bus_drv_.FindLegacyIrqHandler(irq_.legacy.irq_id); 905 if (irq_.legacy.shared_handler == nullptr) { 906 TRACEF("Failed to find or create shared legacy IRQ handler for " 907 "dev %02x:%02x.%01x (pin %u, irq_id %u)\n", 908 bus_id_, dev_id_, func_id_, 909 irq_.legacy.pin, irq_.legacy.irq_id); 910 return ZX_ERR_NO_RESOURCES; 911 } 912 } 913 914 return ZX_OK; 915} 916 917void PcieBusDriver::ShutdownIrqs() { 918 /* Shut off all of our legacy IRQs and free all of our bookkeeping */ 919 AutoLock lock(&legacy_irq_list_lock_); 920 legacy_irq_list_.clear(); 921} 922 923fbl::RefPtr<SharedLegacyIrqHandler> PcieBusDriver::FindLegacyIrqHandler(uint irq_id) { 924 /* Search to see if we have already created a shared handler for this system 925 * level IRQ id already */ 926 AutoLock lock(&legacy_irq_list_lock_); 927 928 auto iter = legacy_irq_list_.begin(); 929 while (iter != legacy_irq_list_.end()) { 930 if (irq_id == iter->irq_id()) 931 return iter.CopyPointer(); 932 ++iter; 933 } 934 935 auto handler = SharedLegacyIrqHandler::Create(irq_id); 936 if (handler != nullptr) 937 legacy_irq_list_.push_front(handler); 938 939 return handler; 940} 941