1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34 35#include <sys/param.h> 36#include <sys/systm.h> 37#include <sys/bus.h> 38#include <sys/lock.h> 39#include <sys/kernel.h> 40#include <sys/mutex.h> 41#include <sys/malloc.h> 42#include <sys/queue.h> 43#include <sys/sysctl.h> 44#include <dev/pci/pcivar.h> 45#include <dev/pci/pcireg.h> 46#include <machine/pci_cfgreg.h> 47#include <machine/pc/bios.h> 48 49#include <vm/vm.h> 50#include <vm/vm_param.h> 51#include <vm/vm_kern.h> 52#include <vm/vm_extern.h> 53#include <vm/pmap.h> 54 55#define PRVERB(a) do { \ 56 if (bootverbose) \ 57 printf a ; \ 58} while(0) 59 60#define PCIE_CACHE 8 61struct pcie_cfg_elem { 62 TAILQ_ENTRY(pcie_cfg_elem) elem; 63 vm_offset_t vapage; 64 vm_paddr_t papage; 65}; 66 67SYSCTL_DECL(_hw_pci); 68 69static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 70static uint64_t pcie_base; 71static int pcie_minbus, pcie_maxbus; 72static uint32_t pcie_badslots; 73int cfgmech; 74static int devmax; 75static struct mtx pcicfg_mtx; 76static int mcfg_enable = 1; 77SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 78 "Enable support for PCI-e memory mapped config access"); 79 80static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 81 int bytes); 82static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 83static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 84static int pcireg_cfgopen(void); 85static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 86 unsigned reg, unsigned bytes); 87static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 88 unsigned reg, int data, unsigned bytes); 89 90/* 91 * Some BIOS writers seem to want to ignore the spec and put 92 * 0 in the intline rather than 255 to indicate none. Some use 93 * numbers in the range 128-254 to indicate something strange and 94 * apparently undocumented anywhere. Assume these are completely bogus 95 * and map them to 255, which means "none". 96 */ 97static __inline int 98pci_i386_map_intline(int line) 99{ 100 if (line == 0 || line >= 128) 101 return (PCI_INVALID_IRQ); 102 return (line); 103} 104 105static u_int16_t 106pcibios_get_version(void) 107{ 108 struct bios_regs args; 109 110 if (PCIbios.ventry == 0) { 111 PRVERB(("pcibios: No call entry point\n")); 112 return (0); 113 } 114 args.eax = PCIBIOS_BIOS_PRESENT; 115 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 116 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 117 return (0); 118 } 119 if (args.edx != 0x20494350) { 120 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 121 return (0); 122 } 123 return (args.ebx & 0xffff); 124} 125 126/* 127 * Initialise access to PCI configuration space 128 */ 129int 130pci_cfgregopen(void) 131{ 132 uint16_t v; 133 static int opened = 0; 134 135 if (opened) 136 return (1); 137 138 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 139 return (0); 140 141 v = pcibios_get_version(); 142 if (v > 0) 143 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 144 v & 0xff)); 145 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 146 opened = 1; 147 148 /* $PIR requires PCI BIOS 2.10 or greater. */ 149 if (v >= 0x0210) 150 pci_pir_open(); 151 152 return (1); 153} 154 155static uint32_t 156pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 157{ 158 159 if (cfgmech == CFGMECH_PCIE && 160 (bus >= pcie_minbus && bus <= pcie_maxbus) && 161 (bus != 0 || !(1 << slot & pcie_badslots))) 162 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 163 else 164 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 165} 166 167/* 168 * Read configuration space register 169 */ 170u_int32_t 171pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 172{ 173 uint32_t line; 174 175 /* 176 * Some BIOS writers seem to want to ignore the spec and put 177 * 0 in the intline rather than 255 to indicate none. The rest of 178 * the code uses 255 as an invalid IRQ. 179 */ 180 if (reg == PCIR_INTLINE && bytes == 1) { 181 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 182 return (pci_i386_map_intline(line)); 183 } 184 return (pci_docfgregread(bus, slot, func, reg, bytes)); 185} 186 187/* 188 * Write configuration space register 189 */ 190void 191pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 192{ 193 194 if (cfgmech == CFGMECH_PCIE && 195 (bus >= pcie_minbus && bus <= pcie_maxbus) && 196 (bus != 0 || !(1 << slot & pcie_badslots))) 197 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 198 else 199 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 200} 201 202/* 203 * Configuration space access using direct register operations 204 */ 205 206/* enable configuration space accesses and return data port address */ 207static int 208pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 209{ 210 int dataport = 0; 211 212 if (bus <= PCI_BUSMAX 213 && slot < devmax 214 && func <= PCI_FUNCMAX 215 && (unsigned)reg <= PCI_REGMAX 216 && bytes != 3 217 && (unsigned)bytes <= 4 218 && (reg & (bytes - 1)) == 0) { 219 switch (cfgmech) { 220 case CFGMECH_PCIE: 221 case CFGMECH_1: 222 outl(CONF1_ADDR_PORT, (1U << 31) 223 | (bus << 16) | (slot << 11) 224 | (func << 8) | (reg & ~0x03)); 225 dataport = CONF1_DATA_PORT + (reg & 0x03); 226 break; 227 case CFGMECH_2: 228 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 229 outb(CONF2_FORWARD_PORT, bus); 230 dataport = 0xc000 | (slot << 8) | reg; 231 break; 232 } 233 } 234 return (dataport); 235} 236 237/* disable configuration space accesses */ 238static void 239pci_cfgdisable(void) 240{ 241 switch (cfgmech) { 242 case CFGMECH_PCIE: 243 case CFGMECH_1: 244 /* 245 * Do nothing for the config mechanism 1 case. 246 * Writing a 0 to the address port can apparently 247 * confuse some bridges and cause spurious 248 * access failures. 249 */ 250 break; 251 case CFGMECH_2: 252 outb(CONF2_ENABLE_PORT, 0); 253 break; 254 } 255} 256 257static int 258pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 259{ 260 int data = -1; 261 int port; 262 263 mtx_lock_spin(&pcicfg_mtx); 264 port = pci_cfgenable(bus, slot, func, reg, bytes); 265 if (port != 0) { 266 switch (bytes) { 267 case 1: 268 data = inb(port); 269 break; 270 case 2: 271 data = inw(port); 272 break; 273 case 4: 274 data = inl(port); 275 break; 276 } 277 pci_cfgdisable(); 278 } 279 mtx_unlock_spin(&pcicfg_mtx); 280 return (data); 281} 282 283static void 284pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 285{ 286 int port; 287 288 mtx_lock_spin(&pcicfg_mtx); 289 port = pci_cfgenable(bus, slot, func, reg, bytes); 290 if (port != 0) { 291 switch (bytes) { 292 case 1: 293 outb(port, data); 294 break; 295 case 2: 296 outw(port, data); 297 break; 298 case 4: 299 outl(port, data); 300 break; 301 } 302 pci_cfgdisable(); 303 } 304 mtx_unlock_spin(&pcicfg_mtx); 305} 306 307/* check whether the configuration mechanism has been correctly identified */ 308static int 309pci_cfgcheck(int maxdev) 310{ 311 uint32_t id, class; 312 uint8_t header; 313 uint8_t device; 314 int port; 315 316 if (bootverbose) 317 printf("pci_cfgcheck:\tdevice "); 318 319 for (device = 0; device < maxdev; device++) { 320 if (bootverbose) 321 printf("%d ", device); 322 323 port = pci_cfgenable(0, device, 0, 0, 4); 324 id = inl(port); 325 if (id == 0 || id == 0xffffffff) 326 continue; 327 328 port = pci_cfgenable(0, device, 0, 8, 4); 329 class = inl(port) >> 8; 330 if (bootverbose) 331 printf("[class=%06x] ", class); 332 if (class == 0 || (class & 0xf870ff) != 0) 333 continue; 334 335 port = pci_cfgenable(0, device, 0, 14, 1); 336 header = inb(port); 337 if (bootverbose) 338 printf("[hdr=%02x] ", header); 339 if ((header & 0x7e) != 0) 340 continue; 341 342 if (bootverbose) 343 printf("is there (id=%08x)\n", id); 344 345 pci_cfgdisable(); 346 return (1); 347 } 348 if (bootverbose) 349 printf("-- nothing found\n"); 350 351 pci_cfgdisable(); 352 return (0); 353} 354 355static int 356pcireg_cfgopen(void) 357{ 358 uint32_t mode1res, oldval1; 359 uint8_t mode2res, oldval2; 360 361 /* Check for type #1 first. */ 362 oldval1 = inl(CONF1_ADDR_PORT); 363 364 if (bootverbose) { 365 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 366 oldval1); 367 } 368 369 cfgmech = CFGMECH_1; 370 devmax = 32; 371 372 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 373 DELAY(1); 374 mode1res = inl(CONF1_ADDR_PORT); 375 outl(CONF1_ADDR_PORT, oldval1); 376 377 if (bootverbose) 378 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 379 CONF1_ENABLE_CHK); 380 381 if (mode1res) { 382 if (pci_cfgcheck(32)) 383 return (cfgmech); 384 } 385 386 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 387 mode1res = inl(CONF1_ADDR_PORT); 388 outl(CONF1_ADDR_PORT, oldval1); 389 390 if (bootverbose) 391 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 392 CONF1_ENABLE_CHK1); 393 394 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 395 if (pci_cfgcheck(32)) 396 return (cfgmech); 397 } 398 399 /* Type #1 didn't work, so try type #2. */ 400 oldval2 = inb(CONF2_ENABLE_PORT); 401 402 if (bootverbose) { 403 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 404 oldval2); 405 } 406 407 if ((oldval2 & 0xf0) == 0) { 408 409 cfgmech = CFGMECH_2; 410 devmax = 16; 411 412 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 413 mode2res = inb(CONF2_ENABLE_PORT); 414 outb(CONF2_ENABLE_PORT, oldval2); 415 416 if (bootverbose) 417 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 418 mode2res, CONF2_ENABLE_CHK); 419 420 if (mode2res == CONF2_ENABLE_RES) { 421 if (bootverbose) 422 printf("pci_open(2a):\tnow trying mechanism 2\n"); 423 424 if (pci_cfgcheck(16)) 425 return (cfgmech); 426 } 427 } 428 429 /* Nothing worked, so punt. */ 430 cfgmech = CFGMECH_NONE; 431 devmax = 0; 432 return (cfgmech); 433} 434 435int 436pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 437{ 438 struct pcie_cfg_list *pcielist; 439 struct pcie_cfg_elem *pcie_array, *elem; 440#ifdef SMP 441 struct pcpu *pc; 442#endif 443 vm_offset_t va; 444 uint32_t val1, val2; 445 int i, slot; 446 447 if (!mcfg_enable) 448 return (0); 449 450 if (minbus != 0) 451 return (0); 452 453#ifndef PAE 454 if (base >= 0x100000000) { 455 if (bootverbose) 456 printf( 457 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 458 (uintmax_t)base); 459 return (0); 460 } 461#endif 462 463 if (bootverbose) 464 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 465 (uintmax_t)base); 466 467#ifdef SMP 468 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 469#endif 470 { 471 472 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 473 M_DEVBUF, M_NOWAIT); 474 if (pcie_array == NULL) 475 return (0); 476 477 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 478 if (va == 0) { 479 free(pcie_array, M_DEVBUF); 480 return (0); 481 } 482 483#ifdef SMP 484 pcielist = &pcie_list[pc->pc_cpuid]; 485#else 486 pcielist = &pcie_list[0]; 487#endif 488 TAILQ_INIT(pcielist); 489 for (i = 0; i < PCIE_CACHE; i++) { 490 elem = &pcie_array[i]; 491 elem->vapage = va + (i * PAGE_SIZE); 492 elem->papage = 0; 493 TAILQ_INSERT_HEAD(pcielist, elem, elem); 494 } 495 } 496 497 pcie_base = base; 498 pcie_minbus = minbus; 499 pcie_maxbus = maxbus; 500 cfgmech = CFGMECH_PCIE; 501 devmax = 32; 502 503 /* 504 * On some AMD systems, some of the devices on bus 0 are 505 * inaccessible using memory-mapped PCI config access. Walk 506 * bus 0 looking for such devices. For these devices, we will 507 * fall back to using type 1 config access instead. 508 */ 509 if (pci_cfgregopen() != 0) { 510 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 511 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 512 if (val1 == 0xffffffff) 513 continue; 514 515 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 516 if (val2 != val1) 517 pcie_badslots |= (1 << slot); 518 } 519 } 520 521 return (1); 522} 523 524#define PCIE_PADDR(base, reg, bus, slot, func) \ 525 ((base) + \ 526 ((((bus) & 0xff) << 20) | \ 527 (((slot) & 0x1f) << 15) | \ 528 (((func) & 0x7) << 12) | \ 529 ((reg) & 0xfff))) 530 531static __inline vm_offset_t 532pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 533{ 534 struct pcie_cfg_list *pcielist; 535 struct pcie_cfg_elem *elem; 536 vm_paddr_t pa, papage; 537 538 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 539 papage = pa & ~PAGE_MASK; 540 541 /* 542 * Find an element in the cache that matches the physical page desired, 543 * or create a new mapping from the least recently used element. 544 * A very simple LRU algorithm is used here, does it need to be more 545 * efficient? 546 */ 547 pcielist = &pcie_list[PCPU_GET(cpuid)]; 548 TAILQ_FOREACH(elem, pcielist, elem) { 549 if (elem->papage == papage) 550 break; 551 } 552 553 if (elem == NULL) { 554 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 555 if (elem->papage != 0) { 556 pmap_kremove(elem->vapage); 557 invlpg(elem->vapage); 558 } 559 pmap_kenter(elem->vapage, papage); 560 elem->papage = papage; 561 } 562 563 if (elem != TAILQ_FIRST(pcielist)) { 564 TAILQ_REMOVE(pcielist, elem, elem); 565 TAILQ_INSERT_HEAD(pcielist, elem, elem); 566 } 567 return (elem->vapage | (pa & PAGE_MASK)); 568} 569 570/* 571 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 572 * have a requirement that all accesses to the memory mapped PCI configuration 573 * space are done using AX class of registers. 574 * Since other vendors do not currently have any contradicting requirements 575 * the AMD access pattern is applied universally. 576 */ 577 578static int 579pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 580 unsigned bytes) 581{ 582 vm_offset_t va; 583 int data = -1; 584 585 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 586 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 587 return (-1); 588 589 critical_enter(); 590 va = pciereg_findaddr(bus, slot, func, reg); 591 592 switch (bytes) { 593 case 4: 594 __asm("movl %1, %0" : "=a" (data) 595 : "m" (*(volatile uint32_t *)va)); 596 break; 597 case 2: 598 __asm("movzwl %1, %0" : "=a" (data) 599 : "m" (*(volatile uint16_t *)va)); 600 break; 601 case 1: 602 __asm("movzbl %1, %0" : "=a" (data) 603 : "m" (*(volatile uint8_t *)va)); 604 break; 605 } 606 607 critical_exit(); 608 return (data); 609} 610 611static void 612pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 613 unsigned bytes) 614{ 615 vm_offset_t va; 616 617 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 618 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 619 return; 620 621 critical_enter(); 622 va = pciereg_findaddr(bus, slot, func, reg); 623 624 switch (bytes) { 625 case 4: 626 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 627 : "a" (data)); 628 break; 629 case 2: 630 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 631 : "a" ((uint16_t)data)); 632 break; 633 case 1: 634 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 635 : "a" ((uint8_t)data)); 636 break; 637 } 638 639 critical_exit(); 640} 641