1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org>
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice unmodified, this list of conditions, and the following
15 *    disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/lock.h>
39#include <sys/kernel.h>
40#include <sys/mutex.h>
41#include <sys/malloc.h>
42#include <sys/queue.h>
43#include <sys/sysctl.h>
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46#include <machine/pci_cfgreg.h>
47#include <machine/pc/bios.h>
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53#include <vm/pmap.h>
54
55#define PRVERB(a) do {							\
56	if (bootverbose)						\
57		printf a ;						\
58} while(0)
59
60#define PCIE_CACHE 8
61struct pcie_cfg_elem {
62	TAILQ_ENTRY(pcie_cfg_elem)	elem;
63	vm_offset_t	vapage;
64	vm_paddr_t	papage;
65};
66
67SYSCTL_DECL(_hw_pci);
68
69static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU];
70static uint64_t pcie_base;
71static int pcie_minbus, pcie_maxbus;
72static uint32_t pcie_badslots;
73int cfgmech;
74static int devmax;
75static struct mtx pcicfg_mtx;
76static int mcfg_enable = 1;
77SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0,
78    "Enable support for PCI-e memory mapped config access");
79
80static uint32_t	pci_docfgregread(int bus, int slot, int func, int reg,
81		    int bytes);
82static int	pcireg_cfgread(int bus, int slot, int func, int reg, int bytes);
83static void	pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes);
84static int	pcireg_cfgopen(void);
85static int	pciereg_cfgread(int bus, unsigned slot, unsigned func,
86		    unsigned reg, unsigned bytes);
87static void	pciereg_cfgwrite(int bus, unsigned slot, unsigned func,
88		    unsigned reg, int data, unsigned bytes);
89
90/*
91 * Some BIOS writers seem to want to ignore the spec and put
92 * 0 in the intline rather than 255 to indicate none.  Some use
93 * numbers in the range 128-254 to indicate something strange and
94 * apparently undocumented anywhere.  Assume these are completely bogus
95 * and map them to 255, which means "none".
96 */
97static __inline int
98pci_i386_map_intline(int line)
99{
100	if (line == 0 || line >= 128)
101		return (PCI_INVALID_IRQ);
102	return (line);
103}
104
105static u_int16_t
106pcibios_get_version(void)
107{
108	struct bios_regs args;
109
110	if (PCIbios.ventry == 0) {
111		PRVERB(("pcibios: No call entry point\n"));
112		return (0);
113	}
114	args.eax = PCIBIOS_BIOS_PRESENT;
115	if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) {
116		PRVERB(("pcibios: BIOS_PRESENT call failed\n"));
117		return (0);
118	}
119	if (args.edx != 0x20494350) {
120		PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n"));
121		return (0);
122	}
123	return (args.ebx & 0xffff);
124}
125
126/*
127 * Initialise access to PCI configuration space
128 */
129int
130pci_cfgregopen(void)
131{
132	uint16_t v;
133	static int opened = 0;
134
135	if (opened)
136		return (1);
137
138	if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0)
139		return (0);
140
141	v = pcibios_get_version();
142	if (v > 0)
143		PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8,
144		    v & 0xff));
145	mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN);
146	opened = 1;
147
148	/* $PIR requires PCI BIOS 2.10 or greater. */
149	if (v >= 0x0210)
150		pci_pir_open();
151
152	return (1);
153}
154
155static uint32_t
156pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
157{
158
159	if (cfgmech == CFGMECH_PCIE &&
160	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
161	    (bus != 0 || !(1 << slot & pcie_badslots)))
162		return (pciereg_cfgread(bus, slot, func, reg, bytes));
163	else
164		return (pcireg_cfgread(bus, slot, func, reg, bytes));
165}
166
167/*
168 * Read configuration space register
169 */
170u_int32_t
171pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
172{
173	uint32_t line;
174
175	/*
176	 * Some BIOS writers seem to want to ignore the spec and put
177	 * 0 in the intline rather than 255 to indicate none.  The rest of
178	 * the code uses 255 as an invalid IRQ.
179	 */
180	if (reg == PCIR_INTLINE && bytes == 1) {
181		line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1);
182		return (pci_i386_map_intline(line));
183	}
184	return (pci_docfgregread(bus, slot, func, reg, bytes));
185}
186
187/*
188 * Write configuration space register
189 */
190void
191pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
192{
193
194	if (cfgmech == CFGMECH_PCIE &&
195	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
196	    (bus != 0 || !(1 << slot & pcie_badslots)))
197		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
198	else
199		pcireg_cfgwrite(bus, slot, func, reg, data, bytes);
200}
201
202/*
203 * Configuration space access using direct register operations
204 */
205
206/* enable configuration space accesses and return data port address */
207static int
208pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes)
209{
210	int dataport = 0;
211
212	if (bus <= PCI_BUSMAX
213	    && slot < devmax
214	    && func <= PCI_FUNCMAX
215	    && (unsigned)reg <= PCI_REGMAX
216	    && bytes != 3
217	    && (unsigned)bytes <= 4
218	    && (reg & (bytes - 1)) == 0) {
219		switch (cfgmech) {
220		case CFGMECH_PCIE:
221		case CFGMECH_1:
222			outl(CONF1_ADDR_PORT, (1U << 31)
223			    | (bus << 16) | (slot << 11)
224			    | (func << 8) | (reg & ~0x03));
225			dataport = CONF1_DATA_PORT + (reg & 0x03);
226			break;
227		case CFGMECH_2:
228			outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1));
229			outb(CONF2_FORWARD_PORT, bus);
230			dataport = 0xc000 | (slot << 8) | reg;
231			break;
232		}
233	}
234	return (dataport);
235}
236
237/* disable configuration space accesses */
238static void
239pci_cfgdisable(void)
240{
241	switch (cfgmech) {
242	case CFGMECH_PCIE:
243	case CFGMECH_1:
244		/*
245		 * Do nothing for the config mechanism 1 case.
246		 * Writing a 0 to the address port can apparently
247		 * confuse some bridges and cause spurious
248		 * access failures.
249		 */
250		break;
251	case CFGMECH_2:
252		outb(CONF2_ENABLE_PORT, 0);
253		break;
254	}
255}
256
257static int
258pcireg_cfgread(int bus, int slot, int func, int reg, int bytes)
259{
260	int data = -1;
261	int port;
262
263	mtx_lock_spin(&pcicfg_mtx);
264	port = pci_cfgenable(bus, slot, func, reg, bytes);
265	if (port != 0) {
266		switch (bytes) {
267		case 1:
268			data = inb(port);
269			break;
270		case 2:
271			data = inw(port);
272			break;
273		case 4:
274			data = inl(port);
275			break;
276		}
277		pci_cfgdisable();
278	}
279	mtx_unlock_spin(&pcicfg_mtx);
280	return (data);
281}
282
283static void
284pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes)
285{
286	int port;
287
288	mtx_lock_spin(&pcicfg_mtx);
289	port = pci_cfgenable(bus, slot, func, reg, bytes);
290	if (port != 0) {
291		switch (bytes) {
292		case 1:
293			outb(port, data);
294			break;
295		case 2:
296			outw(port, data);
297			break;
298		case 4:
299			outl(port, data);
300			break;
301		}
302		pci_cfgdisable();
303	}
304	mtx_unlock_spin(&pcicfg_mtx);
305}
306
307/* check whether the configuration mechanism has been correctly identified */
308static int
309pci_cfgcheck(int maxdev)
310{
311	uint32_t id, class;
312	uint8_t header;
313	uint8_t device;
314	int port;
315
316	if (bootverbose)
317		printf("pci_cfgcheck:\tdevice ");
318
319	for (device = 0; device < maxdev; device++) {
320		if (bootverbose)
321			printf("%d ", device);
322
323		port = pci_cfgenable(0, device, 0, 0, 4);
324		id = inl(port);
325		if (id == 0 || id == 0xffffffff)
326			continue;
327
328		port = pci_cfgenable(0, device, 0, 8, 4);
329		class = inl(port) >> 8;
330		if (bootverbose)
331			printf("[class=%06x] ", class);
332		if (class == 0 || (class & 0xf870ff) != 0)
333			continue;
334
335		port = pci_cfgenable(0, device, 0, 14, 1);
336		header = inb(port);
337		if (bootverbose)
338			printf("[hdr=%02x] ", header);
339		if ((header & 0x7e) != 0)
340			continue;
341
342		if (bootverbose)
343			printf("is there (id=%08x)\n", id);
344
345		pci_cfgdisable();
346		return (1);
347	}
348	if (bootverbose)
349		printf("-- nothing found\n");
350
351	pci_cfgdisable();
352	return (0);
353}
354
355static int
356pcireg_cfgopen(void)
357{
358	uint32_t mode1res, oldval1;
359	uint8_t mode2res, oldval2;
360
361	/* Check for type #1 first. */
362	oldval1 = inl(CONF1_ADDR_PORT);
363
364	if (bootverbose) {
365		printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n",
366		    oldval1);
367	}
368
369	cfgmech = CFGMECH_1;
370	devmax = 32;
371
372	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK);
373	DELAY(1);
374	mode1res = inl(CONF1_ADDR_PORT);
375	outl(CONF1_ADDR_PORT, oldval1);
376
377	if (bootverbose)
378		printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
379		    CONF1_ENABLE_CHK);
380
381	if (mode1res) {
382		if (pci_cfgcheck(32))
383			return (cfgmech);
384	}
385
386	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1);
387	mode1res = inl(CONF1_ADDR_PORT);
388	outl(CONF1_ADDR_PORT, oldval1);
389
390	if (bootverbose)
391		printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
392		    CONF1_ENABLE_CHK1);
393
394	if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) {
395		if (pci_cfgcheck(32))
396			return (cfgmech);
397	}
398
399	/* Type #1 didn't work, so try type #2. */
400	oldval2 = inb(CONF2_ENABLE_PORT);
401
402	if (bootverbose) {
403		printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n",
404		    oldval2);
405	}
406
407	if ((oldval2 & 0xf0) == 0) {
408		cfgmech = CFGMECH_2;
409		devmax = 16;
410
411		outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK);
412		mode2res = inb(CONF2_ENABLE_PORT);
413		outb(CONF2_ENABLE_PORT, oldval2);
414
415		if (bootverbose)
416			printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n",
417			    mode2res, CONF2_ENABLE_CHK);
418
419		if (mode2res == CONF2_ENABLE_RES) {
420			if (bootverbose)
421				printf("pci_open(2a):\tnow trying mechanism 2\n");
422
423			if (pci_cfgcheck(16))
424				return (cfgmech);
425		}
426	}
427
428	/* Nothing worked, so punt. */
429	cfgmech = CFGMECH_NONE;
430	devmax = 0;
431	return (cfgmech);
432}
433
434int
435pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus)
436{
437	struct pcie_cfg_list *pcielist;
438	struct pcie_cfg_elem *pcie_array, *elem;
439#ifdef SMP
440	struct pcpu *pc;
441#endif
442	vm_offset_t va;
443	uint32_t val1, val2;
444	int i, slot;
445
446	if (!mcfg_enable)
447		return (0);
448
449	if (minbus != 0)
450		return (0);
451
452	if (!pae_mode && base >= 0x100000000) {
453		if (bootverbose)
454			printf(
455	    "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n",
456			    (uintmax_t)base);
457		return (0);
458	}
459
460	if (bootverbose)
461		printf("PCIe: Memory Mapped configuration base @ 0x%jx\n",
462		    (uintmax_t)base);
463
464#ifdef SMP
465	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
466#endif
467	{
468		pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE,
469		    M_DEVBUF, M_NOWAIT);
470		if (pcie_array == NULL)
471			return (0);
472
473		va = kva_alloc(PCIE_CACHE * PAGE_SIZE);
474		if (va == 0) {
475			free(pcie_array, M_DEVBUF);
476			return (0);
477		}
478
479#ifdef SMP
480		pcielist = &pcie_list[pc->pc_cpuid];
481#else
482		pcielist = &pcie_list[0];
483#endif
484		TAILQ_INIT(pcielist);
485		for (i = 0; i < PCIE_CACHE; i++) {
486			elem = &pcie_array[i];
487			elem->vapage = va + (i * PAGE_SIZE);
488			elem->papage = 0;
489			TAILQ_INSERT_HEAD(pcielist, elem, elem);
490		}
491	}
492
493	pcie_base = base;
494	pcie_minbus = minbus;
495	pcie_maxbus = maxbus;
496	cfgmech = CFGMECH_PCIE;
497	devmax = 32;
498
499	/*
500	 * On some AMD systems, some of the devices on bus 0 are
501	 * inaccessible using memory-mapped PCI config access.  Walk
502	 * bus 0 looking for such devices.  For these devices, we will
503	 * fall back to using type 1 config access instead.
504	 */
505	if (pci_cfgregopen() != 0) {
506		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
507			val1 = pcireg_cfgread(0, slot, 0, 0, 4);
508			if (val1 == 0xffffffff)
509				continue;
510
511			val2 = pciereg_cfgread(0, slot, 0, 0, 4);
512			if (val2 != val1)
513				pcie_badslots |= (1 << slot);
514		}
515	}
516
517	return (1);
518}
519
520#define PCIE_PADDR(base, reg, bus, slot, func)	\
521	((base)				+	\
522	((((bus) & 0xff) << 20)		|	\
523	(((slot) & 0x1f) << 15)		|	\
524	(((func) & 0x7) << 12)		|	\
525	((reg) & 0xfff)))
526
527static __inline vm_offset_t
528pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg)
529{
530	struct pcie_cfg_list *pcielist;
531	struct pcie_cfg_elem *elem;
532	vm_paddr_t pa, papage;
533
534	pa = PCIE_PADDR(pcie_base, reg, bus, slot, func);
535	papage = pa & ~PAGE_MASK;
536
537	/*
538	 * Find an element in the cache that matches the physical page desired,
539	 * or create a new mapping from the least recently used element.
540	 * A very simple LRU algorithm is used here, does it need to be more
541	 * efficient?
542	 */
543	pcielist = &pcie_list[PCPU_GET(cpuid)];
544	TAILQ_FOREACH(elem, pcielist, elem) {
545		if (elem->papage == papage)
546			break;
547	}
548
549	if (elem == NULL) {
550		elem = TAILQ_LAST(pcielist, pcie_cfg_list);
551		if (elem->papage != 0) {
552			pmap_kremove(elem->vapage);
553			invlpg(elem->vapage);
554		}
555		pmap_kenter(elem->vapage, papage);
556		elem->papage = papage;
557	}
558
559	if (elem != TAILQ_FIRST(pcielist)) {
560		TAILQ_REMOVE(pcielist, elem, elem);
561		TAILQ_INSERT_HEAD(pcielist, elem, elem);
562	}
563	return (elem->vapage | (pa & PAGE_MASK));
564}
565
566/*
567 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h
568 * have a requirement that all accesses to the memory mapped PCI configuration
569 * space are done using AX class of registers.
570 * Since other vendors do not currently have any contradicting requirements
571 * the AMD access pattern is applied universally.
572 */
573
574static int
575pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg,
576    unsigned bytes)
577{
578	vm_offset_t va;
579	int data = -1;
580
581	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
582	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
583		return (-1);
584
585	critical_enter();
586	va = pciereg_findaddr(bus, slot, func, reg);
587
588	switch (bytes) {
589	case 4:
590		__asm("movl %1, %0" : "=a" (data)
591		    : "m" (*(volatile uint32_t *)va));
592		break;
593	case 2:
594		__asm("movzwl %1, %0" : "=a" (data)
595		    : "m" (*(volatile uint16_t *)va));
596		break;
597	case 1:
598		__asm("movzbl %1, %0" : "=a" (data)
599		    : "m" (*(volatile uint8_t *)va));
600		break;
601	}
602
603	critical_exit();
604	return (data);
605}
606
607static void
608pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data,
609    unsigned bytes)
610{
611	vm_offset_t va;
612
613	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
614	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
615		return;
616
617	critical_enter();
618	va = pciereg_findaddr(bus, slot, func, reg);
619
620	switch (bytes) {
621	case 4:
622		__asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va)
623		    : "a" (data));
624		break;
625	case 2:
626		__asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va)
627		    : "a" ((uint16_t)data));
628		break;
629	case 1:
630		__asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va)
631		    : "a" ((uint8_t)data));
632		break;
633	}
634
635	critical_exit();
636}
637