1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org>
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice unmodified, this list of conditions, and the following
15 *    disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/lock.h>
39#include <sys/kernel.h>
40#include <sys/mutex.h>
41#include <sys/malloc.h>
42#include <sys/queue.h>
43#include <sys/sysctl.h>
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46#include <machine/pci_cfgreg.h>
47#include <machine/pc/bios.h>
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53#include <vm/pmap.h>
54
55#define PRVERB(a) do {							\
56	if (bootverbose)						\
57		printf a ;						\
58} while(0)
59
60#define PCIE_CACHE 8
61struct pcie_cfg_elem {
62	TAILQ_ENTRY(pcie_cfg_elem)	elem;
63	vm_offset_t	vapage;
64	vm_paddr_t	papage;
65};
66
67SYSCTL_DECL(_hw_pci);
68
69static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU];
70static uint64_t pcie_base;
71static int pcie_minbus, pcie_maxbus;
72static uint32_t pcie_badslots;
73int cfgmech;
74static int devmax;
75static struct mtx pcicfg_mtx;
76static int mcfg_enable = 1;
77SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0,
78    "Enable support for PCI-e memory mapped config access");
79
80static uint32_t	pci_docfgregread(int bus, int slot, int func, int reg,
81		    int bytes);
82static int	pcireg_cfgread(int bus, int slot, int func, int reg, int bytes);
83static void	pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes);
84static int	pcireg_cfgopen(void);
85static int	pciereg_cfgread(int bus, unsigned slot, unsigned func,
86		    unsigned reg, unsigned bytes);
87static void	pciereg_cfgwrite(int bus, unsigned slot, unsigned func,
88		    unsigned reg, int data, unsigned bytes);
89
90/*
91 * Some BIOS writers seem to want to ignore the spec and put
92 * 0 in the intline rather than 255 to indicate none.  Some use
93 * numbers in the range 128-254 to indicate something strange and
94 * apparently undocumented anywhere.  Assume these are completely bogus
95 * and map them to 255, which means "none".
96 */
97static __inline int
98pci_i386_map_intline(int line)
99{
100	if (line == 0 || line >= 128)
101		return (PCI_INVALID_IRQ);
102	return (line);
103}
104
105static u_int16_t
106pcibios_get_version(void)
107{
108	struct bios_regs args;
109
110	if (PCIbios.ventry == 0) {
111		PRVERB(("pcibios: No call entry point\n"));
112		return (0);
113	}
114	args.eax = PCIBIOS_BIOS_PRESENT;
115	if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) {
116		PRVERB(("pcibios: BIOS_PRESENT call failed\n"));
117		return (0);
118	}
119	if (args.edx != 0x20494350) {
120		PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n"));
121		return (0);
122	}
123	return (args.ebx & 0xffff);
124}
125
126/*
127 * Initialise access to PCI configuration space
128 */
129int
130pci_cfgregopen(void)
131{
132	uint16_t v;
133	static int opened = 0;
134
135	if (opened)
136		return (1);
137
138	if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0)
139		return (0);
140
141	v = pcibios_get_version();
142	if (v > 0)
143		PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8,
144		    v & 0xff));
145	mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN);
146	opened = 1;
147
148	/* $PIR requires PCI BIOS 2.10 or greater. */
149	if (v >= 0x0210)
150		pci_pir_open();
151
152	return (1);
153}
154
155static uint32_t
156pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
157{
158
159	if (cfgmech == CFGMECH_PCIE &&
160	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
161	    (bus != 0 || !(1 << slot & pcie_badslots)))
162		return (pciereg_cfgread(bus, slot, func, reg, bytes));
163	else
164		return (pcireg_cfgread(bus, slot, func, reg, bytes));
165}
166
167/*
168 * Read configuration space register
169 */
170u_int32_t
171pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
172{
173	uint32_t line;
174
175	/*
176	 * Some BIOS writers seem to want to ignore the spec and put
177	 * 0 in the intline rather than 255 to indicate none.  The rest of
178	 * the code uses 255 as an invalid IRQ.
179	 */
180	if (reg == PCIR_INTLINE && bytes == 1) {
181		line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1);
182		return (pci_i386_map_intline(line));
183	}
184	return (pci_docfgregread(bus, slot, func, reg, bytes));
185}
186
187/*
188 * Write configuration space register
189 */
190void
191pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
192{
193
194	if (cfgmech == CFGMECH_PCIE &&
195	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
196	    (bus != 0 || !(1 << slot & pcie_badslots)))
197		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
198	else
199		pcireg_cfgwrite(bus, slot, func, reg, data, bytes);
200}
201
202/*
203 * Configuration space access using direct register operations
204 */
205
206/* enable configuration space accesses and return data port address */
207static int
208pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes)
209{
210	int dataport = 0;
211
212	if (bus <= PCI_BUSMAX
213	    && slot < devmax
214	    && func <= PCI_FUNCMAX
215	    && (unsigned)reg <= PCI_REGMAX
216	    && bytes != 3
217	    && (unsigned)bytes <= 4
218	    && (reg & (bytes - 1)) == 0) {
219		switch (cfgmech) {
220		case CFGMECH_PCIE:
221		case CFGMECH_1:
222			outl(CONF1_ADDR_PORT, (1U << 31)
223			    | (bus << 16) | (slot << 11)
224			    | (func << 8) | (reg & ~0x03));
225			dataport = CONF1_DATA_PORT + (reg & 0x03);
226			break;
227		case CFGMECH_2:
228			outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1));
229			outb(CONF2_FORWARD_PORT, bus);
230			dataport = 0xc000 | (slot << 8) | reg;
231			break;
232		}
233	}
234	return (dataport);
235}
236
237/* disable configuration space accesses */
238static void
239pci_cfgdisable(void)
240{
241	switch (cfgmech) {
242	case CFGMECH_PCIE:
243	case CFGMECH_1:
244		/*
245		 * Do nothing for the config mechanism 1 case.
246		 * Writing a 0 to the address port can apparently
247		 * confuse some bridges and cause spurious
248		 * access failures.
249		 */
250		break;
251	case CFGMECH_2:
252		outb(CONF2_ENABLE_PORT, 0);
253		break;
254	}
255}
256
257static int
258pcireg_cfgread(int bus, int slot, int func, int reg, int bytes)
259{
260	int data = -1;
261	int port;
262
263	mtx_lock_spin(&pcicfg_mtx);
264	port = pci_cfgenable(bus, slot, func, reg, bytes);
265	if (port != 0) {
266		switch (bytes) {
267		case 1:
268			data = inb(port);
269			break;
270		case 2:
271			data = inw(port);
272			break;
273		case 4:
274			data = inl(port);
275			break;
276		}
277		pci_cfgdisable();
278	}
279	mtx_unlock_spin(&pcicfg_mtx);
280	return (data);
281}
282
283static void
284pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes)
285{
286	int port;
287
288	mtx_lock_spin(&pcicfg_mtx);
289	port = pci_cfgenable(bus, slot, func, reg, bytes);
290	if (port != 0) {
291		switch (bytes) {
292		case 1:
293			outb(port, data);
294			break;
295		case 2:
296			outw(port, data);
297			break;
298		case 4:
299			outl(port, data);
300			break;
301		}
302		pci_cfgdisable();
303	}
304	mtx_unlock_spin(&pcicfg_mtx);
305}
306
307/* check whether the configuration mechanism has been correctly identified */
308static int
309pci_cfgcheck(int maxdev)
310{
311	uint32_t id, class;
312	uint8_t header;
313	uint8_t device;
314	int port;
315
316	if (bootverbose)
317		printf("pci_cfgcheck:\tdevice ");
318
319	for (device = 0; device < maxdev; device++) {
320		if (bootverbose)
321			printf("%d ", device);
322
323		port = pci_cfgenable(0, device, 0, 0, 4);
324		id = inl(port);
325		if (id == 0 || id == 0xffffffff)
326			continue;
327
328		port = pci_cfgenable(0, device, 0, 8, 4);
329		class = inl(port) >> 8;
330		if (bootverbose)
331			printf("[class=%06x] ", class);
332		if (class == 0 || (class & 0xf870ff) != 0)
333			continue;
334
335		port = pci_cfgenable(0, device, 0, 14, 1);
336		header = inb(port);
337		if (bootverbose)
338			printf("[hdr=%02x] ", header);
339		if ((header & 0x7e) != 0)
340			continue;
341
342		if (bootverbose)
343			printf("is there (id=%08x)\n", id);
344
345		pci_cfgdisable();
346		return (1);
347	}
348	if (bootverbose)
349		printf("-- nothing found\n");
350
351	pci_cfgdisable();
352	return (0);
353}
354
355static int
356pcireg_cfgopen(void)
357{
358	uint32_t mode1res, oldval1;
359	uint8_t mode2res, oldval2;
360
361	/* Check for type #1 first. */
362	oldval1 = inl(CONF1_ADDR_PORT);
363
364	if (bootverbose) {
365		printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n",
366		    oldval1);
367	}
368
369	cfgmech = CFGMECH_1;
370	devmax = 32;
371
372	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK);
373	DELAY(1);
374	mode1res = inl(CONF1_ADDR_PORT);
375	outl(CONF1_ADDR_PORT, oldval1);
376
377	if (bootverbose)
378		printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
379		    CONF1_ENABLE_CHK);
380
381	if (mode1res) {
382		if (pci_cfgcheck(32))
383			return (cfgmech);
384	}
385
386	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1);
387	mode1res = inl(CONF1_ADDR_PORT);
388	outl(CONF1_ADDR_PORT, oldval1);
389
390	if (bootverbose)
391		printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
392		    CONF1_ENABLE_CHK1);
393
394	if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) {
395		if (pci_cfgcheck(32))
396			return (cfgmech);
397	}
398
399	/* Type #1 didn't work, so try type #2. */
400	oldval2 = inb(CONF2_ENABLE_PORT);
401
402	if (bootverbose) {
403		printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n",
404		    oldval2);
405	}
406
407	if ((oldval2 & 0xf0) == 0) {
408
409		cfgmech = CFGMECH_2;
410		devmax = 16;
411
412		outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK);
413		mode2res = inb(CONF2_ENABLE_PORT);
414		outb(CONF2_ENABLE_PORT, oldval2);
415
416		if (bootverbose)
417			printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n",
418			    mode2res, CONF2_ENABLE_CHK);
419
420		if (mode2res == CONF2_ENABLE_RES) {
421			if (bootverbose)
422				printf("pci_open(2a):\tnow trying mechanism 2\n");
423
424			if (pci_cfgcheck(16))
425				return (cfgmech);
426		}
427	}
428
429	/* Nothing worked, so punt. */
430	cfgmech = CFGMECH_NONE;
431	devmax = 0;
432	return (cfgmech);
433}
434
435int
436pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus)
437{
438	struct pcie_cfg_list *pcielist;
439	struct pcie_cfg_elem *pcie_array, *elem;
440#ifdef SMP
441	struct pcpu *pc;
442#endif
443	vm_offset_t va;
444	uint32_t val1, val2;
445	int i, slot;
446
447	if (!mcfg_enable)
448		return (0);
449
450	if (minbus != 0)
451		return (0);
452
453#ifndef PAE
454	if (base >= 0x100000000) {
455		if (bootverbose)
456			printf(
457	    "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n",
458			    (uintmax_t)base);
459		return (0);
460	}
461#endif
462
463	if (bootverbose)
464		printf("PCIe: Memory Mapped configuration base @ 0x%jx\n",
465		    (uintmax_t)base);
466
467#ifdef SMP
468	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
469#endif
470	{
471
472		pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE,
473		    M_DEVBUF, M_NOWAIT);
474		if (pcie_array == NULL)
475			return (0);
476
477		va = kva_alloc(PCIE_CACHE * PAGE_SIZE);
478		if (va == 0) {
479			free(pcie_array, M_DEVBUF);
480			return (0);
481		}
482
483#ifdef SMP
484		pcielist = &pcie_list[pc->pc_cpuid];
485#else
486		pcielist = &pcie_list[0];
487#endif
488		TAILQ_INIT(pcielist);
489		for (i = 0; i < PCIE_CACHE; i++) {
490			elem = &pcie_array[i];
491			elem->vapage = va + (i * PAGE_SIZE);
492			elem->papage = 0;
493			TAILQ_INSERT_HEAD(pcielist, elem, elem);
494		}
495	}
496
497	pcie_base = base;
498	pcie_minbus = minbus;
499	pcie_maxbus = maxbus;
500	cfgmech = CFGMECH_PCIE;
501	devmax = 32;
502
503	/*
504	 * On some AMD systems, some of the devices on bus 0 are
505	 * inaccessible using memory-mapped PCI config access.  Walk
506	 * bus 0 looking for such devices.  For these devices, we will
507	 * fall back to using type 1 config access instead.
508	 */
509	if (pci_cfgregopen() != 0) {
510		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
511			val1 = pcireg_cfgread(0, slot, 0, 0, 4);
512			if (val1 == 0xffffffff)
513				continue;
514
515			val2 = pciereg_cfgread(0, slot, 0, 0, 4);
516			if (val2 != val1)
517				pcie_badslots |= (1 << slot);
518		}
519	}
520
521	return (1);
522}
523
524#define PCIE_PADDR(base, reg, bus, slot, func)	\
525	((base)				+	\
526	((((bus) & 0xff) << 20)		|	\
527	(((slot) & 0x1f) << 15)		|	\
528	(((func) & 0x7) << 12)		|	\
529	((reg) & 0xfff)))
530
531static __inline vm_offset_t
532pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg)
533{
534	struct pcie_cfg_list *pcielist;
535	struct pcie_cfg_elem *elem;
536	vm_paddr_t pa, papage;
537
538	pa = PCIE_PADDR(pcie_base, reg, bus, slot, func);
539	papage = pa & ~PAGE_MASK;
540
541	/*
542	 * Find an element in the cache that matches the physical page desired,
543	 * or create a new mapping from the least recently used element.
544	 * A very simple LRU algorithm is used here, does it need to be more
545	 * efficient?
546	 */
547	pcielist = &pcie_list[PCPU_GET(cpuid)];
548	TAILQ_FOREACH(elem, pcielist, elem) {
549		if (elem->papage == papage)
550			break;
551	}
552
553	if (elem == NULL) {
554		elem = TAILQ_LAST(pcielist, pcie_cfg_list);
555		if (elem->papage != 0) {
556			pmap_kremove(elem->vapage);
557			invlpg(elem->vapage);
558		}
559		pmap_kenter(elem->vapage, papage);
560		elem->papage = papage;
561	}
562
563	if (elem != TAILQ_FIRST(pcielist)) {
564		TAILQ_REMOVE(pcielist, elem, elem);
565		TAILQ_INSERT_HEAD(pcielist, elem, elem);
566	}
567	return (elem->vapage | (pa & PAGE_MASK));
568}
569
570/*
571 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h
572 * have a requirement that all accesses to the memory mapped PCI configuration
573 * space are done using AX class of registers.
574 * Since other vendors do not currently have any contradicting requirements
575 * the AMD access pattern is applied universally.
576 */
577
578static int
579pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg,
580    unsigned bytes)
581{
582	vm_offset_t va;
583	int data = -1;
584
585	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
586	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
587		return (-1);
588
589	critical_enter();
590	va = pciereg_findaddr(bus, slot, func, reg);
591
592	switch (bytes) {
593	case 4:
594		__asm("movl %1, %0" : "=a" (data)
595		    : "m" (*(volatile uint32_t *)va));
596		break;
597	case 2:
598		__asm("movzwl %1, %0" : "=a" (data)
599		    : "m" (*(volatile uint16_t *)va));
600		break;
601	case 1:
602		__asm("movzbl %1, %0" : "=a" (data)
603		    : "m" (*(volatile uint8_t *)va));
604		break;
605	}
606
607	critical_exit();
608	return (data);
609}
610
611static void
612pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data,
613    unsigned bytes)
614{
615	vm_offset_t va;
616
617	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
618	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
619		return;
620
621	critical_enter();
622	va = pciereg_findaddr(bus, slot, func, reg);
623
624	switch (bytes) {
625	case 4:
626		__asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va)
627		    : "a" (data));
628		break;
629	case 2:
630		__asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va)
631		    : "a" ((uint16_t)data));
632		break;
633	case 1:
634		__asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va)
635		    : "a" ((uint8_t)data));
636		break;
637	}
638
639	critical_exit();
640}
641