1/*-
2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
4 * Copyright (c) 2000, BSDi
5 * Copyright (c) 2004, Scott Long <scottl@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice unmodified, this list of conditions, and the following
13 *    disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include "opt_xbox.h"
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/lock.h>
39#include <sys/kernel.h>
40#include <sys/mutex.h>
41#include <sys/malloc.h>
42#include <sys/queue.h>
43#include <sys/sysctl.h>
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46#include <machine/pci_cfgreg.h>
47#include <machine/pc/bios.h>
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53#include <vm/pmap.h>
54#include <machine/pmap.h>
55
56#ifdef XBOX
57#include <machine/xbox.h>
58#endif
59
60#define PRVERB(a) do {							\
61	if (bootverbose)						\
62		printf a ;						\
63} while(0)
64
65#define PCIE_CACHE 8
66struct pcie_cfg_elem {
67	TAILQ_ENTRY(pcie_cfg_elem)	elem;
68	vm_offset_t	vapage;
69	vm_paddr_t	papage;
70};
71
72enum {
73	CFGMECH_NONE = 0,
74	CFGMECH_1,
75	CFGMECH_2,
76	CFGMECH_PCIE,
77};
78
79SYSCTL_DECL(_hw_pci);
80
81static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU];
82static uint64_t pcie_base;
83static int pcie_minbus, pcie_maxbus;
84static uint32_t pcie_badslots;
85static int cfgmech;
86static int devmax;
87static struct mtx pcicfg_mtx;
88static int mcfg_enable = 1;
89TUNABLE_INT("hw.pci.mcfg", &mcfg_enable);
90SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0,
91    "Enable support for PCI-e memory mapped config access");
92
93static uint32_t	pci_docfgregread(int bus, int slot, int func, int reg,
94		    int bytes);
95static int	pcireg_cfgread(int bus, int slot, int func, int reg, int bytes);
96static void	pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes);
97#ifndef XEN
98static int	pcireg_cfgopen(void);
99#endif
100static int	pciereg_cfgread(int bus, unsigned slot, unsigned func,
101		    unsigned reg, unsigned bytes);
102static void	pciereg_cfgwrite(int bus, unsigned slot, unsigned func,
103		    unsigned reg, int data, unsigned bytes);
104
105/*
106 * Some BIOS writers seem to want to ignore the spec and put
107 * 0 in the intline rather than 255 to indicate none.  Some use
108 * numbers in the range 128-254 to indicate something strange and
109 * apparently undocumented anywhere.  Assume these are completely bogus
110 * and map them to 255, which means "none".
111 */
112static __inline int
113pci_i386_map_intline(int line)
114{
115	if (line == 0 || line >= 128)
116		return (PCI_INVALID_IRQ);
117	return (line);
118}
119
120#ifndef XEN
121static u_int16_t
122pcibios_get_version(void)
123{
124	struct bios_regs args;
125
126	if (PCIbios.ventry == 0) {
127		PRVERB(("pcibios: No call entry point\n"));
128		return (0);
129	}
130	args.eax = PCIBIOS_BIOS_PRESENT;
131	if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) {
132		PRVERB(("pcibios: BIOS_PRESENT call failed\n"));
133		return (0);
134	}
135	if (args.edx != 0x20494350) {
136		PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n"));
137		return (0);
138	}
139	return (args.ebx & 0xffff);
140}
141#endif
142
143/*
144 * Initialise access to PCI configuration space
145 */
146int
147pci_cfgregopen(void)
148{
149#ifdef XEN
150	return (0);
151#else
152	static int		opened = 0;
153	uint64_t		pciebar;
154	u_int16_t		vid, did;
155	u_int16_t		v;
156
157	if (opened)
158		return (1);
159
160	if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0)
161		return (0);
162
163	v = pcibios_get_version();
164	if (v > 0)
165		PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8,
166		    v & 0xff));
167	mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN);
168	opened = 1;
169
170	/* $PIR requires PCI BIOS 2.10 or greater. */
171	if (v >= 0x0210)
172		pci_pir_open();
173
174	if (cfgmech == CFGMECH_PCIE)
175		return (1);
176
177	/*
178	 * Grope around in the PCI config space to see if this is a
179	 * chipset that is capable of doing memory-mapped config cycles.
180	 * This also implies that it can do PCIe extended config cycles.
181	 */
182
183	/* Check for supported chipsets */
184	vid = pci_cfgregread(0, 0, 0, PCIR_VENDOR, 2);
185	did = pci_cfgregread(0, 0, 0, PCIR_DEVICE, 2);
186	switch (vid) {
187	case 0x8086:
188		switch (did) {
189		case 0x3590:
190		case 0x3592:
191			/* Intel 7520 or 7320 */
192			pciebar = pci_cfgregread(0, 0, 0, 0xce, 2) << 16;
193			pcie_cfgregopen(pciebar, 0, 255);
194			break;
195		case 0x2580:
196		case 0x2584:
197		case 0x2590:
198			/* Intel 915, 925, or 915GM */
199			pciebar = pci_cfgregread(0, 0, 0, 0x48, 4);
200			pcie_cfgregopen(pciebar, 0, 255);
201			break;
202		}
203	}
204
205	return(1);
206#endif
207}
208
209static uint32_t
210pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
211{
212
213	if (cfgmech == CFGMECH_PCIE &&
214	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
215	    (bus != 0 || !(1 << slot & pcie_badslots)))
216		return (pciereg_cfgread(bus, slot, func, reg, bytes));
217	else
218		return (pcireg_cfgread(bus, slot, func, reg, bytes));
219}
220
221/*
222 * Read configuration space register
223 */
224u_int32_t
225pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
226{
227	uint32_t line;
228
229	/*
230	 * Some BIOS writers seem to want to ignore the spec and put
231	 * 0 in the intline rather than 255 to indicate none.  The rest of
232	 * the code uses 255 as an invalid IRQ.
233	 */
234	if (reg == PCIR_INTLINE && bytes == 1) {
235		line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1);
236		return (pci_i386_map_intline(line));
237	}
238	return (pci_docfgregread(bus, slot, func, reg, bytes));
239}
240
241/*
242 * Write configuration space register
243 */
244void
245pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
246{
247
248	if (cfgmech == CFGMECH_PCIE &&
249	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
250	    (bus != 0 || !(1 << slot & pcie_badslots)))
251		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
252	else
253		pcireg_cfgwrite(bus, slot, func, reg, data, bytes);
254}
255
256/*
257 * Configuration space access using direct register operations
258 */
259
260/* enable configuration space accesses and return data port address */
261static int
262pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes)
263{
264	int dataport = 0;
265
266#ifdef XBOX
267	if (arch_i386_is_xbox) {
268		/*
269		 * The Xbox MCPX chipset is a derivative of the nForce 1
270		 * chipset. It almost has the same bus layout; some devices
271		 * cannot be used, because they have been removed.
272		 */
273
274		/*
275		 * Devices 00:00.1 and 00:00.2 used to be memory controllers on
276		 * the nForce chipset, but on the Xbox, using them will lockup
277		 * the chipset.
278		 */
279		if (bus == 0 && slot == 0 && (func == 1 || func == 2))
280			return dataport;
281
282		/*
283		 * Bus 1 only contains a VGA controller at 01:00.0. When you try
284		 * to probe beyond that device, you only get garbage, which
285		 * could cause lockups.
286		 */
287		if (bus == 1 && (slot != 0 || func != 0))
288			return dataport;
289
290		/*
291		 * Bus 2 used to contain the AGP controller, but the Xbox MCPX
292		 * doesn't have one. Probing it can cause lockups.
293		 */
294		if (bus >= 2)
295			return dataport;
296	}
297#endif
298
299	if (bus <= PCI_BUSMAX
300	    && slot < devmax
301	    && func <= PCI_FUNCMAX
302	    && (unsigned)reg <= PCI_REGMAX
303	    && bytes != 3
304	    && (unsigned)bytes <= 4
305	    && (reg & (bytes - 1)) == 0) {
306		switch (cfgmech) {
307		case CFGMECH_PCIE:
308		case CFGMECH_1:
309			outl(CONF1_ADDR_PORT, (1U << 31)
310			    | (bus << 16) | (slot << 11)
311			    | (func << 8) | (reg & ~0x03));
312			dataport = CONF1_DATA_PORT + (reg & 0x03);
313			break;
314		case CFGMECH_2:
315			outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1));
316			outb(CONF2_FORWARD_PORT, bus);
317			dataport = 0xc000 | (slot << 8) | reg;
318			break;
319		}
320	}
321	return (dataport);
322}
323
324/* disable configuration space accesses */
325static void
326pci_cfgdisable(void)
327{
328	switch (cfgmech) {
329	case CFGMECH_PCIE:
330	case CFGMECH_1:
331		/*
332		 * Do nothing for the config mechanism 1 case.
333		 * Writing a 0 to the address port can apparently
334		 * confuse some bridges and cause spurious
335		 * access failures.
336		 */
337		break;
338	case CFGMECH_2:
339		outb(CONF2_ENABLE_PORT, 0);
340		break;
341	}
342}
343
344static int
345pcireg_cfgread(int bus, int slot, int func, int reg, int bytes)
346{
347	int data = -1;
348	int port;
349
350	mtx_lock_spin(&pcicfg_mtx);
351	port = pci_cfgenable(bus, slot, func, reg, bytes);
352	if (port != 0) {
353		switch (bytes) {
354		case 1:
355			data = inb(port);
356			break;
357		case 2:
358			data = inw(port);
359			break;
360		case 4:
361			data = inl(port);
362			break;
363		}
364		pci_cfgdisable();
365	}
366	mtx_unlock_spin(&pcicfg_mtx);
367	return (data);
368}
369
370static void
371pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes)
372{
373	int port;
374
375	mtx_lock_spin(&pcicfg_mtx);
376	port = pci_cfgenable(bus, slot, func, reg, bytes);
377	if (port != 0) {
378		switch (bytes) {
379		case 1:
380			outb(port, data);
381			break;
382		case 2:
383			outw(port, data);
384			break;
385		case 4:
386			outl(port, data);
387			break;
388		}
389		pci_cfgdisable();
390	}
391	mtx_unlock_spin(&pcicfg_mtx);
392}
393
394#ifndef XEN
395/* check whether the configuration mechanism has been correctly identified */
396static int
397pci_cfgcheck(int maxdev)
398{
399	uint32_t id, class;
400	uint8_t header;
401	uint8_t device;
402	int port;
403
404	if (bootverbose)
405		printf("pci_cfgcheck:\tdevice ");
406
407	for (device = 0; device < maxdev; device++) {
408		if (bootverbose)
409			printf("%d ", device);
410
411		port = pci_cfgenable(0, device, 0, 0, 4);
412		id = inl(port);
413		if (id == 0 || id == 0xffffffff)
414			continue;
415
416		port = pci_cfgenable(0, device, 0, 8, 4);
417		class = inl(port) >> 8;
418		if (bootverbose)
419			printf("[class=%06x] ", class);
420		if (class == 0 || (class & 0xf870ff) != 0)
421			continue;
422
423		port = pci_cfgenable(0, device, 0, 14, 1);
424		header = inb(port);
425		if (bootverbose)
426			printf("[hdr=%02x] ", header);
427		if ((header & 0x7e) != 0)
428			continue;
429
430		if (bootverbose)
431			printf("is there (id=%08x)\n", id);
432
433		pci_cfgdisable();
434		return (1);
435	}
436	if (bootverbose)
437		printf("-- nothing found\n");
438
439	pci_cfgdisable();
440	return (0);
441}
442
443static int
444pcireg_cfgopen(void)
445{
446	uint32_t mode1res, oldval1;
447	uint8_t mode2res, oldval2;
448
449	/* Check for type #1 first. */
450	oldval1 = inl(CONF1_ADDR_PORT);
451
452	if (bootverbose) {
453		printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n",
454		    oldval1);
455	}
456
457	cfgmech = CFGMECH_1;
458	devmax = 32;
459
460	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK);
461	DELAY(1);
462	mode1res = inl(CONF1_ADDR_PORT);
463	outl(CONF1_ADDR_PORT, oldval1);
464
465	if (bootverbose)
466		printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
467		    CONF1_ENABLE_CHK);
468
469	if (mode1res) {
470		if (pci_cfgcheck(32))
471			return (cfgmech);
472	}
473
474	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1);
475	mode1res = inl(CONF1_ADDR_PORT);
476	outl(CONF1_ADDR_PORT, oldval1);
477
478	if (bootverbose)
479		printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
480		    CONF1_ENABLE_CHK1);
481
482	if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) {
483		if (pci_cfgcheck(32))
484			return (cfgmech);
485	}
486
487	/* Type #1 didn't work, so try type #2. */
488	oldval2 = inb(CONF2_ENABLE_PORT);
489
490	if (bootverbose) {
491		printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n",
492		    oldval2);
493	}
494
495	if ((oldval2 & 0xf0) == 0) {
496
497		cfgmech = CFGMECH_2;
498		devmax = 16;
499
500		outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK);
501		mode2res = inb(CONF2_ENABLE_PORT);
502		outb(CONF2_ENABLE_PORT, oldval2);
503
504		if (bootverbose)
505			printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n",
506			    mode2res, CONF2_ENABLE_CHK);
507
508		if (mode2res == CONF2_ENABLE_RES) {
509			if (bootverbose)
510				printf("pci_open(2a):\tnow trying mechanism 2\n");
511
512			if (pci_cfgcheck(16))
513				return (cfgmech);
514		}
515	}
516
517	/* Nothing worked, so punt. */
518	cfgmech = CFGMECH_NONE;
519	devmax = 0;
520	return (cfgmech);
521}
522
523int
524pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus)
525{
526	struct pcie_cfg_list *pcielist;
527	struct pcie_cfg_elem *pcie_array, *elem;
528#ifdef SMP
529	struct pcpu *pc;
530#endif
531	vm_offset_t va;
532	uint32_t val1, val2;
533	int i, slot;
534
535	if (!mcfg_enable)
536		return (0);
537
538	if (minbus != 0)
539		return (0);
540
541#ifndef PAE
542	if (base >= 0x100000000) {
543		if (bootverbose)
544			printf(
545	    "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n",
546			    (uintmax_t)base);
547		return (0);
548	}
549#endif
550
551	if (bootverbose)
552		printf("PCIe: Memory Mapped configuration base @ 0x%jx\n",
553		    (uintmax_t)base);
554
555#ifdef SMP
556	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
557#endif
558	{
559
560		pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE,
561		    M_DEVBUF, M_NOWAIT);
562		if (pcie_array == NULL)
563			return (0);
564
565		va = kva_alloc(PCIE_CACHE * PAGE_SIZE);
566		if (va == 0) {
567			free(pcie_array, M_DEVBUF);
568			return (0);
569		}
570
571#ifdef SMP
572		pcielist = &pcie_list[pc->pc_cpuid];
573#else
574		pcielist = &pcie_list[0];
575#endif
576		TAILQ_INIT(pcielist);
577		for (i = 0; i < PCIE_CACHE; i++) {
578			elem = &pcie_array[i];
579			elem->vapage = va + (i * PAGE_SIZE);
580			elem->papage = 0;
581			TAILQ_INSERT_HEAD(pcielist, elem, elem);
582		}
583	}
584
585	pcie_base = base;
586	pcie_minbus = minbus;
587	pcie_maxbus = maxbus;
588	cfgmech = CFGMECH_PCIE;
589	devmax = 32;
590
591	/*
592	 * On some AMD systems, some of the devices on bus 0 are
593	 * inaccessible using memory-mapped PCI config access.  Walk
594	 * bus 0 looking for such devices.  For these devices, we will
595	 * fall back to using type 1 config access instead.
596	 */
597	if (pci_cfgregopen() != 0) {
598		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
599			val1 = pcireg_cfgread(0, slot, 0, 0, 4);
600			if (val1 == 0xffffffff)
601				continue;
602
603			val2 = pciereg_cfgread(0, slot, 0, 0, 4);
604			if (val2 != val1)
605				pcie_badslots |= (1 << slot);
606		}
607	}
608
609	return (1);
610}
611#endif /* !XEN */
612
613#define PCIE_PADDR(base, reg, bus, slot, func)	\
614	((base)				+	\
615	((((bus) & 0xff) << 20)		|	\
616	(((slot) & 0x1f) << 15)		|	\
617	(((func) & 0x7) << 12)		|	\
618	((reg) & 0xfff)))
619
620static __inline vm_offset_t
621pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg)
622{
623	struct pcie_cfg_list *pcielist;
624	struct pcie_cfg_elem *elem;
625	vm_paddr_t pa, papage;
626
627	pa = PCIE_PADDR(pcie_base, reg, bus, slot, func);
628	papage = pa & ~PAGE_MASK;
629
630	/*
631	 * Find an element in the cache that matches the physical page desired,
632	 * or create a new mapping from the least recently used element.
633	 * A very simple LRU algorithm is used here, does it need to be more
634	 * efficient?
635	 */
636	pcielist = &pcie_list[PCPU_GET(cpuid)];
637	TAILQ_FOREACH(elem, pcielist, elem) {
638		if (elem->papage == papage)
639			break;
640	}
641
642	if (elem == NULL) {
643		elem = TAILQ_LAST(pcielist, pcie_cfg_list);
644		if (elem->papage != 0) {
645			pmap_kremove(elem->vapage);
646			invlpg(elem->vapage);
647		}
648		pmap_kenter(elem->vapage, papage);
649		elem->papage = papage;
650	}
651
652	if (elem != TAILQ_FIRST(pcielist)) {
653		TAILQ_REMOVE(pcielist, elem, elem);
654		TAILQ_INSERT_HEAD(pcielist, elem, elem);
655	}
656	return (elem->vapage | (pa & PAGE_MASK));
657}
658
659/*
660 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h
661 * have a requirement that all accesses to the memory mapped PCI configuration
662 * space are done using AX class of registers.
663 * Since other vendors do not currently have any contradicting requirements
664 * the AMD access pattern is applied universally.
665 */
666
667static int
668pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg,
669    unsigned bytes)
670{
671	vm_offset_t va;
672	int data = -1;
673
674	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
675	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
676		return (-1);
677
678	critical_enter();
679	va = pciereg_findaddr(bus, slot, func, reg);
680
681	switch (bytes) {
682	case 4:
683		__asm("movl %1, %0" : "=a" (data)
684		    : "m" (*(volatile uint32_t *)va));
685		break;
686	case 2:
687		__asm("movzwl %1, %0" : "=a" (data)
688		    : "m" (*(volatile uint16_t *)va));
689		break;
690	case 1:
691		__asm("movzbl %1, %0" : "=a" (data)
692		    : "m" (*(volatile uint8_t *)va));
693		break;
694	}
695
696	critical_exit();
697	return (data);
698}
699
700static void
701pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data,
702    unsigned bytes)
703{
704	vm_offset_t va;
705
706	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
707	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
708		return;
709
710	critical_enter();
711	va = pciereg_findaddr(bus, slot, func, reg);
712
713	switch (bytes) {
714	case 4:
715		__asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va)
716		    : "a" (data));
717		break;
718	case 2:
719		__asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va)
720		    : "a" ((uint16_t)data));
721		break;
722	case 1:
723		__asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va)
724		    : "a" ((uint8_t)data));
725		break;
726	}
727
728	critical_exit();
729}
730