1/*-
2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
3 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
4 * Copyright (c) 2000, BSDi
5 * Copyright (c) 2004, Scott Long <scottl@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice unmodified, this list of conditions, and the following
13 *    disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include "opt_xbox.h"
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/bus.h>
38#include <sys/lock.h>
39#include <sys/kernel.h>
40#include <sys/mutex.h>
41#include <sys/malloc.h>
42#include <sys/queue.h>
43#include <sys/sysctl.h>
44#include <dev/pci/pcivar.h>
45#include <dev/pci/pcireg.h>
46#include <machine/pci_cfgreg.h>
47#include <machine/pc/bios.h>
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53#include <vm/pmap.h>
54
55#ifdef XBOX
56#include <machine/xbox.h>
57#endif
58
59#define PRVERB(a) do {							\
60	if (bootverbose)						\
61		printf a ;						\
62} while(0)
63
64#define PCIE_CACHE 8
65struct pcie_cfg_elem {
66	TAILQ_ENTRY(pcie_cfg_elem)	elem;
67	vm_offset_t	vapage;
68	vm_paddr_t	papage;
69};
70
71enum {
72	CFGMECH_NONE = 0,
73	CFGMECH_1,
74	CFGMECH_2,
75	CFGMECH_PCIE,
76};
77
78SYSCTL_DECL(_hw_pci);
79
80static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU];
81static uint64_t pcie_base;
82static int pcie_minbus, pcie_maxbus;
83static uint32_t pcie_badslots;
84static int cfgmech;
85static int devmax;
86static struct mtx pcicfg_mtx;
87static int mcfg_enable = 1;
88SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0,
89    "Enable support for PCI-e memory mapped config access");
90
91static uint32_t	pci_docfgregread(int bus, int slot, int func, int reg,
92		    int bytes);
93static int	pcireg_cfgread(int bus, int slot, int func, int reg, int bytes);
94static void	pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes);
95static int	pcireg_cfgopen(void);
96static int	pciereg_cfgread(int bus, unsigned slot, unsigned func,
97		    unsigned reg, unsigned bytes);
98static void	pciereg_cfgwrite(int bus, unsigned slot, unsigned func,
99		    unsigned reg, int data, unsigned bytes);
100
101/*
102 * Some BIOS writers seem to want to ignore the spec and put
103 * 0 in the intline rather than 255 to indicate none.  Some use
104 * numbers in the range 128-254 to indicate something strange and
105 * apparently undocumented anywhere.  Assume these are completely bogus
106 * and map them to 255, which means "none".
107 */
108static __inline int
109pci_i386_map_intline(int line)
110{
111	if (line == 0 || line >= 128)
112		return (PCI_INVALID_IRQ);
113	return (line);
114}
115
116static u_int16_t
117pcibios_get_version(void)
118{
119	struct bios_regs args;
120
121	if (PCIbios.ventry == 0) {
122		PRVERB(("pcibios: No call entry point\n"));
123		return (0);
124	}
125	args.eax = PCIBIOS_BIOS_PRESENT;
126	if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) {
127		PRVERB(("pcibios: BIOS_PRESENT call failed\n"));
128		return (0);
129	}
130	if (args.edx != 0x20494350) {
131		PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n"));
132		return (0);
133	}
134	return (args.ebx & 0xffff);
135}
136
137/*
138 * Initialise access to PCI configuration space
139 */
140int
141pci_cfgregopen(void)
142{
143	static int		opened = 0;
144	uint64_t		pciebar;
145	u_int16_t		vid, did;
146	u_int16_t		v;
147
148	if (opened)
149		return (1);
150
151	if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0)
152		return (0);
153
154	v = pcibios_get_version();
155	if (v > 0)
156		PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8,
157		    v & 0xff));
158	mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN);
159	opened = 1;
160
161	/* $PIR requires PCI BIOS 2.10 or greater. */
162	if (v >= 0x0210)
163		pci_pir_open();
164
165	if (cfgmech == CFGMECH_PCIE)
166		return (1);
167
168	/*
169	 * Grope around in the PCI config space to see if this is a
170	 * chipset that is capable of doing memory-mapped config cycles.
171	 * This also implies that it can do PCIe extended config cycles.
172	 */
173
174	/* Check for supported chipsets */
175	vid = pci_cfgregread(0, 0, 0, PCIR_VENDOR, 2);
176	did = pci_cfgregread(0, 0, 0, PCIR_DEVICE, 2);
177	switch (vid) {
178	case 0x8086:
179		switch (did) {
180		case 0x3590:
181		case 0x3592:
182			/* Intel 7520 or 7320 */
183			pciebar = pci_cfgregread(0, 0, 0, 0xce, 2) << 16;
184			pcie_cfgregopen(pciebar, 0, 255);
185			break;
186		case 0x2580:
187		case 0x2584:
188		case 0x2590:
189			/* Intel 915, 925, or 915GM */
190			pciebar = pci_cfgregread(0, 0, 0, 0x48, 4);
191			pcie_cfgregopen(pciebar, 0, 255);
192			break;
193		}
194	}
195
196	return(1);
197}
198
199static uint32_t
200pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
201{
202
203	if (cfgmech == CFGMECH_PCIE &&
204	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
205	    (bus != 0 || !(1 << slot & pcie_badslots)))
206		return (pciereg_cfgread(bus, slot, func, reg, bytes));
207	else
208		return (pcireg_cfgread(bus, slot, func, reg, bytes));
209}
210
211/*
212 * Read configuration space register
213 */
214u_int32_t
215pci_cfgregread(int bus, int slot, int func, int reg, int bytes)
216{
217	uint32_t line;
218
219	/*
220	 * Some BIOS writers seem to want to ignore the spec and put
221	 * 0 in the intline rather than 255 to indicate none.  The rest of
222	 * the code uses 255 as an invalid IRQ.
223	 */
224	if (reg == PCIR_INTLINE && bytes == 1) {
225		line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1);
226		return (pci_i386_map_intline(line));
227	}
228	return (pci_docfgregread(bus, slot, func, reg, bytes));
229}
230
231/*
232 * Write configuration space register
233 */
234void
235pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
236{
237
238	if (cfgmech == CFGMECH_PCIE &&
239	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
240	    (bus != 0 || !(1 << slot & pcie_badslots)))
241		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
242	else
243		pcireg_cfgwrite(bus, slot, func, reg, data, bytes);
244}
245
246/*
247 * Configuration space access using direct register operations
248 */
249
250/* enable configuration space accesses and return data port address */
251static int
252pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes)
253{
254	int dataport = 0;
255
256#ifdef XBOX
257	if (arch_i386_is_xbox) {
258		/*
259		 * The Xbox MCPX chipset is a derivative of the nForce 1
260		 * chipset. It almost has the same bus layout; some devices
261		 * cannot be used, because they have been removed.
262		 */
263
264		/*
265		 * Devices 00:00.1 and 00:00.2 used to be memory controllers on
266		 * the nForce chipset, but on the Xbox, using them will lockup
267		 * the chipset.
268		 */
269		if (bus == 0 && slot == 0 && (func == 1 || func == 2))
270			return dataport;
271
272		/*
273		 * Bus 1 only contains a VGA controller at 01:00.0. When you try
274		 * to probe beyond that device, you only get garbage, which
275		 * could cause lockups.
276		 */
277		if (bus == 1 && (slot != 0 || func != 0))
278			return dataport;
279
280		/*
281		 * Bus 2 used to contain the AGP controller, but the Xbox MCPX
282		 * doesn't have one. Probing it can cause lockups.
283		 */
284		if (bus >= 2)
285			return dataport;
286	}
287#endif
288
289	if (bus <= PCI_BUSMAX
290	    && slot < devmax
291	    && func <= PCI_FUNCMAX
292	    && (unsigned)reg <= PCI_REGMAX
293	    && bytes != 3
294	    && (unsigned)bytes <= 4
295	    && (reg & (bytes - 1)) == 0) {
296		switch (cfgmech) {
297		case CFGMECH_PCIE:
298		case CFGMECH_1:
299			outl(CONF1_ADDR_PORT, (1U << 31)
300			    | (bus << 16) | (slot << 11)
301			    | (func << 8) | (reg & ~0x03));
302			dataport = CONF1_DATA_PORT + (reg & 0x03);
303			break;
304		case CFGMECH_2:
305			outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1));
306			outb(CONF2_FORWARD_PORT, bus);
307			dataport = 0xc000 | (slot << 8) | reg;
308			break;
309		}
310	}
311	return (dataport);
312}
313
314/* disable configuration space accesses */
315static void
316pci_cfgdisable(void)
317{
318	switch (cfgmech) {
319	case CFGMECH_PCIE:
320	case CFGMECH_1:
321		/*
322		 * Do nothing for the config mechanism 1 case.
323		 * Writing a 0 to the address port can apparently
324		 * confuse some bridges and cause spurious
325		 * access failures.
326		 */
327		break;
328	case CFGMECH_2:
329		outb(CONF2_ENABLE_PORT, 0);
330		break;
331	}
332}
333
334static int
335pcireg_cfgread(int bus, int slot, int func, int reg, int bytes)
336{
337	int data = -1;
338	int port;
339
340	mtx_lock_spin(&pcicfg_mtx);
341	port = pci_cfgenable(bus, slot, func, reg, bytes);
342	if (port != 0) {
343		switch (bytes) {
344		case 1:
345			data = inb(port);
346			break;
347		case 2:
348			data = inw(port);
349			break;
350		case 4:
351			data = inl(port);
352			break;
353		}
354		pci_cfgdisable();
355	}
356	mtx_unlock_spin(&pcicfg_mtx);
357	return (data);
358}
359
360static void
361pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes)
362{
363	int port;
364
365	mtx_lock_spin(&pcicfg_mtx);
366	port = pci_cfgenable(bus, slot, func, reg, bytes);
367	if (port != 0) {
368		switch (bytes) {
369		case 1:
370			outb(port, data);
371			break;
372		case 2:
373			outw(port, data);
374			break;
375		case 4:
376			outl(port, data);
377			break;
378		}
379		pci_cfgdisable();
380	}
381	mtx_unlock_spin(&pcicfg_mtx);
382}
383
384/* check whether the configuration mechanism has been correctly identified */
385static int
386pci_cfgcheck(int maxdev)
387{
388	uint32_t id, class;
389	uint8_t header;
390	uint8_t device;
391	int port;
392
393	if (bootverbose)
394		printf("pci_cfgcheck:\tdevice ");
395
396	for (device = 0; device < maxdev; device++) {
397		if (bootverbose)
398			printf("%d ", device);
399
400		port = pci_cfgenable(0, device, 0, 0, 4);
401		id = inl(port);
402		if (id == 0 || id == 0xffffffff)
403			continue;
404
405		port = pci_cfgenable(0, device, 0, 8, 4);
406		class = inl(port) >> 8;
407		if (bootverbose)
408			printf("[class=%06x] ", class);
409		if (class == 0 || (class & 0xf870ff) != 0)
410			continue;
411
412		port = pci_cfgenable(0, device, 0, 14, 1);
413		header = inb(port);
414		if (bootverbose)
415			printf("[hdr=%02x] ", header);
416		if ((header & 0x7e) != 0)
417			continue;
418
419		if (bootverbose)
420			printf("is there (id=%08x)\n", id);
421
422		pci_cfgdisable();
423		return (1);
424	}
425	if (bootverbose)
426		printf("-- nothing found\n");
427
428	pci_cfgdisable();
429	return (0);
430}
431
432static int
433pcireg_cfgopen(void)
434{
435	uint32_t mode1res, oldval1;
436	uint8_t mode2res, oldval2;
437
438	/* Check for type #1 first. */
439	oldval1 = inl(CONF1_ADDR_PORT);
440
441	if (bootverbose) {
442		printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n",
443		    oldval1);
444	}
445
446	cfgmech = CFGMECH_1;
447	devmax = 32;
448
449	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK);
450	DELAY(1);
451	mode1res = inl(CONF1_ADDR_PORT);
452	outl(CONF1_ADDR_PORT, oldval1);
453
454	if (bootverbose)
455		printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
456		    CONF1_ENABLE_CHK);
457
458	if (mode1res) {
459		if (pci_cfgcheck(32))
460			return (cfgmech);
461	}
462
463	outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1);
464	mode1res = inl(CONF1_ADDR_PORT);
465	outl(CONF1_ADDR_PORT, oldval1);
466
467	if (bootverbose)
468		printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n",  mode1res,
469		    CONF1_ENABLE_CHK1);
470
471	if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) {
472		if (pci_cfgcheck(32))
473			return (cfgmech);
474	}
475
476	/* Type #1 didn't work, so try type #2. */
477	oldval2 = inb(CONF2_ENABLE_PORT);
478
479	if (bootverbose) {
480		printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n",
481		    oldval2);
482	}
483
484	if ((oldval2 & 0xf0) == 0) {
485
486		cfgmech = CFGMECH_2;
487		devmax = 16;
488
489		outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK);
490		mode2res = inb(CONF2_ENABLE_PORT);
491		outb(CONF2_ENABLE_PORT, oldval2);
492
493		if (bootverbose)
494			printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n",
495			    mode2res, CONF2_ENABLE_CHK);
496
497		if (mode2res == CONF2_ENABLE_RES) {
498			if (bootverbose)
499				printf("pci_open(2a):\tnow trying mechanism 2\n");
500
501			if (pci_cfgcheck(16))
502				return (cfgmech);
503		}
504	}
505
506	/* Nothing worked, so punt. */
507	cfgmech = CFGMECH_NONE;
508	devmax = 0;
509	return (cfgmech);
510}
511
512int
513pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus)
514{
515	struct pcie_cfg_list *pcielist;
516	struct pcie_cfg_elem *pcie_array, *elem;
517#ifdef SMP
518	struct pcpu *pc;
519#endif
520	vm_offset_t va;
521	uint32_t val1, val2;
522	int i, slot;
523
524	if (!mcfg_enable)
525		return (0);
526
527	if (minbus != 0)
528		return (0);
529
530#ifndef PAE
531	if (base >= 0x100000000) {
532		if (bootverbose)
533			printf(
534	    "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n",
535			    (uintmax_t)base);
536		return (0);
537	}
538#endif
539
540	if (bootverbose)
541		printf("PCIe: Memory Mapped configuration base @ 0x%jx\n",
542		    (uintmax_t)base);
543
544#ifdef SMP
545	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
546#endif
547	{
548
549		pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE,
550		    M_DEVBUF, M_NOWAIT);
551		if (pcie_array == NULL)
552			return (0);
553
554		va = kva_alloc(PCIE_CACHE * PAGE_SIZE);
555		if (va == 0) {
556			free(pcie_array, M_DEVBUF);
557			return (0);
558		}
559
560#ifdef SMP
561		pcielist = &pcie_list[pc->pc_cpuid];
562#else
563		pcielist = &pcie_list[0];
564#endif
565		TAILQ_INIT(pcielist);
566		for (i = 0; i < PCIE_CACHE; i++) {
567			elem = &pcie_array[i];
568			elem->vapage = va + (i * PAGE_SIZE);
569			elem->papage = 0;
570			TAILQ_INSERT_HEAD(pcielist, elem, elem);
571		}
572	}
573
574	pcie_base = base;
575	pcie_minbus = minbus;
576	pcie_maxbus = maxbus;
577	cfgmech = CFGMECH_PCIE;
578	devmax = 32;
579
580	/*
581	 * On some AMD systems, some of the devices on bus 0 are
582	 * inaccessible using memory-mapped PCI config access.  Walk
583	 * bus 0 looking for such devices.  For these devices, we will
584	 * fall back to using type 1 config access instead.
585	 */
586	if (pci_cfgregopen() != 0) {
587		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
588			val1 = pcireg_cfgread(0, slot, 0, 0, 4);
589			if (val1 == 0xffffffff)
590				continue;
591
592			val2 = pciereg_cfgread(0, slot, 0, 0, 4);
593			if (val2 != val1)
594				pcie_badslots |= (1 << slot);
595		}
596	}
597
598	return (1);
599}
600
601#define PCIE_PADDR(base, reg, bus, slot, func)	\
602	((base)				+	\
603	((((bus) & 0xff) << 20)		|	\
604	(((slot) & 0x1f) << 15)		|	\
605	(((func) & 0x7) << 12)		|	\
606	((reg) & 0xfff)))
607
608static __inline vm_offset_t
609pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg)
610{
611	struct pcie_cfg_list *pcielist;
612	struct pcie_cfg_elem *elem;
613	vm_paddr_t pa, papage;
614
615	pa = PCIE_PADDR(pcie_base, reg, bus, slot, func);
616	papage = pa & ~PAGE_MASK;
617
618	/*
619	 * Find an element in the cache that matches the physical page desired,
620	 * or create a new mapping from the least recently used element.
621	 * A very simple LRU algorithm is used here, does it need to be more
622	 * efficient?
623	 */
624	pcielist = &pcie_list[PCPU_GET(cpuid)];
625	TAILQ_FOREACH(elem, pcielist, elem) {
626		if (elem->papage == papage)
627			break;
628	}
629
630	if (elem == NULL) {
631		elem = TAILQ_LAST(pcielist, pcie_cfg_list);
632		if (elem->papage != 0) {
633			pmap_kremove(elem->vapage);
634			invlpg(elem->vapage);
635		}
636		pmap_kenter(elem->vapage, papage);
637		elem->papage = papage;
638	}
639
640	if (elem != TAILQ_FIRST(pcielist)) {
641		TAILQ_REMOVE(pcielist, elem, elem);
642		TAILQ_INSERT_HEAD(pcielist, elem, elem);
643	}
644	return (elem->vapage | (pa & PAGE_MASK));
645}
646
647/*
648 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h
649 * have a requirement that all accesses to the memory mapped PCI configuration
650 * space are done using AX class of registers.
651 * Since other vendors do not currently have any contradicting requirements
652 * the AMD access pattern is applied universally.
653 */
654
655static int
656pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg,
657    unsigned bytes)
658{
659	vm_offset_t va;
660	int data = -1;
661
662	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
663	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
664		return (-1);
665
666	critical_enter();
667	va = pciereg_findaddr(bus, slot, func, reg);
668
669	switch (bytes) {
670	case 4:
671		__asm("movl %1, %0" : "=a" (data)
672		    : "m" (*(volatile uint32_t *)va));
673		break;
674	case 2:
675		__asm("movzwl %1, %0" : "=a" (data)
676		    : "m" (*(volatile uint16_t *)va));
677		break;
678	case 1:
679		__asm("movzbl %1, %0" : "=a" (data)
680		    : "m" (*(volatile uint8_t *)va));
681		break;
682	}
683
684	critical_exit();
685	return (data);
686}
687
688static void
689pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data,
690    unsigned bytes)
691{
692	vm_offset_t va;
693
694	if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX ||
695	    func > PCI_FUNCMAX || reg > PCIE_REGMAX)
696		return;
697
698	critical_enter();
699	va = pciereg_findaddr(bus, slot, func, reg);
700
701	switch (bytes) {
702	case 4:
703		__asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va)
704		    : "a" (data));
705		break;
706	case 2:
707		__asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va)
708		    : "a" ((uint16_t)data));
709		break;
710	case 1:
711		__asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va)
712		    : "a" ((uint8_t)data));
713		break;
714	}
715
716	critical_exit();
717}
718