1/*-
2 * Copyright (c) 2003 Matthew N. Dodd <winter@jurai.net>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30/*
31 * Written using information gleaned from the
32 * NVIDIA nForce/nForce2 AGPGART Linux Kernel Patch.
33 */
34
35#include <sys/param.h>
36#include <sys/systm.h>
37#include <sys/malloc.h>
38#include <sys/kernel.h>
39#include <sys/module.h>
40#include <sys/bus.h>
41#include <sys/lock.h>
42
43#if __FreeBSD_version < 500000
44#include "opt_pci.h"
45#endif
46
47#if __FreeBSD_version > 500000
48#include <sys/mutex.h>
49#include <sys/proc.h>
50#endif
51
52#include <dev/agp/agppriv.h>
53#include <dev/agp/agpreg.h>
54#include <dev/pci/pcivar.h>
55#include <dev/pci/pcireg.h>
56
57#include <vm/vm.h>
58#include <vm/vm_object.h>
59#include <vm/pmap.h>
60
61#include <machine/bus.h>
62#include <machine/resource.h>
63#include <sys/rman.h>
64
65#define	NVIDIA_VENDORID		0x10de
66#define	NVIDIA_DEVICEID_NFORCE	0x01a4
67#define	NVIDIA_DEVICEID_NFORCE2	0x01e0
68
69struct agp_nvidia_softc {
70	struct agp_softc	agp;
71	u_int32_t		initial_aperture; /* aperture size at startup */
72	struct agp_gatt *	gatt;
73
74	device_t		dev;		/* AGP Controller */
75	device_t		mc1_dev;	/* Memory Controller */
76	device_t		mc2_dev;	/* Memory Controller */
77	device_t		bdev;		/* Bridge */
78
79	u_int32_t		wbc_mask;
80	int			num_dirs;
81	int			num_active_entries;
82	off_t			pg_offset;
83};
84
85static const char *agp_nvidia_match(device_t dev);
86static int agp_nvidia_probe(device_t);
87static int agp_nvidia_attach(device_t);
88static int agp_nvidia_detach(device_t);
89static u_int32_t agp_nvidia_get_aperture(device_t);
90static int agp_nvidia_set_aperture(device_t, u_int32_t);
91static int agp_nvidia_bind_page(device_t, vm_offset_t, vm_offset_t);
92static int agp_nvidia_unbind_page(device_t, vm_offset_t);
93
94static int nvidia_init_iorr(u_int32_t, u_int32_t);
95
96static const char *
97agp_nvidia_match (device_t dev)
98{
99	if (pci_get_class(dev) != PCIC_BRIDGE ||
100	    pci_get_subclass(dev) != PCIS_BRIDGE_HOST ||
101	    pci_get_vendor(dev) != NVIDIA_VENDORID)
102		return (NULL);
103
104	switch (pci_get_device(dev)) {
105	case NVIDIA_DEVICEID_NFORCE:
106		return ("NVIDIA nForce AGP Controller");
107	case NVIDIA_DEVICEID_NFORCE2:
108		return ("NVIDIA nForce2 AGP Controller");
109	}
110	return (NULL);
111}
112
113static int
114agp_nvidia_probe (device_t dev)
115{
116	const char *desc;
117
118	if (resource_disabled("agp", device_get_unit(dev)))
119		return (ENXIO);
120	desc = agp_nvidia_match(dev);
121	if (desc) {
122		device_set_desc(dev, desc);
123		return (BUS_PROBE_DEFAULT);
124	}
125	return (ENXIO);
126}
127
128static int
129agp_nvidia_attach (device_t dev)
130{
131	struct agp_nvidia_softc *sc = device_get_softc(dev);
132	struct agp_gatt *gatt;
133	u_int32_t apbase;
134	u_int32_t aplimit;
135	u_int32_t temp;
136	int size;
137	int i;
138	int error;
139
140	switch (pci_get_device(dev)) {
141	case NVIDIA_DEVICEID_NFORCE:
142		sc->wbc_mask = 0x00010000;
143		break;
144	case NVIDIA_DEVICEID_NFORCE2:
145		sc->wbc_mask = 0x80000000;
146		break;
147	default:
148		device_printf(dev, "Bad chip id\n");
149		return (ENODEV);
150	}
151
152	/* AGP Controller */
153	sc->dev = dev;
154
155	/* Memory Controller 1 */
156	sc->mc1_dev = pci_find_bsf(pci_get_bus(dev), 0, 1);
157	if (sc->mc1_dev == NULL) {
158		device_printf(dev,
159			"Unable to find NVIDIA Memory Controller 1.\n");
160		return (ENODEV);
161	}
162
163	/* Memory Controller 2 */
164	sc->mc2_dev = pci_find_bsf(pci_get_bus(dev), 0, 2);
165	if (sc->mc2_dev == NULL) {
166		device_printf(dev,
167			"Unable to find NVIDIA Memory Controller 2.\n");
168		return (ENODEV);
169	}
170
171	/* AGP Host to PCI Bridge */
172	sc->bdev = pci_find_bsf(pci_get_bus(dev), 30, 0);
173	if (sc->bdev == NULL) {
174		device_printf(dev,
175			"Unable to find NVIDIA AGP Host to PCI Bridge.\n");
176		return (ENODEV);
177	}
178
179	error = agp_generic_attach(dev);
180	if (error)
181		return (error);
182
183	sc->initial_aperture = AGP_GET_APERTURE(dev);
184
185	for (;;) {
186		gatt = agp_alloc_gatt(dev);
187		if (gatt)
188			break;
189		/*
190		 * Probably contigmalloc failure. Try reducing the
191		 * aperture so that the gatt size reduces.
192		 */
193		if (AGP_SET_APERTURE(dev, AGP_GET_APERTURE(dev) / 2))
194			goto fail;
195	}
196	sc->gatt = gatt;
197
198	apbase = rman_get_start(sc->agp.as_aperture);
199	aplimit = apbase + AGP_GET_APERTURE(dev) - 1;
200	pci_write_config(sc->mc2_dev, AGP_NVIDIA_2_APBASE, apbase, 4);
201	pci_write_config(sc->mc2_dev, AGP_NVIDIA_2_APLIMIT, aplimit, 4);
202	pci_write_config(sc->bdev, AGP_NVIDIA_3_APBASE, apbase, 4);
203	pci_write_config(sc->bdev, AGP_NVIDIA_3_APLIMIT, aplimit, 4);
204
205	error = nvidia_init_iorr(apbase, AGP_GET_APERTURE(dev));
206	if (error) {
207		device_printf(dev, "Failed to setup IORRs\n");
208		goto fail;
209	}
210
211	/* directory size is 64k */
212	size = AGP_GET_APERTURE(dev) / 1024 / 1024;
213	sc->num_dirs = size / 64;
214	sc->num_active_entries = (size == 32) ? 16384 : ((size * 1024) / 4);
215	sc->pg_offset = 0;
216	if (sc->num_dirs == 0) {
217		sc->num_dirs = 1;
218		sc->num_active_entries /= (64 / size);
219		sc->pg_offset = (apbase & (64 * 1024 * 1024 - 1) &
220				 ~(AGP_GET_APERTURE(dev) - 1)) / PAGE_SIZE;
221	}
222
223	/* (G)ATT Base Address */
224	for (i = 0; i < 8; i++) {
225		pci_write_config(sc->mc2_dev, AGP_NVIDIA_2_ATTBASE(i),
226				 (sc->gatt->ag_physical +
227				   (i % sc->num_dirs) * 64 * 1024) | 1, 4);
228	}
229
230	/* GTLB Control */
231	temp = pci_read_config(sc->mc2_dev, AGP_NVIDIA_2_GARTCTRL, 4);
232	pci_write_config(sc->mc2_dev, AGP_NVIDIA_2_GARTCTRL, temp | 0x11, 4);
233
234	/* GART Control */
235	temp = pci_read_config(sc->dev, AGP_NVIDIA_0_APSIZE, 4);
236	pci_write_config(sc->dev, AGP_NVIDIA_0_APSIZE, temp | 0x100, 4);
237
238	return (0);
239fail:
240	agp_generic_detach(dev);
241	return (ENOMEM);
242}
243
244static int
245agp_nvidia_detach (device_t dev)
246{
247	struct agp_nvidia_softc *sc = device_get_softc(dev);
248	u_int32_t temp;
249
250	agp_free_cdev(dev);
251
252	/* GART Control */
253	temp = pci_read_config(sc->dev, AGP_NVIDIA_0_APSIZE, 4);
254	pci_write_config(sc->dev, AGP_NVIDIA_0_APSIZE, temp & ~(0x100), 4);
255
256	/* GTLB Control */
257	temp = pci_read_config(sc->mc2_dev, AGP_NVIDIA_2_GARTCTRL, 4);
258	pci_write_config(sc->mc2_dev, AGP_NVIDIA_2_GARTCTRL, temp & ~(0x11), 4);
259
260	/* Put the aperture back the way it started. */
261	AGP_SET_APERTURE(dev, sc->initial_aperture);
262
263	/* restore iorr for previous aperture size */
264	nvidia_init_iorr(rman_get_start(sc->agp.as_aperture),
265			 sc->initial_aperture);
266
267	agp_free_gatt(sc->gatt);
268	agp_free_res(dev);
269
270	return (0);
271}
272
273static u_int32_t
274agp_nvidia_get_aperture(device_t dev)
275{
276	switch (pci_read_config(dev, AGP_NVIDIA_0_APSIZE, 1) & 0x0f) {
277	case 0: return (512 * 1024 * 1024); break;
278	case 8: return (256 * 1024 * 1024); break;
279	case 12: return (128 * 1024 * 1024); break;
280	case 14: return (64 * 1024 * 1024); break;
281	case 15: return (32 * 1024 * 1024); break;
282	default:
283		device_printf(dev, "Invalid aperture setting 0x%x\n",
284		    pci_read_config(dev, AGP_NVIDIA_0_APSIZE, 1));
285		return 0;
286	}
287}
288
289static int
290agp_nvidia_set_aperture(device_t dev, u_int32_t aperture)
291{
292	u_int8_t val;
293	u_int8_t key;
294
295	switch (aperture) {
296	case (512 * 1024 * 1024): key = 0; break;
297	case (256 * 1024 * 1024): key = 8; break;
298	case (128 * 1024 * 1024): key = 12; break;
299	case (64 * 1024 * 1024): key = 14; break;
300	case (32 * 1024 * 1024): key = 15; break;
301	default:
302		device_printf(dev, "Invalid aperture size (%dMb)\n",
303				aperture / 1024 / 1024);
304		return (EINVAL);
305	}
306	val = pci_read_config(dev, AGP_NVIDIA_0_APSIZE, 1);
307	pci_write_config(dev, AGP_NVIDIA_0_APSIZE, ((val & ~0x0f) | key), 1);
308
309	return (0);
310}
311
312static int
313agp_nvidia_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
314{
315	struct agp_nvidia_softc *sc = device_get_softc(dev);
316	u_int32_t index;
317
318	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
319		return (EINVAL);
320
321	index = (sc->pg_offset + offset) >> AGP_PAGE_SHIFT;
322	sc->gatt->ag_virtual[index] = physical | 1;
323
324	return (0);
325}
326
327static int
328agp_nvidia_unbind_page(device_t dev, vm_offset_t offset)
329{
330	struct agp_nvidia_softc *sc = device_get_softc(dev);
331	u_int32_t index;
332
333	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
334		return (EINVAL);
335
336	index = (sc->pg_offset + offset) >> AGP_PAGE_SHIFT;
337	sc->gatt->ag_virtual[index] = 0;
338
339	return (0);
340}
341
342static void
343agp_nvidia_flush_tlb (device_t dev)
344{
345	struct agp_nvidia_softc *sc;
346	u_int32_t wbc_reg, temp;
347	volatile u_int32_t *ag_virtual;
348	int i, pages;
349
350	sc = (struct agp_nvidia_softc *)device_get_softc(dev);
351
352	if (sc->wbc_mask) {
353		wbc_reg = pci_read_config(sc->mc1_dev, AGP_NVIDIA_1_WBC, 4);
354		wbc_reg |= sc->wbc_mask;
355		pci_write_config(sc->mc1_dev, AGP_NVIDIA_1_WBC, wbc_reg, 4);
356
357		/* Wait no more than 3 seconds. */
358		for (i = 0; i < 3000; i++) {
359			wbc_reg = pci_read_config(sc->mc1_dev,
360						  AGP_NVIDIA_1_WBC, 4);
361			if ((sc->wbc_mask & wbc_reg) == 0)
362				break;
363			else
364				DELAY(1000);
365		}
366		if (i == 3000)
367			device_printf(dev,
368				"TLB flush took more than 3 seconds.\n");
369	}
370
371	ag_virtual = (volatile u_int32_t *)sc->gatt->ag_virtual;
372
373	/* Flush TLB entries. */
374	pages = sc->gatt->ag_entries * sizeof(u_int32_t) / PAGE_SIZE;
375	for(i = 0; i < pages; i++)
376		temp = ag_virtual[i * PAGE_SIZE / sizeof(u_int32_t)];
377	for(i = 0; i < pages; i++)
378		temp = ag_virtual[i * PAGE_SIZE / sizeof(u_int32_t)];
379}
380
381#define	SYSCFG		0xC0010010
382#define	IORR_BASE0	0xC0010016
383#define	IORR_MASK0	0xC0010017
384#define	AMD_K7_NUM_IORR	2
385
386static int
387nvidia_init_iorr(u_int32_t addr, u_int32_t size)
388{
389	quad_t base, mask, sys;
390	u_int32_t iorr_addr, free_iorr_addr;
391
392	/* Find the iorr that is already used for the addr */
393	/* If not found, determine the uppermost available iorr */
394	free_iorr_addr = AMD_K7_NUM_IORR;
395	for(iorr_addr = 0; iorr_addr < AMD_K7_NUM_IORR; iorr_addr++) {
396		base = rdmsr(IORR_BASE0 + 2 * iorr_addr);
397		mask = rdmsr(IORR_MASK0 + 2 * iorr_addr);
398
399		if ((base & 0xfffff000ULL) == (addr & 0xfffff000))
400			break;
401
402		if ((mask & 0x00000800ULL) == 0)
403			free_iorr_addr = iorr_addr;
404	}
405
406	if (iorr_addr >= AMD_K7_NUM_IORR) {
407		iorr_addr = free_iorr_addr;
408		if (iorr_addr >= AMD_K7_NUM_IORR)
409			return (EINVAL);
410	}
411
412	base = (addr & ~0xfff) | 0x18;
413	mask = (0xfULL << 32) | ((~(size - 1)) & 0xfffff000) | 0x800;
414	wrmsr(IORR_BASE0 + 2 * iorr_addr, base);
415	wrmsr(IORR_MASK0 + 2 * iorr_addr, mask);
416
417	sys = rdmsr(SYSCFG);
418	sys |= 0x00100000ULL;
419	wrmsr(SYSCFG, sys);
420
421	return (0);
422}
423
424static device_method_t agp_nvidia_methods[] = {
425	/* Device interface */
426	DEVMETHOD(device_probe,		agp_nvidia_probe),
427	DEVMETHOD(device_attach,	agp_nvidia_attach),
428	DEVMETHOD(device_detach,	agp_nvidia_detach),
429	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
430	DEVMETHOD(device_suspend,	bus_generic_suspend),
431	DEVMETHOD(device_resume,	bus_generic_resume),
432
433	/* AGP interface */
434	DEVMETHOD(agp_get_aperture,	agp_nvidia_get_aperture),
435	DEVMETHOD(agp_set_aperture,	agp_nvidia_set_aperture),
436	DEVMETHOD(agp_bind_page,	agp_nvidia_bind_page),
437	DEVMETHOD(agp_unbind_page,	agp_nvidia_unbind_page),
438	DEVMETHOD(agp_flush_tlb,	agp_nvidia_flush_tlb),
439
440	DEVMETHOD(agp_enable,		agp_generic_enable),
441	DEVMETHOD(agp_alloc_memory,	agp_generic_alloc_memory),
442	DEVMETHOD(agp_free_memory,	agp_generic_free_memory),
443	DEVMETHOD(agp_bind_memory,	agp_generic_bind_memory),
444	DEVMETHOD(agp_unbind_memory,	agp_generic_unbind_memory),
445
446	{ 0, 0 }
447};
448
449static driver_t agp_nvidia_driver = {
450	"agp",
451	agp_nvidia_methods,
452	sizeof(struct agp_nvidia_softc),
453};
454
455static devclass_t agp_devclass;
456
457DRIVER_MODULE(agp_nvidia, hostb, agp_nvidia_driver, agp_devclass, 0, 0);
458MODULE_DEPEND(agp_nvidia, agp, 1, 1, 1);
459MODULE_DEPEND(agp_nvidia, pci, 1, 1, 1);
460