1/*
2 * arch/powerpc/sysdev/dart_iommu.c
3 *
4 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
5 * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
6 *                    IBM Corporation
7 *
8 * Based on pSeries_iommu.c:
9 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
10 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
11 *
12 * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
13 *
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 *
20 * This program is distributed in the hope that it will be useful,
21 * but WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23 * GNU General Public License for more details.
24 *
25 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
28 */
29
30#include <linux/init.h>
31#include <linux/types.h>
32#include <linux/slab.h>
33#include <linux/mm.h>
34#include <linux/spinlock.h>
35#include <linux/string.h>
36#include <linux/pci.h>
37#include <linux/dma-mapping.h>
38#include <linux/vmalloc.h>
39#include <linux/suspend.h>
40#include <asm/io.h>
41#include <asm/prom.h>
42#include <asm/iommu.h>
43#include <asm/pci-bridge.h>
44#include <asm/machdep.h>
45#include <asm/abs_addr.h>
46#include <asm/cacheflush.h>
47#include <asm/lmb.h>
48#include <asm/ppc-pci.h>
49
50#include "dart.h"
51
52/* Physical base address and size of the DART table */
53unsigned long dart_tablebase; /* exported to htab_initialize */
54static unsigned long dart_tablesize;
55
56/* Virtual base address of the DART table */
57static u32 *dart_vbase;
58#ifdef CONFIG_PM
59static u32 *dart_copy;
60#endif
61
62/* Mapped base address for the dart */
63static unsigned int __iomem *dart;
64
65/* Dummy val that entries are set to when unused */
66static unsigned int dart_emptyval;
67
68static struct iommu_table iommu_table_dart;
69static int iommu_table_dart_inited;
70static int dart_dirty;
71static int dart_is_u4;
72
73#define DBG(...)
74
75static inline void dart_tlb_invalidate_all(void)
76{
77	unsigned long l = 0;
78	unsigned int reg, inv_bit;
79	unsigned long limit;
80
81	DBG("dart: flush\n");
82
83	/* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
84	 * control register and wait for it to clear.
85	 *
86	 * Gotcha: Sometimes, the DART won't detect that the bit gets
87	 * set. If so, clear it and set it again.
88	 */
89
90	limit = 0;
91
92	inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
93retry:
94	l = 0;
95	reg = DART_IN(DART_CNTL);
96	reg |= inv_bit;
97	DART_OUT(DART_CNTL, reg);
98
99	while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
100		l++;
101	if (l == (1L << limit)) {
102		if (limit < 4) {
103			limit++;
104			reg = DART_IN(DART_CNTL);
105			reg &= ~inv_bit;
106			DART_OUT(DART_CNTL, reg);
107			goto retry;
108		} else
109			panic("DART: TLB did not flush after waiting a long "
110			      "time. Buggy U3 ?");
111	}
112}
113
114static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
115{
116	unsigned int reg;
117	unsigned int l, limit;
118
119	reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
120		(bus_rpn & DART_CNTL_U4_IONE_MASK);
121	DART_OUT(DART_CNTL, reg);
122
123	limit = 0;
124wait_more:
125	l = 0;
126	while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
127		rmb();
128		l++;
129	}
130
131	if (l == (1L << limit)) {
132		if (limit < 4) {
133			limit++;
134			goto wait_more;
135		} else
136			panic("DART: TLB did not flush after waiting a long "
137			      "time. Buggy U4 ?");
138	}
139}
140
141static void dart_flush(struct iommu_table *tbl)
142{
143	mb();
144	if (dart_dirty) {
145		dart_tlb_invalidate_all();
146		dart_dirty = 0;
147	}
148}
149
150static void dart_build(struct iommu_table *tbl, long index,
151		       long npages, unsigned long uaddr,
152		       enum dma_data_direction direction)
153{
154	unsigned int *dp;
155	unsigned int rpn;
156	long l;
157
158	DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
159
160	dp = ((unsigned int*)tbl->it_base) + index;
161
162	/* On U3, all memory is contigous, so we can move this
163	 * out of the loop.
164	 */
165	l = npages;
166	while (l--) {
167		rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT;
168
169		*(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
170
171		uaddr += DART_PAGE_SIZE;
172	}
173
174	/* make sure all updates have reached memory */
175	mb();
176	in_be32((unsigned __iomem *)dp);
177	mb();
178
179	if (dart_is_u4) {
180		rpn = index;
181		while (npages--)
182			dart_tlb_invalidate_one(rpn++);
183	} else {
184		dart_dirty = 1;
185	}
186}
187
188
189static void dart_free(struct iommu_table *tbl, long index, long npages)
190{
191	unsigned int *dp;
192
193	/* We don't worry about flushing the TLB cache. The only drawback of
194	 * not doing it is that we won't catch buggy device drivers doing
195	 * bad DMAs, but then no 32-bit architecture ever does either.
196	 */
197
198	DBG("dart: free at: %lx, %lx\n", index, npages);
199
200	dp  = ((unsigned int *)tbl->it_base) + index;
201
202	while (npages--)
203		*(dp++) = dart_emptyval;
204}
205
206
207static int dart_init(struct device_node *dart_node)
208{
209	unsigned int i;
210	unsigned long tmp, base, size;
211	struct resource r;
212
213	if (dart_tablebase == 0 || dart_tablesize == 0) {
214		printk(KERN_INFO "DART: table not allocated, using "
215		       "direct DMA\n");
216		return -ENODEV;
217	}
218
219	if (of_address_to_resource(dart_node, 0, &r))
220		panic("DART: can't get register base ! ");
221
222	/* Make sure nothing from the DART range remains in the CPU cache
223	 * from a previous mapping that existed before the kernel took
224	 * over
225	 */
226	flush_dcache_phys_range(dart_tablebase,
227				dart_tablebase + dart_tablesize);
228
229	tmp = lmb_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
230	dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
231					 DARTMAP_RPNMASK);
232
233	/* Map in DART registers */
234	dart = ioremap(r.start, r.end - r.start + 1);
235	if (dart == NULL)
236		panic("DART: Cannot map registers!");
237
238	/* Map in DART table */
239	dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize);
240
241	/* Fill initial table */
242	for (i = 0; i < dart_tablesize/4; i++)
243		dart_vbase[i] = dart_emptyval;
244
245	/* Initialize DART with table base and enable it. */
246	base = dart_tablebase >> DART_PAGE_SHIFT;
247	size = dart_tablesize >> DART_PAGE_SHIFT;
248	if (dart_is_u4) {
249		size &= DART_SIZE_U4_SIZE_MASK;
250		DART_OUT(DART_BASE_U4, base);
251		DART_OUT(DART_SIZE_U4, size);
252		DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
253	} else {
254		size &= DART_CNTL_U3_SIZE_MASK;
255		DART_OUT(DART_CNTL,
256			 DART_CNTL_U3_ENABLE |
257			 (base << DART_CNTL_U3_BASE_SHIFT) |
258			 (size << DART_CNTL_U3_SIZE_SHIFT));
259	}
260
261	/* Invalidate DART to get rid of possible stale TLBs */
262	dart_tlb_invalidate_all();
263
264	printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
265	       dart_is_u4 ? "U4" : "U3");
266
267	return 0;
268}
269
270static void iommu_table_dart_setup(void)
271{
272	iommu_table_dart.it_busno = 0;
273	iommu_table_dart.it_offset = 0;
274	/* it_size is in number of entries */
275	iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
276
277	/* Initialize the common IOMMU code */
278	iommu_table_dart.it_base = (unsigned long)dart_vbase;
279	iommu_table_dart.it_index = 0;
280	iommu_table_dart.it_blocksize = 1;
281	iommu_init_table(&iommu_table_dart, -1);
282
283	/* Reserve the last page of the DART to avoid possible prefetch
284	 * past the DART mapped area
285	 */
286	set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
287}
288
289static void pci_dma_dev_setup_dart(struct pci_dev *dev)
290{
291	/* We only have one iommu table on the mac for now, which makes
292	 * things simple. Setup all PCI devices to point to this table
293	 */
294	dev->dev.archdata.dma_data = &iommu_table_dart;
295}
296
297static void pci_dma_bus_setup_dart(struct pci_bus *bus)
298{
299	struct device_node *dn;
300
301	if (!iommu_table_dart_inited) {
302		iommu_table_dart_inited = 1;
303		iommu_table_dart_setup();
304	}
305
306	dn = pci_bus_to_OF_node(bus);
307
308	if (dn)
309		PCI_DN(dn)->iommu_table = &iommu_table_dart;
310}
311
312void iommu_init_early_dart(void)
313{
314	struct device_node *dn;
315
316	/* Find the DART in the device-tree */
317	dn = of_find_compatible_node(NULL, "dart", "u3-dart");
318	if (dn == NULL) {
319		dn = of_find_compatible_node(NULL, "dart", "u4-dart");
320		if (dn == NULL)
321			goto bail;
322		dart_is_u4 = 1;
323	}
324
325	/* Setup low level TCE operations for the core IOMMU code */
326	ppc_md.tce_build = dart_build;
327	ppc_md.tce_free  = dart_free;
328	ppc_md.tce_flush = dart_flush;
329
330	/* Initialize the DART HW */
331	if (dart_init(dn) == 0) {
332		ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_dart;
333		ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_dart;
334
335		/* Setup pci_dma ops */
336		set_pci_dma_ops(&dma_iommu_ops);
337		return;
338	}
339
340 bail:
341	/* If init failed, use direct iommu and null setup functions */
342	ppc_md.pci_dma_dev_setup = NULL;
343	ppc_md.pci_dma_bus_setup = NULL;
344
345	/* Setup pci_dma ops */
346	set_pci_dma_ops(&dma_direct_ops);
347}
348
349#ifdef CONFIG_PM
350static void iommu_dart_save(void)
351{
352	memcpy(dart_copy, dart_vbase, 2*1024*1024);
353}
354
355static void iommu_dart_restore(void)
356{
357	memcpy(dart_vbase, dart_copy, 2*1024*1024);
358	dart_tlb_invalidate_all();
359}
360
361static int __init iommu_init_late_dart(void)
362{
363	unsigned long tbasepfn;
364	struct page *p;
365
366	/* if no dart table exists then we won't need to save it
367	 * and the area has also not been reserved */
368	if (!dart_tablebase)
369		return 0;
370
371	tbasepfn = __pa(dart_tablebase) >> PAGE_SHIFT;
372	register_nosave_region_late(tbasepfn,
373				    tbasepfn + ((1<<24) >> PAGE_SHIFT));
374
375	/* For suspend we need to copy the dart contents because
376	 * it is not part of the regular mapping (see above) and
377	 * thus not saved automatically. The memory for this copy
378	 * must be allocated early because we need 2 MB. */
379	p = alloc_pages(GFP_KERNEL, 21 - PAGE_SHIFT);
380	BUG_ON(!p);
381	dart_copy = page_address(p);
382
383	ppc_md.iommu_save = iommu_dart_save;
384	ppc_md.iommu_restore = iommu_dart_restore;
385
386	return 0;
387}
388
389late_initcall(iommu_init_late_dart);
390#endif
391
392void __init alloc_dart_table(void)
393{
394	/* Only reserve DART space if machine has more than 1GB of RAM
395	 * or if requested with iommu=on on cmdline.
396	 *
397	 * 1GB of RAM is picked as limit because some default devices
398	 * (i.e. Airport Extreme) have 30 bit address range limits.
399	 */
400
401	if (iommu_is_off)
402		return;
403
404	if (!iommu_force_on && lmb_end_of_DRAM() <= 0x40000000ull)
405		return;
406
407	/* 512 pages (2MB) is max DART tablesize. */
408	dart_tablesize = 1UL << 21;
409	/* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
410	 * will blow up an entire large page anyway in the kernel mapping
411	 */
412	dart_tablebase = (unsigned long)
413		abs_to_virt(lmb_alloc_base(1UL<<24, 1UL<<24, 0x80000000L));
414
415	printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase);
416}
417