1/* $OpenBSD: cia_dma.c,v 1.11 2014/06/14 23:11:20 jmatthew Exp $ */
2/* $NetBSD: cia_dma.c,v 1.16 2000/06/29 08:58:46 mrg Exp $ */
3
4/*-
5 * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10 * NASA Ames Research Center.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 *    notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 *    notice, this list of conditions and the following disclaimer in the
19 *    documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34/*
35 * XXX - We should define this before including bus.h, but since other stuff
36 *       pulls in bus.h we must do this here.
37 */
38#define _ALPHA_BUS_DMA_PRIVATE
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/device.h>
44#include <sys/malloc.h>
45
46#include <uvm/uvm_extern.h>
47
48#include <machine/bus.h>
49
50#include <dev/pci/pcireg.h>
51#include <dev/pci/pcivar.h>
52#include <alpha/pci/ciareg.h>
53#include <alpha/pci/ciavar.h>
54
55bus_dma_tag_t cia_dma_get_tag(bus_dma_tag_t, alpha_bus_t);
56
57int	cia_bus_dmamap_create_direct(bus_dma_tag_t, bus_size_t, int,
58	    bus_size_t, bus_size_t, int, bus_dmamap_t *);
59void	cia_bus_dmamap_destroy_direct(bus_dma_tag_t, bus_dmamap_t);
60
61int	cia_bus_dmamap_load_sgmap(bus_dma_tag_t, bus_dmamap_t, void *,
62	    bus_size_t, struct proc *, int);
63
64int	cia_bus_dmamap_load_mbuf_sgmap(bus_dma_tag_t, bus_dmamap_t,
65	    struct mbuf *, int);
66
67int	cia_bus_dmamap_load_uio_sgmap(bus_dma_tag_t, bus_dmamap_t,
68	    struct uio *, int);
69
70int	cia_bus_dmamap_load_raw_sgmap(bus_dma_tag_t, bus_dmamap_t,
71	    bus_dma_segment_t *, int, bus_size_t, int);
72
73void	cia_bus_dmamap_unload_sgmap(bus_dma_tag_t, bus_dmamap_t);
74
75/*
76 * Direct-mapped window: 1G at 1G
77 */
78#define	CIA_DIRECT_MAPPED_BASE	(1*1024*1024*1024)
79#define	CIA_DIRECT_MAPPED_SIZE	(1*1024*1024*1024)
80
81/*
82 * SGMAP window: 8M at 8M
83 */
84#define	CIA_SGMAP_MAPPED_BASE	(8*1024*1024)
85#define	CIA_SGMAP_MAPPED_SIZE	(8*1024*1024)
86
87void	cia_tlb_invalidate(void);
88void	cia_broken_pyxis_tlb_invalidate(void);
89
90void	(*cia_tlb_invalidate_fn)(void);
91
92#define	CIA_TLB_INVALIDATE()	(*cia_tlb_invalidate_fn)()
93
94struct alpha_sgmap cia_pyxis_bug_sgmap;
95#define	CIA_PYXIS_BUG_BASE	(128*1024*1024)
96#define	CIA_PYXIS_BUG_SIZE	(2*1024*1024)
97
98void
99cia_dma_init(ccp)
100	struct cia_config *ccp;
101{
102	bus_addr_t tbase;
103	bus_dma_tag_t t;
104
105	/*
106	 * Initialize the DMA tag used for direct-mapped DMA.
107	 */
108	t = &ccp->cc_dmat_direct;
109	t->_cookie = ccp;
110	t->_wbase = CIA_DIRECT_MAPPED_BASE;
111	t->_wsize = CIA_DIRECT_MAPPED_SIZE;
112	t->_next_window = &ccp->cc_dmat_sgmap;
113	t->_boundary = 0;
114	t->_sgmap = NULL;
115	t->_get_tag = cia_dma_get_tag;
116	t->_dmamap_create = cia_bus_dmamap_create_direct;
117	t->_dmamap_destroy = cia_bus_dmamap_destroy_direct;
118	t->_dmamap_load = _bus_dmamap_load_direct;
119	t->_dmamap_load_mbuf = _bus_dmamap_load_mbuf_direct;
120	t->_dmamap_load_uio = _bus_dmamap_load_uio_direct;
121	t->_dmamap_load_raw = _bus_dmamap_load_raw_direct;
122	t->_dmamap_unload = _bus_dmamap_unload;
123	t->_dmamap_sync = _bus_dmamap_sync;
124
125	t->_dmamem_alloc = _bus_dmamem_alloc;
126	t->_dmamem_free = _bus_dmamem_free;
127	t->_dmamem_map = _bus_dmamem_map;
128	t->_dmamem_unmap = _bus_dmamem_unmap;
129	t->_dmamem_mmap = _bus_dmamem_mmap;
130
131	/*
132	 * Initialize the DMA tag used for sgmap-mapped DMA.
133	 */
134	t = &ccp->cc_dmat_sgmap;
135	t->_cookie = ccp;
136	t->_wbase = CIA_SGMAP_MAPPED_BASE;
137	t->_wsize = CIA_SGMAP_MAPPED_SIZE;
138	t->_next_window = NULL;
139	t->_boundary = 0;
140	t->_sgmap = &ccp->cc_sgmap;
141	t->_get_tag = cia_dma_get_tag;
142	t->_dmamap_create = alpha_sgmap_dmamap_create;
143	t->_dmamap_destroy = alpha_sgmap_dmamap_destroy;
144	t->_dmamap_load = cia_bus_dmamap_load_sgmap;
145	t->_dmamap_load_mbuf = cia_bus_dmamap_load_mbuf_sgmap;
146	t->_dmamap_load_uio = cia_bus_dmamap_load_uio_sgmap;
147	t->_dmamap_load_raw = cia_bus_dmamap_load_raw_sgmap;
148	t->_dmamap_unload = cia_bus_dmamap_unload_sgmap;
149	t->_dmamap_sync = _bus_dmamap_sync;
150
151	t->_dmamem_alloc = _bus_dmamem_alloc;
152	t->_dmamem_free = _bus_dmamem_free;
153	t->_dmamem_map = _bus_dmamem_map;
154	t->_dmamem_unmap = _bus_dmamem_unmap;
155	t->_dmamem_mmap = _bus_dmamem_mmap;
156
157	/*
158	 * The firmware has set up window 1 as a 1G direct-mapped DMA
159	 * window beginning at 1G.  We leave it alone.  Leave window
160	 * 0 alone until we reconfigure it for SGMAP-mapped DMA.
161	 * Windows 2 and 3 are already disabled.
162	 */
163
164	/*
165	 * Initialize the SGMAP.  Must align page table to 32k
166	 * (hardware bug?).
167	 */
168	alpha_sgmap_init(t, &ccp->cc_sgmap, "cia_sgmap",
169	    CIA_SGMAP_MAPPED_BASE, 0, CIA_SGMAP_MAPPED_SIZE,
170	    sizeof(u_int64_t), NULL, (32*1024));
171
172	/*
173	 * Set up window 0 as an 8MB SGMAP-mapped window
174	 * starting at 8MB.
175	 */
176	REGVAL(CIA_PCI_W0BASE) = CIA_SGMAP_MAPPED_BASE |
177	    CIA_PCI_WnBASE_SG_EN | CIA_PCI_WnBASE_W_EN;
178	alpha_mb();
179
180	REGVAL(CIA_PCI_W0MASK) = CIA_PCI_WnMASK_8M;
181	alpha_mb();
182
183	tbase = ccp->cc_sgmap.aps_ptpa >> CIA_PCI_TnBASE_SHIFT;
184	if ((tbase & CIA_PCI_TnBASE_MASK) != tbase)
185		panic("cia_dma_init: bad page table address");
186	REGVAL(CIA_PCI_T0BASE) = tbase;
187	alpha_mb();
188
189	/*
190	 * Pass 1 and 2 (i.e. revision <= 1) of the Pyxis have a
191	 * broken scatter/gather TLB; it cannot be invalidated.  To
192	 * work around this problem, we configure window 2 as an SG
193	 * 2M window at 128M, which we use in DMA loopback mode to
194	 * read a spill page.  This works by causing TLB misses,
195	 * causing the old entries to be purged to make room for
196	 * the new entries coming in for the spill page.
197	 */
198	if ((ccp->cc_flags & CCF_ISPYXIS) != 0 && ccp->cc_rev <= 1) {
199		u_int64_t *page_table;
200		int i;
201
202		cia_tlb_invalidate_fn =
203		    cia_broken_pyxis_tlb_invalidate;
204
205		alpha_sgmap_init(t, &cia_pyxis_bug_sgmap,
206		    "pyxis_bug_sgmap", CIA_PYXIS_BUG_BASE, 0,
207		    CIA_PYXIS_BUG_SIZE, sizeof(u_int64_t), NULL,
208		    (32*1024));
209
210		REGVAL(CIA_PCI_W2BASE) = CIA_PYXIS_BUG_BASE |
211		    CIA_PCI_WnBASE_SG_EN | CIA_PCI_WnBASE_W_EN;
212		alpha_mb();
213
214		REGVAL(CIA_PCI_W2MASK) = CIA_PCI_WnMASK_2M;
215		alpha_mb();
216
217		tbase = cia_pyxis_bug_sgmap.aps_ptpa >>
218		    CIA_PCI_TnBASE_SHIFT;
219		if ((tbase & CIA_PCI_TnBASE_MASK) != tbase)
220			panic("cia_dma_init: bad page table address");
221		REGVAL(CIA_PCI_T2BASE) = tbase;
222		alpha_mb();
223
224		/*
225		 * Initialize the page table to point at the spill
226		 * page.  Leave the last entry invalid.
227		 */
228		pci_sgmap_pte64_init_spill_page_pte();
229		for (i = 0, page_table = cia_pyxis_bug_sgmap.aps_pt;
230		     i < (CIA_PYXIS_BUG_SIZE / PAGE_SIZE) - 1; i++) {
231			page_table[i] =
232			    pci_sgmap_pte64_prefetch_spill_page_pte;
233		}
234		alpha_mb();
235	} else
236		cia_tlb_invalidate_fn = cia_tlb_invalidate;
237
238	CIA_TLB_INVALIDATE();
239
240	/* XXX XXX BEGIN XXX XXX */
241	{							/* XXX */
242		extern paddr_t alpha_XXX_dmamap_or;		/* XXX */
243		alpha_XXX_dmamap_or = CIA_DIRECT_MAPPED_BASE;	/* XXX */
244	}							/* XXX */
245	/* XXX XXX END XXX XXX */
246}
247
248/*
249 * Return the bus dma tag to be used for the specified bus type.
250 * INTERNAL USE ONLY!
251 */
252bus_dma_tag_t
253cia_dma_get_tag(t, bustype)
254	bus_dma_tag_t t;
255	alpha_bus_t bustype;
256{
257	struct cia_config *ccp = t->_cookie;
258
259	switch (bustype) {
260	case ALPHA_BUS_PCI:
261	case ALPHA_BUS_EISA:
262		/*
263		 * Systems with a CIA can only support 1G
264		 * of memory, so we use the direct-mapped window
265		 * on busses that have 32-bit DMA.
266		 *
267		 * Ahem:  I have a PWS 500au with 1.5G of memory, and it
268		 * had problems doing DMA because it was not falling back
269		 * to using SGMAPs.  I've fixed that and my PWS now works with
270		 * 1.5G.  There have been other reports about failures with
271		 * more than 1.0G of memory.  Michael Hitch
272		 */
273		return (&ccp->cc_dmat_direct);
274
275	case ALPHA_BUS_ISA:
276		/*
277		 * ISA doesn't have enough address bits to use
278		 * the direct-mapped DMA window, so we must use
279		 * SGMAPs.
280		 */
281		return (&ccp->cc_dmat_sgmap);
282
283	default:
284		panic("cia_dma_get_tag: shouldn't be here, really...");
285	}
286}
287
288/*
289 * Create a CIA direct-mapped DMA map.
290 */
291int
292cia_bus_dmamap_create_direct(t, size, nsegments, maxsegsz, boundary,
293    flags, dmamp)
294	bus_dma_tag_t t;
295	bus_size_t size;
296	int nsegments;
297	bus_size_t maxsegsz;
298	bus_size_t boundary;
299	int flags;
300	bus_dmamap_t *dmamp;
301{
302	struct cia_config *ccp = t->_cookie;
303	bus_dmamap_t map;
304	int error;
305
306	error = _bus_dmamap_create(t, size, nsegments, maxsegsz,
307	    boundary, flags, dmamp);
308	if (error)
309		return (error);
310
311	/*
312	 * Since we fall back to sgmap if the direct mapping fails,
313	 * we need to set up for sgmap in any case.
314	 */
315	map = *dmamp;
316	if (alpha_sgmap_dmamap_setup(map, nsegments, flags)) {
317		_bus_dmamap_destroy(t, map);
318		return (ENOMEM);
319	}
320
321	if ((ccp->cc_flags & CCF_PYXISBUG) != 0 &&
322	    map->_dm_segcnt > 1) {
323		/*
324		 * We have a Pyxis with the DMA page crossing bug, make
325		 * sure we don't coalesce adjacent DMA segments.
326		 *
327		 * NOTE: We can only do this if the max segment count
328		 * is greater than 1.  This is because many network
329		 * drivers allocate large contiguous blocks of memory
330		 * for control data structures, even though they won't
331		 * do any single DMA that crosses a page boundary.
332		 *	-- thorpej@netbsd.org, 2/5/2000
333		 */
334		map->_dm_flags |= DMAMAP_NO_COALESCE;
335	}
336
337	return (0);
338}
339
340/*
341 * Destroy a CIA direct-mapped DMA map.
342 */
343void
344cia_bus_dmamap_destroy_direct(t, map)
345	bus_dma_tag_t t;
346	bus_dmamap_t map;
347{
348	alpha_sgmap_dmamap_teardown(map);
349	_bus_dmamap_destroy(t, map);
350}
351
352/*
353 * Load a CIA SGMAP-mapped DMA map with a linear buffer.
354 */
355int
356cia_bus_dmamap_load_sgmap(t, map, buf, buflen, p, flags)
357	bus_dma_tag_t t;
358	bus_dmamap_t map;
359	void *buf;
360	bus_size_t buflen;
361	struct proc *p;
362	int flags;
363{
364	int error;
365
366	error = pci_sgmap_pte64_load(t, map, buf, buflen, p, flags,
367	    t->_sgmap);
368	if (error == 0)
369		CIA_TLB_INVALIDATE();
370
371	return (error);
372}
373
374/*
375 * Load a CIA SGMAP-mapped DMA map with an mbuf chain.
376 */
377int
378cia_bus_dmamap_load_mbuf_sgmap(t, map, m, flags)
379	bus_dma_tag_t t;
380	bus_dmamap_t map;
381	struct mbuf *m;
382	int flags;
383{
384	int error;
385
386	error = pci_sgmap_pte64_load_mbuf(t, map, m, flags, t->_sgmap);
387	if (error == 0)
388		CIA_TLB_INVALIDATE();
389
390	return (error);
391}
392
393/*
394 * Load a CIA SGMAP-mapped DMA map with a uio.
395 */
396int
397cia_bus_dmamap_load_uio_sgmap(t, map, uio, flags)
398	bus_dma_tag_t t;
399	bus_dmamap_t map;
400	struct uio *uio;
401	int flags;
402{
403	int error;
404
405	error = pci_sgmap_pte64_load_uio(t, map, uio, flags, t->_sgmap);
406	if (error == 0)
407		CIA_TLB_INVALIDATE();
408
409	return (error);
410}
411
412/*
413 * Load a CIA SGMAP-mapped DMA map with raw memory.
414 */
415int
416cia_bus_dmamap_load_raw_sgmap(t, map, segs, nsegs, size, flags)
417	bus_dma_tag_t t;
418	bus_dmamap_t map;
419	bus_dma_segment_t *segs;
420	int nsegs;
421	bus_size_t size;
422	int flags;
423{
424	int error;
425
426	error = pci_sgmap_pte64_load_raw(t, map, segs, nsegs, size, flags,
427	    t->_sgmap);
428	if (error == 0)
429		CIA_TLB_INVALIDATE();
430
431	return (error);
432}
433
434/*
435 * Unload a CIA DMA map.
436 */
437void
438cia_bus_dmamap_unload_sgmap(t, map)
439	bus_dma_tag_t t;
440	bus_dmamap_t map;
441{
442
443	/*
444	 * Invalidate any SGMAP page table entries used by this
445	 * mapping.
446	 */
447	pci_sgmap_pte64_unload(t, map, t->_sgmap);
448	CIA_TLB_INVALIDATE();
449
450	/*
451	 * Do the generic bits of the unload.
452	 */
453	_bus_dmamap_unload(t, map);
454}
455
456/*
457 * Flush the CIA scatter/gather TLB.
458 */
459void
460cia_tlb_invalidate()
461{
462
463	alpha_mb();
464	REGVAL(CIA_PCI_TBIA) = CIA_PCI_TBIA_ALL;
465	alpha_mb();
466}
467
468/*
469 * Flush the scatter/gather TLB on broken Pyxis chips.
470 */
471void
472cia_broken_pyxis_tlb_invalidate()
473{
474	volatile u_int64_t dummy;
475	u_int32_t ctrl;
476	int i, s;
477
478	s = splhigh();
479
480	/*
481	 * Put the Pyxis into PCI loopback mode.
482	 */
483	alpha_mb();
484	ctrl = REGVAL(CIA_CSR_CTRL);
485	REGVAL(CIA_CSR_CTRL) = ctrl | CTRL_PCI_LOOP_EN;
486	alpha_mb();
487
488	/*
489	 * Now, read from PCI dense memory space at offset 128M (our
490	 * target window base), skipping 64k on each read.  This forces
491	 * S/G TLB misses.
492	 *
493	 * XXX Looks like the TLB entries are `not quite LRU'.  We need
494	 * XXX to read more times than there are actual tags!
495	 */
496	for (i = 0; i < CIA_TLB_NTAGS + 4; i++) {
497		dummy = *((volatile u_int64_t *)
498		    ALPHA_PHYS_TO_K0SEG(CIA_PCI_DENSE + CIA_PYXIS_BUG_BASE +
499		    (i * 65536)));
500	}
501
502	/*
503	 * Restore normal PCI operation.
504	 */
505	alpha_mb();
506	REGVAL(CIA_CSR_CTRL) = ctrl;
507	alpha_mb();
508
509	splx(s);
510}
511