1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License.  See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved.
7 */
8
9#include <linux/types.h>
10#include <linux/pci.h>
11#include <asm/sn/addrs.h>
12#include <asm/sn/geo.h>
13#include <asm/sn/pcibr_provider.h>
14#include <asm/sn/pcibus_provider_defs.h>
15#include <asm/sn/pcidev.h>
16#include <asm/sn/pic.h>
17#include <asm/sn/sn_sal.h>
18#include <asm/sn/tiocp.h>
19#include "tio.h"
20#include "xtalk/xwidgetdev.h"
21#include "xtalk/hubdev.h"
22
23extern int sn_ioif_inited;
24
25/* =====================================================================
26 *    DMA MANAGEMENT
27 *
28 *      The Bridge ASIC provides three methods of doing DMA: via a "direct map"
29 *      register available in 32-bit PCI space (which selects a contiguous 2G
30 *	address space on some other widget), via "direct" addressing via 64-bit
31 *      PCI space (all destination information comes from the PCI address,
32 *      including transfer attributes), and via a "mapped" region that allows
33 *      a bunch of different small mappings to be established with the PMU.
34 *
35 *      For efficiency, we most prefer to use the 32bit direct mapping facility,
36 *      since it requires no resource allocations. The advantage of using the
37 *      PMU over the 64-bit direct is that single-cycle PCI addressing can be
38 *      used; the advantage of using 64-bit direct over PMU addressing is that
39 *      we do not have to allocate entries in the PMU.
40 */
41
42static dma_addr_t
43pcibr_dmamap_ate32(struct pcidev_info *info,
44		   u64 paddr, size_t req_size, u64 flags, int dma_flags)
45{
46
47	struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info;
48	struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info->
49	    pdi_pcibus_info;
50	u8 internal_device = (PCI_SLOT(pcidev_info->pdi_host_pcidev_info->
51					    pdi_linux_pcidev->devfn)) - 1;
52	int ate_count;
53	int ate_index;
54	u64 ate_flags = flags | PCI32_ATE_V;
55	u64 ate;
56	u64 pci_addr;
57	u64 xio_addr;
58	u64 offset;
59
60	/* PIC in PCI-X mode does not supports 32bit PageMap mode */
61	if (IS_PIC_SOFT(pcibus_info) && IS_PCIX(pcibus_info)) {
62		return 0;
63	}
64
65	/* Calculate the number of ATEs needed. */
66	if (!(MINIMAL_ATE_FLAG(paddr, req_size))) {
67		ate_count = IOPG((IOPGSIZE - 1)	/* worst case start offset */
68				 +req_size	/* max mapping bytes */
69				 - 1) + 1;	/* round UP */
70	} else {		/* assume requested target is page aligned */
71		ate_count = IOPG(req_size	/* max mapping bytes */
72				 - 1) + 1;	/* round UP */
73	}
74
75	/* Get the number of ATEs required. */
76	ate_index = pcibr_ate_alloc(pcibus_info, ate_count);
77	if (ate_index < 0)
78		return 0;
79
80	/* In PCI-X mode, Prefetch not supported */
81	if (IS_PCIX(pcibus_info))
82		ate_flags &= ~(PCI32_ATE_PREF);
83
84	if (SN_DMA_ADDRTYPE(dma_flags == SN_DMA_ADDR_PHYS))
85		xio_addr = IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) :
86	    					      PHYS_TO_TIODMA(paddr);
87	else
88		xio_addr = paddr;
89
90	offset = IOPGOFF(xio_addr);
91	ate = ate_flags | (xio_addr - offset);
92
93	/* If PIC, put the targetid in the ATE */
94	if (IS_PIC_SOFT(pcibus_info)) {
95		ate |= (pcibus_info->pbi_hub_xid << PIC_ATE_TARGETID_SHFT);
96	}
97
98	/*
99	 * If we're mapping for MSI, set the MSI bit in the ATE.  If it's a
100	 * TIOCP based pci bus, we also need to set the PIO bit in the ATE.
101	 */
102	if (dma_flags & SN_DMA_MSI) {
103		ate |= PCI32_ATE_MSI;
104		if (IS_TIOCP_SOFT(pcibus_info))
105			ate |= PCI32_ATE_PIO;
106	}
107
108	ate_write(pcibus_info, ate_index, ate_count, ate);
109
110	/*
111	 * Set up the DMA mapped Address.
112	 */
113	pci_addr = PCI32_MAPPED_BASE + offset + IOPGSIZE * ate_index;
114
115	/*
116	 * If swap was set in device in pcibr_endian_set()
117	 * we need to turn swapping on.
118	 */
119	if (pcibus_info->pbi_devreg[internal_device] & PCIBR_DEV_SWAP_DIR)
120		ATE_SWAP_ON(pci_addr);
121
122
123	return pci_addr;
124}
125
126static dma_addr_t
127pcibr_dmatrans_direct64(struct pcidev_info * info, u64 paddr,
128			u64 dma_attributes, int dma_flags)
129{
130	struct pcibus_info *pcibus_info = (struct pcibus_info *)
131	    ((info->pdi_host_pcidev_info)->pdi_pcibus_info);
132	u64 pci_addr;
133
134	/* Translate to Crosstalk View of Physical Address */
135	if (SN_DMA_ADDRTYPE(dma_flags) == SN_DMA_ADDR_PHYS)
136		pci_addr = IS_PIC_SOFT(pcibus_info) ?
137				PHYS_TO_DMA(paddr) :
138		    		PHYS_TO_TIODMA(paddr) | dma_attributes;
139	else
140		pci_addr = IS_PIC_SOFT(pcibus_info) ?
141				paddr :
142				paddr | dma_attributes;
143
144	/* Handle Bus mode */
145	if (IS_PCIX(pcibus_info))
146		pci_addr &= ~PCI64_ATTR_PREF;
147
148	/* Handle Bridge Chipset differences */
149	if (IS_PIC_SOFT(pcibus_info)) {
150		pci_addr |=
151		    ((u64) pcibus_info->
152		     pbi_hub_xid << PIC_PCI64_ATTR_TARG_SHFT);
153	} else
154		pci_addr |= (dma_flags & SN_DMA_MSI) ?
155				TIOCP_PCI64_CMDTYPE_MSI :
156				TIOCP_PCI64_CMDTYPE_MEM;
157
158	/* If PCI mode, func zero uses VCHAN0, every other func uses VCHAN1 */
159	if (!IS_PCIX(pcibus_info) && PCI_FUNC(info->pdi_linux_pcidev->devfn))
160		pci_addr |= PCI64_ATTR_VIRTUAL;
161
162	return pci_addr;
163}
164
165static dma_addr_t
166pcibr_dmatrans_direct32(struct pcidev_info * info,
167			u64 paddr, size_t req_size, u64 flags, int dma_flags)
168{
169	struct pcidev_info *pcidev_info = info->pdi_host_pcidev_info;
170	struct pcibus_info *pcibus_info = (struct pcibus_info *)pcidev_info->
171	    pdi_pcibus_info;
172	u64 xio_addr;
173
174	u64 xio_base;
175	u64 offset;
176	u64 endoff;
177
178	if (IS_PCIX(pcibus_info)) {
179		return 0;
180	}
181
182	if (dma_flags & SN_DMA_MSI)
183		return 0;
184
185	if (SN_DMA_ADDRTYPE(dma_flags) == SN_DMA_ADDR_PHYS)
186		xio_addr = IS_PIC_SOFT(pcibus_info) ? PHYS_TO_DMA(paddr) :
187	    					      PHYS_TO_TIODMA(paddr);
188	else
189		xio_addr = paddr;
190
191	xio_base = pcibus_info->pbi_dir_xbase;
192	offset = xio_addr - xio_base;
193	endoff = req_size + offset;
194	if ((req_size > (1ULL << 31)) ||	/* Too Big */
195	    (xio_addr < xio_base) ||	/* Out of range for mappings */
196	    (endoff > (1ULL << 31))) {	/* Too Big */
197		return 0;
198	}
199
200	return PCI32_DIRECT_BASE | offset;
201}
202
203/*
204 * Wrapper routine for freeing DMA maps
205 * DMA mappings for Direct 64 and 32 do not have any DMA maps.
206 */
207void
208pcibr_dma_unmap(struct pci_dev *hwdev, dma_addr_t dma_handle, int direction)
209{
210	struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev);
211	struct pcibus_info *pcibus_info =
212	    (struct pcibus_info *)pcidev_info->pdi_pcibus_info;
213
214	if (IS_PCI32_MAPPED(dma_handle)) {
215		int ate_index;
216
217		ate_index =
218		    IOPG((ATE_SWAP_OFF(dma_handle) - PCI32_MAPPED_BASE));
219		pcibr_ate_free(pcibus_info, ate_index);
220	}
221}
222
223/*
224 * On SN systems there is a race condition between a PIO read response and
225 * DMA's.  In rare cases, the read response may beat the DMA, causing the
226 * driver to think that data in memory is complete and meaningful.  This code
227 * eliminates that race.  This routine is called by the PIO read routines
228 * after doing the read.  For PIC this routine then forces a fake interrupt
229 * on another line, which is logically associated with the slot that the PIO
230 * is addressed to.  It then spins while watching the memory location that
231 * the interrupt is targetted to.  When the interrupt response arrives, we
232 * are sure that the DMA has landed in memory and it is safe for the driver
233 * to proceed.	For TIOCP use the Device(x) Write Request Buffer Flush
234 * Bridge register since it ensures the data has entered the coherence domain,
235 * unlike the PIC Device(x) Write Request Buffer Flush register.
236 */
237
238void sn_dma_flush(u64 addr)
239{
240	nasid_t nasid;
241	int is_tio;
242	int wid_num;
243	int i, j;
244	unsigned long flags;
245	u64 itte;
246	struct hubdev_info *hubinfo;
247	struct sn_flush_device_kernel *p;
248	struct sn_flush_device_common *common;
249	struct sn_flush_nasid_entry *flush_nasid_list;
250
251	if (!sn_ioif_inited)
252		return;
253
254	nasid = NASID_GET(addr);
255	if (-1 == nasid_to_cnodeid(nasid))
256		return;
257
258	hubinfo = (NODEPDA(nasid_to_cnodeid(nasid)))->pdinfo;
259
260	if (!hubinfo) {
261		BUG();
262	}
263
264	flush_nasid_list = &hubinfo->hdi_flush_nasid_list;
265	if (flush_nasid_list->widget_p == NULL)
266		return;
267
268	is_tio = (nasid & 1);
269	if (is_tio) {
270		int itte_index;
271
272		if (TIO_HWIN(addr))
273			itte_index = 0;
274		else if (TIO_BWIN_WINDOWNUM(addr))
275			itte_index = TIO_BWIN_WINDOWNUM(addr);
276		else
277			itte_index = -1;
278
279		if (itte_index >= 0) {
280			itte = flush_nasid_list->iio_itte[itte_index];
281			if (! TIO_ITTE_VALID(itte))
282				return;
283			wid_num = TIO_ITTE_WIDGET(itte);
284		} else
285			wid_num = TIO_SWIN_WIDGETNUM(addr);
286	} else {
287		if (BWIN_WINDOWNUM(addr)) {
288			itte = flush_nasid_list->iio_itte[BWIN_WINDOWNUM(addr)];
289			wid_num = IIO_ITTE_WIDGET(itte);
290		} else
291			wid_num = SWIN_WIDGETNUM(addr);
292	}
293	if (flush_nasid_list->widget_p[wid_num] == NULL)
294		return;
295	p = &flush_nasid_list->widget_p[wid_num][0];
296
297	/* find a matching BAR */
298	for (i = 0; i < DEV_PER_WIDGET; i++,p++) {
299		common = p->common;
300		for (j = 0; j < PCI_ROM_RESOURCE; j++) {
301			if (common->sfdl_bar_list[j].start == 0)
302				break;
303			if (addr >= common->sfdl_bar_list[j].start
304			    && addr <= common->sfdl_bar_list[j].end)
305				break;
306		}
307		if (j < PCI_ROM_RESOURCE && common->sfdl_bar_list[j].start != 0)
308			break;
309	}
310
311	/* if no matching BAR, return without doing anything. */
312	if (i == DEV_PER_WIDGET)
313		return;
314
315	/*
316	 * For TIOCP use the Device(x) Write Request Buffer Flush Bridge
317	 * register since it ensures the data has entered the coherence
318	 * domain, unlike PIC.
319	 */
320	if (is_tio) {
321		/*
322	 	 * Note:  devices behind TIOCE should never be matched in the
323		 * above code, and so the following code is PIC/CP centric.
324		 * If CE ever needs the sn_dma_flush mechanism, we will have
325		 * to account for that here and in tioce_bus_fixup().
326	 	 */
327		u32 tio_id = HUB_L(TIO_IOSPACE_ADDR(nasid, TIO_NODE_ID));
328		u32 revnum = XWIDGET_PART_REV_NUM(tio_id);
329
330		/* TIOCP BRINGUP WAR (PV907516): Don't write buffer flush reg */
331		if ((1 << XWIDGET_PART_REV_NUM_REV(revnum)) & PV907516) {
332			return;
333		} else {
334			pcireg_wrb_flush_get(common->sfdl_pcibus_info,
335					     (common->sfdl_slot - 1));
336		}
337	} else {
338		spin_lock_irqsave(&p->sfdl_flush_lock, flags);
339		*common->sfdl_flush_addr = 0;
340
341		/* force an interrupt. */
342		*(volatile u32 *)(common->sfdl_force_int_addr) = 1;
343
344		/* wait for the interrupt to come back. */
345		while (*(common->sfdl_flush_addr) != 0x10f)
346			cpu_relax();
347
348		/* okay, everything is synched up. */
349		spin_unlock_irqrestore(&p->sfdl_flush_lock, flags);
350	}
351	return;
352}
353
354/*
355 * DMA interfaces.  Called from pci_dma.c routines.
356 */
357
358dma_addr_t
359pcibr_dma_map(struct pci_dev * hwdev, unsigned long phys_addr, size_t size, int dma_flags)
360{
361	dma_addr_t dma_handle;
362	struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev);
363
364	/* SN cannot support DMA addresses smaller than 32 bits. */
365	if (hwdev->dma_mask < 0x7fffffff) {
366		return 0;
367	}
368
369	if (hwdev->dma_mask == ~0UL) {
370		/*
371		 * Handle the most common case: 64 bit cards.  This
372		 * call should always succeed.
373		 */
374
375		dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr,
376						     PCI64_ATTR_PREF, dma_flags);
377	} else {
378		/* Handle 32-63 bit cards via direct mapping */
379		dma_handle = pcibr_dmatrans_direct32(pcidev_info, phys_addr,
380						     size, 0, dma_flags);
381		if (!dma_handle) {
382			/*
383			 * It is a 32 bit card and we cannot do direct mapping,
384			 * so we use an ATE.
385			 */
386
387			dma_handle = pcibr_dmamap_ate32(pcidev_info, phys_addr,
388							size, PCI32_ATE_PREF,
389							dma_flags);
390		}
391	}
392
393	return dma_handle;
394}
395
396dma_addr_t
397pcibr_dma_map_consistent(struct pci_dev * hwdev, unsigned long phys_addr,
398			 size_t size, int dma_flags)
399{
400	dma_addr_t dma_handle;
401	struct pcidev_info *pcidev_info = SN_PCIDEV_INFO(hwdev);
402
403	if (hwdev->dev.coherent_dma_mask == ~0UL) {
404		dma_handle = pcibr_dmatrans_direct64(pcidev_info, phys_addr,
405					    PCI64_ATTR_BAR, dma_flags);
406	} else {
407		dma_handle = (dma_addr_t) pcibr_dmamap_ate32(pcidev_info,
408						    phys_addr, size,
409						    PCI32_ATE_BAR, dma_flags);
410	}
411
412	return dma_handle;
413}
414
415EXPORT_SYMBOL(sn_dma_flush);
416