1/*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice unmodified, this list of conditions, and the following
13 *    disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31#ifndef	_LINUX_IO_H_
32#define	_LINUX_IO_H_
33
34#include <sys/endian.h>
35#include <sys/types.h>
36
37#include <machine/vm.h>
38
39#include <linux/compiler.h>
40#include <linux/types.h>
41
42/*
43 * XXX This is all x86 specific.  It should be bus space access.
44 */
45
46
47/* rmb and wmb are declared in machine/atomic.h, so should be included first. */
48#ifndef __io_br
49#define	__io_br()	__compiler_membar()
50#endif
51
52#ifndef __io_ar
53#ifdef rmb
54#define	__io_ar()	rmb()
55#else
56#define	__io_ar()	__compiler_membar()
57#endif
58#endif
59
60#ifndef __io_bw
61#ifdef wmb
62#define	__io_bw()	wmb()
63#else
64#define	__io_bw()	__compiler_membar()
65#endif
66#endif
67
68#ifndef __io_aw
69#define	__io_aw()	__compiler_membar()
70#endif
71
72/* Access MMIO registers atomically without barriers and byte swapping. */
73
74static inline uint8_t
75__raw_readb(const volatile void *addr)
76{
77	return (*(const volatile uint8_t *)addr);
78}
79#define	__raw_readb(addr)	__raw_readb(addr)
80
81static inline void
82__raw_writeb(uint8_t v, volatile void *addr)
83{
84	*(volatile uint8_t *)addr = v;
85}
86#define	__raw_writeb(v, addr)	__raw_writeb(v, addr)
87
88static inline uint16_t
89__raw_readw(const volatile void *addr)
90{
91	return (*(const volatile uint16_t *)addr);
92}
93#define	__raw_readw(addr)	__raw_readw(addr)
94
95static inline void
96__raw_writew(uint16_t v, volatile void *addr)
97{
98	*(volatile uint16_t *)addr = v;
99}
100#define	__raw_writew(v, addr)	__raw_writew(v, addr)
101
102static inline uint32_t
103__raw_readl(const volatile void *addr)
104{
105	return (*(const volatile uint32_t *)addr);
106}
107#define	__raw_readl(addr)	__raw_readl(addr)
108
109static inline void
110__raw_writel(uint32_t v, volatile void *addr)
111{
112	*(volatile uint32_t *)addr = v;
113}
114#define	__raw_writel(v, addr)	__raw_writel(v, addr)
115
116#ifdef __LP64__
117static inline uint64_t
118__raw_readq(const volatile void *addr)
119{
120	return (*(const volatile uint64_t *)addr);
121}
122#define	__raw_readq(addr)	__raw_readq(addr)
123
124static inline void
125__raw_writeq(uint64_t v, volatile void *addr)
126{
127	*(volatile uint64_t *)addr = v;
128}
129#define	__raw_writeq(v, addr)	__raw_writeq(v, addr)
130#endif
131
132#define	mmiowb()	barrier()
133
134/* Access little-endian MMIO registers atomically with memory barriers. */
135
136#undef readb
137static inline uint8_t
138readb(const volatile void *addr)
139{
140	uint8_t v;
141
142	__io_br();
143	v = *(const volatile uint8_t *)addr;
144	__io_ar();
145	return (v);
146}
147#define	readb(addr)		readb(addr)
148
149#undef writeb
150static inline void
151writeb(uint8_t v, volatile void *addr)
152{
153	__io_bw();
154	*(volatile uint8_t *)addr = v;
155	__io_aw();
156}
157#define	writeb(v, addr)		writeb(v, addr)
158
159#undef readw
160static inline uint16_t
161readw(const volatile void *addr)
162{
163	uint16_t v;
164
165	__io_br();
166	v = le16toh(__raw_readw(addr));
167	__io_ar();
168	return (v);
169}
170#define	readw(addr)		readw(addr)
171
172#undef writew
173static inline void
174writew(uint16_t v, volatile void *addr)
175{
176	__io_bw();
177	__raw_writew(htole16(v), addr);
178	__io_aw();
179}
180#define	writew(v, addr)		writew(v, addr)
181
182#undef readl
183static inline uint32_t
184readl(const volatile void *addr)
185{
186	uint32_t v;
187
188	__io_br();
189	v = le32toh(__raw_readl(addr));
190	__io_ar();
191	return (v);
192}
193#define	readl(addr)		readl(addr)
194
195#undef writel
196static inline void
197writel(uint32_t v, volatile void *addr)
198{
199	__io_bw();
200	__raw_writel(htole32(v), addr);
201	__io_aw();
202}
203#define	writel(v, addr)		writel(v, addr)
204
205#undef readq
206#undef writeq
207#ifdef __LP64__
208static inline uint64_t
209readq(const volatile void *addr)
210{
211	uint64_t v;
212
213	__io_br();
214	v = le64toh(__raw_readq(addr));
215	__io_ar();
216	return (v);
217}
218#define	readq(addr)		readq(addr)
219
220static inline void
221writeq(uint64_t v, volatile void *addr)
222{
223	__io_bw();
224	__raw_writeq(htole64(v), addr);
225	__io_aw();
226}
227#define	writeq(v, addr)		writeq(v, addr)
228#endif
229
230/* Access little-endian MMIO registers atomically without memory barriers. */
231
232#undef readb_relaxed
233static inline uint8_t
234readb_relaxed(const volatile void *addr)
235{
236	return (__raw_readb(addr));
237}
238#define	readb_relaxed(addr)	readb_relaxed(addr)
239
240#undef writeb_relaxed
241static inline void
242writeb_relaxed(uint8_t v, volatile void *addr)
243{
244	__raw_writeb(v, addr);
245}
246#define	writeb_relaxed(v, addr)	writeb_relaxed(v, addr)
247
248#undef readw_relaxed
249static inline uint16_t
250readw_relaxed(const volatile void *addr)
251{
252	return (le16toh(__raw_readw(addr)));
253}
254#define	readw_relaxed(addr)	readw_relaxed(addr)
255
256#undef writew_relaxed
257static inline void
258writew_relaxed(uint16_t v, volatile void *addr)
259{
260	__raw_writew(htole16(v), addr);
261}
262#define	writew_relaxed(v, addr)	writew_relaxed(v, addr)
263
264#undef readl_relaxed
265static inline uint32_t
266readl_relaxed(const volatile void *addr)
267{
268	return (le32toh(__raw_readl(addr)));
269}
270#define	readl_relaxed(addr)	readl_relaxed(addr)
271
272#undef writel_relaxed
273static inline void
274writel_relaxed(uint32_t v, volatile void *addr)
275{
276	__raw_writel(htole32(v), addr);
277}
278#define	writel_relaxed(v, addr)	writel_relaxed(v, addr)
279
280#undef readq_relaxed
281#undef writeq_relaxed
282#ifdef __LP64__
283static inline uint64_t
284readq_relaxed(const volatile void *addr)
285{
286	return (le64toh(__raw_readq(addr)));
287}
288#define	readq_relaxed(addr)	readq_relaxed(addr)
289
290static inline void
291writeq_relaxed(uint64_t v, volatile void *addr)
292{
293	__raw_writeq(htole64(v), addr);
294}
295#define	writeq_relaxed(v, addr)	writeq_relaxed(v, addr)
296#endif
297
298/* XXX On Linux ioread and iowrite handle both MMIO and port IO. */
299
300#undef ioread8
301static inline uint8_t
302ioread8(const volatile void *addr)
303{
304	return (readb(addr));
305}
306#define	ioread8(addr)		ioread8(addr)
307
308#undef ioread16
309static inline uint16_t
310ioread16(const volatile void *addr)
311{
312	return (readw(addr));
313}
314#define	ioread16(addr)		ioread16(addr)
315
316#undef ioread16be
317static inline uint16_t
318ioread16be(const volatile void *addr)
319{
320	uint16_t v;
321
322	__io_br();
323	v = (be16toh(__raw_readw(addr)));
324	__io_ar();
325
326	return (v);
327}
328#define	ioread16be(addr)	ioread16be(addr)
329
330#undef ioread32
331static inline uint32_t
332ioread32(const volatile void *addr)
333{
334	return (readl(addr));
335}
336#define	ioread32(addr)		ioread32(addr)
337
338#undef ioread32be
339static inline uint32_t
340ioread32be(const volatile void *addr)
341{
342	uint32_t v;
343
344	__io_br();
345	v = (be32toh(__raw_readl(addr)));
346	__io_ar();
347
348	return (v);
349}
350#define	ioread32be(addr)	ioread32be(addr)
351
352#undef iowrite8
353static inline void
354iowrite8(uint8_t v, volatile void *addr)
355{
356	writeb(v, addr);
357}
358#define	iowrite8(v, addr)	iowrite8(v, addr)
359
360#undef iowrite16
361static inline void
362iowrite16(uint16_t v, volatile void *addr)
363{
364	writew(v, addr);
365}
366#define	iowrite16	iowrite16
367
368#undef iowrite32
369static inline void
370iowrite32(uint32_t v, volatile void *addr)
371{
372	writel(v, addr);
373}
374#define	iowrite32(v, addr)	iowrite32(v, addr)
375
376#undef iowrite32be
377static inline void
378iowrite32be(uint32_t v, volatile void *addr)
379{
380	__io_bw();
381	__raw_writel(htobe32(v), addr);
382	__io_aw();
383}
384#define	iowrite32be(v, addr)	iowrite32be(v, addr)
385
386#if defined(__i386__) || defined(__amd64__)
387static inline void
388_outb(u_char data, u_int port)
389{
390	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
391}
392#endif
393
394#if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || defined(__aarch64__)
395void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr);
396#else
397#define	_ioremap_attr(...) NULL
398#endif
399
400#ifdef VM_MEMATTR_DEVICE
401#define	ioremap_nocache(addr, size)					\
402    _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
403#define	ioremap_wt(addr, size)						\
404    _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
405#define	ioremap(addr, size)						\
406    _ioremap_attr((addr), (size), VM_MEMATTR_DEVICE)
407#else
408#define	ioremap_nocache(addr, size)					\
409    _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE)
410#define	ioremap_wt(addr, size)						\
411    _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_THROUGH)
412#define	ioremap(addr, size)						\
413    _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE)
414#endif
415#define	ioremap_wc(addr, size)						\
416    _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_COMBINING)
417#define	ioremap_wb(addr, size)						\
418    _ioremap_attr((addr), (size), VM_MEMATTR_WRITE_BACK)
419void iounmap(void *addr);
420
421#define	memset_io(a, b, c)	memset((a), (b), (c))
422#define	memcpy_fromio(a, b, c)	memcpy((a), (b), (c))
423#define	memcpy_toio(a, b, c)	memcpy((a), (b), (c))
424
425static inline void
426__iowrite32_copy(void *to, void *from, size_t count)
427{
428	uint32_t *src;
429	uint32_t *dst;
430	int i;
431
432	for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
433		__raw_writel(*src, dst);
434}
435
436static inline void
437__iowrite64_copy(void *to, void *from, size_t count)
438{
439#ifdef __LP64__
440	uint64_t *src;
441	uint64_t *dst;
442	int i;
443
444	for (i = 0, src = from, dst = to; i < count; i++, src++, dst++)
445		__raw_writeq(*src, dst);
446#else
447	__iowrite32_copy(to, from, count * 2);
448#endif
449}
450
451enum {
452	MEMREMAP_WB = 1 << 0,
453	MEMREMAP_WT = 1 << 1,
454	MEMREMAP_WC = 1 << 2,
455};
456
457static inline void *
458memremap(resource_size_t offset, size_t size, unsigned long flags)
459{
460	void *addr = NULL;
461
462	if ((flags & MEMREMAP_WB) &&
463	    (addr = ioremap_wb(offset, size)) != NULL)
464		goto done;
465	if ((flags & MEMREMAP_WT) &&
466	    (addr = ioremap_wt(offset, size)) != NULL)
467		goto done;
468	if ((flags & MEMREMAP_WC) &&
469	    (addr = ioremap_wc(offset, size)) != NULL)
470		goto done;
471done:
472	return (addr);
473}
474
475static inline void
476memunmap(void *addr)
477{
478	/* XXX May need to check if this is RAM */
479	iounmap(addr);
480}
481
482#endif	/* _LINUX_IO_H_ */
483