ena_plat.h revision 361467 
1/*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#ifndef ENA_PLAT_H_
35#define ENA_PLAT_H_
36
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD: stable/11/sys/contrib/ena-com/ena_plat.h 361467 2020-05-25 17:41:20Z mw $");
39
40#include <sys/param.h>
41#include <sys/systm.h>
42
43#include <sys/bus.h>
44#include <sys/condvar.h>
45#include <sys/endian.h>
46#include <sys/kernel.h>
47#include <sys/kthread.h>
48#include <sys/malloc.h>
49#include <sys/mbuf.h>
50#include <sys/module.h>
51#include <sys/rman.h>
52#include <sys/proc.h>
53#include <sys/smp.h>
54#include <sys/socket.h>
55#include <sys/sockio.h>
56#include <sys/sysctl.h>
57#include <sys/taskqueue.h>
58#include <sys/eventhandler.h>
59#include <sys/types.h>
60#include <sys/timetc.h>
61#include <sys/cdefs.h>
62
63#include <machine/atomic.h>
64#include <machine/bus.h>
65#include <machine/in_cksum.h>
66#include <machine/pcpu.h>
67#include <machine/resource.h>
68
69#include <net/bpf.h>
70#include <net/ethernet.h>
71#include <net/if.h>
72#include <net/if_var.h>
73#include <net/if_arp.h>
74#include <net/if_dl.h>
75#include <net/if_media.h>
76
77#include <net/if_types.h>
78#include <net/if_vlan_var.h>
79
80#include <netinet/in_systm.h>
81#include <netinet/in.h>
82#include <netinet/if_ether.h>
83#include <netinet/ip.h>
84#include <netinet/ip6.h>
85#include <netinet/tcp.h>
86#include <netinet/tcp_lro.h>
87#include <netinet/udp.h>
88
89#include <dev/led/led.h>
90#include <dev/pci/pcivar.h>
91#include <dev/pci/pcireg.h>
92
93extern struct ena_bus_space ebs;
94
95/* Levels */
96#define ENA_ALERT 	(1 << 0) /* Alerts are providing more error info.     */
97#define ENA_WARNING 	(1 << 1) /* Driver output is more error sensitive.    */
98#define ENA_INFO 	(1 << 2) /* Provides additional driver info. 	      */
99#define ENA_DBG 	(1 << 3) /* Driver output for debugging.	      */
100/* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. 	      */
101#define ENA_TXPTH 	(1 << 4) /* Allows TX path tracing. 		      */
102#define ENA_RXPTH 	(1 << 5) /* Allows RX path tracing.		      */
103#define ENA_RSC 	(1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
104#define ENA_IOQ 	(1 << 7) /* Detailed info about IO queues. 	      */
105#define ENA_ADMQ	(1 << 8) /* Detailed info about admin queue. 	      */
106
107extern int ena_log_level;
108
109#define ena_trace_raw(level, fmt, args...)			\
110	do {							\
111		if (((level) & ena_log_level) != (level))	\
112			break;					\
113		printf(fmt, ##args);				\
114	} while (0)
115
116#define ena_trace(level, fmt, args...)				\
117	ena_trace_raw(level, "%s() [TID:%d]: "			\
118	    fmt, __func__, curthread->td_tid, ##args)
119
120
121#define ena_trc_dbg(format, arg...) 	ena_trace(ENA_DBG, format, ##arg)
122#define ena_trc_info(format, arg...) 	ena_trace(ENA_INFO, format, ##arg)
123#define ena_trc_warn(format, arg...) 	ena_trace(ENA_WARNING, format, ##arg)
124#define ena_trc_err(format, arg...) 	ena_trace(ENA_ALERT, format, ##arg)
125
126#define unlikely(x)	__predict_false(x)
127#define likely(x)  	__predict_true(x)
128
129#define __iomem
130#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
131
132#define MAX_ERRNO 4095
133#define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
134
135#define ENA_ASSERT(cond, format, arg...)				\
136	do {								\
137		if (unlikely(!(cond))) {				\
138			ena_trc_err(					\
139				"Assert failed on %s:%s:%d:" format,	\
140				__FILE__, __func__, __LINE__, ##arg);	\
141		}							\
142	} while (0)
143
144#define ENA_WARN(cond, format, arg...)					\
145	do {								\
146		if (unlikely((cond))) {					\
147			ena_trc_warn(format, ##arg);			\
148		}							\
149	} while (0)
150
151static inline long IS_ERR(const void *ptr)
152{
153	return IS_ERR_VALUE((unsigned long)ptr);
154}
155
156static inline void *ERR_PTR(long error)
157{
158	return (void *)error;
159}
160
161static inline long PTR_ERR(const void *ptr)
162{
163	return (long) ptr;
164}
165
166#define GENMASK(h, l)	(((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
167#define GENMASK_ULL(h, l)	(((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
168#define BIT(x)			(1UL << (x))
169
170#define ENA_ABORT() 		BUG()
171#define BUG() 			panic("ENA BUG")
172
173#define SZ_256			(256)
174#define SZ_4K			(4096)
175
176#define	ENA_COM_OK		0
177#define ENA_COM_FAULT		EFAULT
178#define	ENA_COM_INVAL		EINVAL
179#define ENA_COM_NO_MEM		ENOMEM
180#define	ENA_COM_NO_SPACE	ENOSPC
181#define ENA_COM_TRY_AGAIN	-1
182#define	ENA_COM_UNSUPPORTED	EOPNOTSUPP
183#define	ENA_COM_NO_DEVICE	ENODEV
184#define	ENA_COM_PERMISSION	EPERM
185#define ENA_COM_TIMER_EXPIRED	ETIMEDOUT
186
187#define ENA_MSLEEP(x) 		pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
188#define ENA_UDELAY(x) 		DELAY(x)
189#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
190    ((long)cputick2usec(cpu_ticks()) + (timeout_us))
191#define ENA_TIME_EXPIRE(timeout)  ((timeout) < (long)cputick2usec(cpu_ticks()))
192#define ENA_MIGHT_SLEEP()
193
194#define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
195#define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
196
197#define ENA_MIN32(x,y) 	MIN(x, y)
198#define ENA_MIN16(x,y)	MIN(x, y)
199#define ENA_MIN8(x,y)	MIN(x, y)
200
201#define ENA_MAX32(x,y) 	MAX(x, y)
202#define ENA_MAX16(x,y) 	MAX(x, y)
203#define ENA_MAX8(x,y) 	MAX(x, y)
204
205/* Spinlock related methods */
206#define ena_spinlock_t 	struct mtx
207#define ENA_SPINLOCK_INIT(spinlock)				\
208	mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
209#define ENA_SPINLOCK_DESTROY(spinlock)				\
210	do {							\
211		if (mtx_initialized(&(spinlock)))		\
212		    mtx_destroy(&(spinlock));			\
213	} while (0)
214#define ENA_SPINLOCK_LOCK(spinlock, flags)			\
215	do {							\
216		(void)(flags);					\
217		mtx_lock_spin(&(spinlock));			\
218	} while (0)
219#define ENA_SPINLOCK_UNLOCK(spinlock, flags)			\
220	do {							\
221		(void)(flags);					\
222		mtx_unlock_spin(&(spinlock));			\
223	} while (0)
224
225
226/* Wait queue related methods */
227#define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
228#define ENA_WAIT_EVENT_INIT(waitqueue)					\
229	do {								\
230		cv_init(&((waitqueue).wq), "cv");			\
231		mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF);	\
232	} while (0)
233#define ENA_WAIT_EVENT_DESTROY(waitqueue)				\
234	do {								\
235		cv_destroy(&((waitqueue).wq));				\
236		mtx_destroy(&((waitqueue).mtx));			\
237	} while (0)
238#define ENA_WAIT_EVENT_CLEAR(waitqueue)					\
239	cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
240#define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us)			\
241	do {								\
242		mtx_lock(&((waitqueue).mtx));				\
243		cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx),	\
244		    timeout_us * hz / 1000 / 1000 );			\
245		mtx_unlock(&((waitqueue).mtx));				\
246	} while (0)
247#define ENA_WAIT_EVENT_SIGNAL(waitqueue)		\
248	do {						\
249		mtx_lock(&((waitqueue).mtx));		\
250		cv_broadcast(&((waitqueue).wq));	\
251		mtx_unlock(&((waitqueue).mtx));		\
252	} while (0)
253
254#define dma_addr_t 	bus_addr_t
255#define u8 		uint8_t
256#define u16 		uint16_t
257#define u32 		uint32_t
258#define u64 		uint64_t
259
260typedef struct {
261	bus_addr_t              paddr;
262	caddr_t                 vaddr;
263        bus_dma_tag_t           tag;
264	bus_dmamap_t            map;
265        bus_dma_segment_t       seg;
266	int                     nseg;
267} ena_mem_handle_t;
268
269struct ena_bus {
270	bus_space_handle_t 	reg_bar_h;
271	bus_space_tag_t 	reg_bar_t;
272	bus_space_handle_t	mem_bar_h;
273	bus_space_tag_t 	mem_bar_t;
274};
275
276typedef uint32_t ena_atomic32_t;
277
278void	ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
279    int error);
280int	ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
281    int mapflags);
282
283#define ENA_MEMCPY_TO_DEVICE_64(dst, src, size)				\
284	do {								\
285		int count, i;						\
286		volatile uint64_t *to = (volatile uint64_t *)(dst);	\
287		const uint64_t *from = (const uint64_t *)(src);		\
288		count = (size) / 8;					\
289									\
290		for (i = 0; i < count; i++, from++, to++)		\
291			*to = *from;					\
292	} while (0)
293
294#define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
295#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
296#define ENA_MEM_FREE(dmadev, ptr) free(ptr, M_DEVBUF)
297#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \
298    dev_node)								\
299	do {								\
300		((virt) = NULL);					\
301		(void)(dev_node);					\
302	} while (0)
303
304#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma)		\
305	do {								\
306		ena_dma_alloc((dmadev), (size), &(dma), 0);		\
307		(virt) = (void *)(dma).vaddr;				\
308		(phys) = (dma).paddr;					\
309	} while (0)
310
311#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma)		\
312	do {								\
313		(void)size;						\
314		bus_dmamap_unload((dma).tag, (dma).map);		\
315		bus_dmamem_free((dma).tag, (virt), (dma).map);		\
316		bus_dma_tag_destroy((dma).tag);				\
317		(dma).tag = NULL;					\
318		(virt) = NULL;						\
319	} while (0)
320
321/* Register R/W methods */
322#define ENA_REG_WRITE32(bus, value, offset)				\
323	bus_space_write_4(						\
324			  ((struct ena_bus*)bus)->reg_bar_t,		\
325			  ((struct ena_bus*)bus)->reg_bar_h,		\
326			  (bus_size_t)(offset), (value))
327#define ENA_REG_WRITE32_RELAXED(bus, value, offset)			\
328	ENA_REG_WRITE32(bus, value, offset)
329
330#define ENA_REG_READ32(bus, offset)					\
331	bus_space_read_4(						\
332			 ((struct ena_bus*)bus)->reg_bar_t,		\
333			 ((struct ena_bus*)bus)->reg_bar_h,		\
334			 (bus_size_t)(offset))
335
336#define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync(			\
337	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE)
338#define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync(		\
339	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD)
340#define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync(		\
341	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD)
342#define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle)
343
344#define time_after(a,b)	((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
345
346#define VLAN_HLEN 	sizeof(struct ether_vlan_header)
347#define CSUM_OFFLOAD 	(CSUM_IP|CSUM_TCP|CSUM_UDP)
348
349#define prefetch(x)	(void)(x)
350#define prefetchw(x)	(void)(x)
351
352/* DMA buffers access */
353#define	dma_unmap_addr(p, name)			((p)->dma->name)
354#define	dma_unmap_addr_set(p, name, v)		(((p)->dma->name) = (v))
355#define	dma_unmap_len(p, name)			((p)->name)
356#define	dma_unmap_len_set(p, name, v)		(((p)->name) = (v))
357
358#define memcpy_toio memcpy
359
360#define ATOMIC32_INC(I32_PTR)		atomic_add_int(I32_PTR, 1)
361#define ATOMIC32_DEC(I32_PTR) 		atomic_add_int(I32_PTR, -1)
362#define ATOMIC32_READ(I32_PTR) 		atomic_load_acq_int(I32_PTR)
363#define ATOMIC32_SET(I32_PTR, VAL) 	atomic_store_rel_int(I32_PTR, VAL)
364
365#define	barrier() __asm__ __volatile__("": : :"memory")
366#define dma_rmb() barrier()
367#define mmiowb() barrier()
368
369#define	ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
370#define READ_ONCE(x)  ({			\
371			__typeof(x) __var;	\
372			barrier();		\
373			__var = ACCESS_ONCE(x);	\
374			barrier();		\
375			__var;			\
376		})
377#define READ_ONCE8(x) READ_ONCE(x)
378#define READ_ONCE16(x) READ_ONCE(x)
379#define READ_ONCE32(x) READ_ONCE(x)
380
381#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
382#define lower_32_bits(n) ((uint32_t)(n))
383
384#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
385
386#include "ena_defs/ena_includes.h"
387
388#endif /* ENA_PLAT_H_ */
389