ena_plat.h revision 361468
1193323Sed/*- 2193323Sed * BSD LICENSE 3193323Sed * 4193323Sed * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. 5193323Sed * All rights reserved. 6193323Sed * 7193323Sed * Redistribution and use in source and binary forms, with or without 8193323Sed * modification, are permitted provided that the following conditions 9193323Sed * are met: 10193323Sed * 11193323Sed * * Redistributions of source code must retain the above copyright 12193323Sed * notice, this list of conditions and the following disclaimer. 13193323Sed * * Redistributions in binary form must reproduce the above copyright 14193323Sed * notice, this list of conditions and the following disclaimer in 15193323Sed * the documentation and/or other materials provided with the 16193323Sed * distribution. 17193323Sed * * Neither the name of copyright holder nor the names of its 18193323Sed * contributors may be used to endorse or promote products derived 19193323Sed * from this software without specific prior written permission. 20193323Sed * 21193323Sed * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22193323Sed * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23193323Sed * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24193323Sed * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25193323Sed * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26193323Sed * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27193323Sed * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28193323Sed * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29193323Sed * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30193323Sed * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31193323Sed * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32193323Sed */ 33193323Sed 34193323Sed#ifndef ENA_PLAT_H_ 35193323Sed#define ENA_PLAT_H_ 36193323Sed 37193323Sed#include <sys/cdefs.h> 38193323Sed__FBSDID("$FreeBSD: stable/11/sys/contrib/ena-com/ena_plat.h 361468 2020-05-25 17:47:31Z mw $"); 39193323Sed 40193323Sed#include <sys/param.h> 41193323Sed#include <sys/systm.h> 42193323Sed 43193323Sed#include <sys/bus.h> 44193323Sed#include <sys/condvar.h> 45221345Sdim#include <sys/endian.h> 46193323Sed#include <sys/kernel.h> 47221345Sdim#include <sys/kthread.h> 48193323Sed#include <sys/malloc.h> 49193323Sed#include <sys/mbuf.h> 50221345Sdim#include <sys/module.h> 51221345Sdim#include <sys/rman.h> 52193323Sed#include <sys/proc.h> 53221345Sdim#include <sys/smp.h> 54193323Sed#include <sys/socket.h> 55193323Sed#include <sys/sockio.h> 56193323Sed#include <sys/sysctl.h> 57193323Sed#include <sys/taskqueue.h> 58193323Sed#include <sys/eventhandler.h> 59193323Sed#include <sys/types.h> 60193323Sed#include <sys/timetc.h> 61193323Sed#include <sys/cdefs.h> 62193323Sed 63193323Sed#include <machine/atomic.h> 64193323Sed#include <machine/bus.h> 65193323Sed#include <machine/in_cksum.h> 66221345Sdim#include <machine/pcpu.h> 67193323Sed#include <machine/resource.h> 68193323Sed 69193323Sed#include <net/bpf.h> 70193323Sed#include <net/ethernet.h> 71193323Sed#include <net/if.h> 72221345Sdim#include <net/if_var.h> 73221345Sdim#include <net/if_arp.h> 74193323Sed#include <net/if_dl.h> 75193323Sed#include <net/if_media.h> 76193323Sed 77193323Sed#include <net/if_types.h> 78221345Sdim#include <net/if_vlan_var.h> 79221345Sdim 80193323Sed#include <netinet/in_systm.h> 81193323Sed#include <netinet/in.h> 82193323Sed#include <netinet/if_ether.h> 83193323Sed#include <netinet/ip.h> 84193323Sed#include <netinet/ip6.h> 85193323Sed#include <netinet/tcp.h> 86193323Sed#include <netinet/tcp_lro.h> 87193323Sed#include <netinet/udp.h> 88193323Sed 89193323Sed#include <dev/led/led.h> 90193323Sed#include <dev/pci/pcivar.h> 91193323Sed#include <dev/pci/pcireg.h> 92193323Sed 93193323Sedextern struct ena_bus_space ebs; 94193323Sed 95193323Sed/* Levels */ 96193323Sed#define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */ 97193323Sed#define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */ 98193323Sed#define ENA_INFO (1 << 2) /* Provides additional driver info. */ 99193323Sed#define ENA_DBG (1 << 3) /* Driver output for debugging. */ 100193323Sed/* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */ 101193323Sed#define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */ 102193323Sed#define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */ 103193323Sed#define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */ 104193323Sed#define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */ 105193323Sed#define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */ 106193323Sed#define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */ 107193323Sed 108193323Sedextern int ena_log_level; 109193323Sed 110193323Sed#define ena_trace_raw(level, fmt, args...) \ 111193323Sed do { \ 112193323Sed if (((level) & ena_log_level) != (level)) \ 113193323Sed break; \ 114193323Sed printf(fmt, ##args); \ 115193323Sed } while (0) 116193323Sed 117193323Sed#define ena_trace(level, fmt, args...) \ 118193323Sed ena_trace_raw(level, "%s() [TID:%d]: " \ 119193323Sed fmt, __func__, curthread->td_tid, ##args) 120193323Sed 121193323Sed 122193323Sed#define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg) 123193323Sed#define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg) 124193323Sed#define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg) 125193323Sed#define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg) 126193323Sed 127193323Sed#define unlikely(x) __predict_false(x) 128193323Sed#define likely(x) __predict_true(x) 129193323Sed 130193323Sed#define __iomem 131193323Sed#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 132193323Sed 133193323Sed#define MAX_ERRNO 4095 134193323Sed#define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 135193323Sed 136193323Sed#define ENA_ASSERT(cond, format, arg...) \ 137193323Sed do { \ 138193323Sed if (unlikely(!(cond))) { \ 139193323Sed ena_trc_err( \ 140193323Sed "Assert failed on %s:%s:%d:" format, \ 141193323Sed __FILE__, __func__, __LINE__, ##arg); \ 142193323Sed } \ 143193323Sed } while (0) 144193323Sed 145193323Sed#define ENA_WARN(cond, format, arg...) \ 146193323Sed do { \ 147193323Sed if (unlikely((cond))) { \ 148193323Sed ena_trc_warn(format, ##arg); \ 149193323Sed } \ 150193323Sed } while (0) 151193323Sed 152193323Sedstatic inline long IS_ERR(const void *ptr) 153193323Sed{ 154193323Sed return IS_ERR_VALUE((unsigned long)ptr); 155193323Sed} 156221345Sdim 157221345Sdimstatic inline void *ERR_PTR(long error) 158221345Sdim{ 159221345Sdim return (void *)error; 160193323Sed} 161193323Sed 162193323Sedstatic inline long PTR_ERR(const void *ptr) 163193323Sed{ 164193323Sed return (long) ptr; 165193323Sed} 166193323Sed 167193323Sed#define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 168193323Sed#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 169193323Sed#define BIT(x) (1UL << (x)) 170193323Sed 171193323Sed#define ENA_ABORT() BUG() 172193323Sed#define BUG() panic("ENA BUG") 173193323Sed 174193323Sed#define SZ_256 (256) 175193323Sed#define SZ_4K (4096) 176193323Sed 177193323Sed#define ENA_COM_OK 0 178193323Sed#define ENA_COM_FAULT EFAULT 179193323Sed#define ENA_COM_INVAL EINVAL 180193323Sed#define ENA_COM_NO_MEM ENOMEM 181193323Sed#define ENA_COM_NO_SPACE ENOSPC 182193323Sed#define ENA_COM_TRY_AGAIN -1 183193323Sed#define ENA_COM_UNSUPPORTED EOPNOTSUPP 184193323Sed#define ENA_COM_NO_DEVICE ENODEV 185193323Sed#define ENA_COM_PERMISSION EPERM 186193323Sed#define ENA_COM_TIMER_EXPIRED ETIMEDOUT 187193323Sed 188193323Sed#define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 189193323Sed#define ENA_UDELAY(x) DELAY(x) 190193323Sed#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 191193323Sed ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 192193323Sed#define ENA_TIME_EXPIRE(timeout) ((timeout) < (long)cputick2usec(cpu_ticks())) 193193323Sed#define ENA_MIGHT_SLEEP() 194193323Sed 195193323Sed#define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 196193323Sed#define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 197193323Sed 198193323Sed#define ENA_MIN32(x,y) MIN(x, y) 199193323Sed#define ENA_MIN16(x,y) MIN(x, y) 200193323Sed#define ENA_MIN8(x,y) MIN(x, y) 201193323Sed 202193323Sed#define ENA_MAX32(x,y) MAX(x, y) 203193323Sed#define ENA_MAX16(x,y) MAX(x, y) 204193323Sed#define ENA_MAX8(x,y) MAX(x, y) 205193323Sed 206193323Sed/* Spinlock related methods */ 207193323Sed#define ena_spinlock_t struct mtx 208193323Sed#define ENA_SPINLOCK_INIT(spinlock) \ 209193323Sed mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 210193323Sed#define ENA_SPINLOCK_DESTROY(spinlock) \ 211193323Sed do { \ 212193323Sed if (mtx_initialized(&(spinlock))) \ 213193323Sed mtx_destroy(&(spinlock)); \ 214193323Sed } while (0) 215193323Sed#define ENA_SPINLOCK_LOCK(spinlock, flags) \ 216193323Sed do { \ 217193323Sed (void)(flags); \ 218193323Sed mtx_lock_spin(&(spinlock)); \ 219193323Sed } while (0) 220193323Sed#define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 221193323Sed do { \ 222193323Sed (void)(flags); \ 223193323Sed mtx_unlock_spin(&(spinlock)); \ 224193323Sed } while (0) 225193323Sed 226193323Sed 227193323Sed/* Wait queue related methods */ 228193323Sed#define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 229193323Sed#define ENA_WAIT_EVENT_INIT(waitqueue) \ 230193323Sed do { \ 231193323Sed cv_init(&((waitqueue).wq), "cv"); \ 232193323Sed mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 233 } while (0) 234#define ENA_WAIT_EVENT_DESTROY(waitqueue) \ 235 do { \ 236 cv_destroy(&((waitqueue).wq)); \ 237 mtx_destroy(&((waitqueue).mtx)); \ 238 } while (0) 239#define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 240 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 241#define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 242 do { \ 243 mtx_lock(&((waitqueue).mtx)); \ 244 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 245 timeout_us * hz / 1000 / 1000 ); \ 246 mtx_unlock(&((waitqueue).mtx)); \ 247 } while (0) 248#define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 249 do { \ 250 mtx_lock(&((waitqueue).mtx)); \ 251 cv_broadcast(&((waitqueue).wq)); \ 252 mtx_unlock(&((waitqueue).mtx)); \ 253 } while (0) 254 255#define dma_addr_t bus_addr_t 256#define u8 uint8_t 257#define u16 uint16_t 258#define u32 uint32_t 259#define u64 uint64_t 260 261typedef struct { 262 bus_addr_t paddr; 263 caddr_t vaddr; 264 bus_dma_tag_t tag; 265 bus_dmamap_t map; 266 bus_dma_segment_t seg; 267 int nseg; 268} ena_mem_handle_t; 269 270struct ena_bus { 271 bus_space_handle_t reg_bar_h; 272 bus_space_tag_t reg_bar_t; 273 bus_space_handle_t mem_bar_h; 274 bus_space_tag_t mem_bar_t; 275}; 276 277typedef uint32_t ena_atomic32_t; 278 279void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 280 int error); 281int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 282 int mapflags); 283 284#define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 285 do { \ 286 int count, i; \ 287 volatile uint64_t *to = (volatile uint64_t *)(dst); \ 288 const uint64_t *from = (const uint64_t *)(src); \ 289 count = (size) / 8; \ 290 \ 291 for (i = 0; i < count; i++, from++, to++) \ 292 *to = *from; \ 293 } while (0) 294 295#define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 296#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL) 297#define ENA_MEM_FREE(dmadev, ptr) free(ptr, M_DEVBUF) 298#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \ 299 dev_node) \ 300 do { \ 301 ((virt) = NULL); \ 302 (void)(dev_node); \ 303 } while (0) 304 305#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 306 do { \ 307 ena_dma_alloc((dmadev), (size), &(dma), 0); \ 308 (virt) = (void *)(dma).vaddr; \ 309 (phys) = (dma).paddr; \ 310 } while (0) 311 312#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 313 do { \ 314 (void)size; \ 315 bus_dmamap_unload((dma).tag, (dma).map); \ 316 bus_dmamem_free((dma).tag, (virt), (dma).map); \ 317 bus_dma_tag_destroy((dma).tag); \ 318 (dma).tag = NULL; \ 319 (virt) = NULL; \ 320 } while (0) 321 322/* Register R/W methods */ 323#define ENA_REG_WRITE32(bus, value, offset) \ 324 bus_space_write_4( \ 325 ((struct ena_bus*)bus)->reg_bar_t, \ 326 ((struct ena_bus*)bus)->reg_bar_h, \ 327 (bus_size_t)(offset), (value)) 328#define ENA_REG_WRITE32_RELAXED(bus, value, offset) \ 329 ENA_REG_WRITE32(bus, value, offset) 330 331#define ENA_REG_READ32(bus, offset) \ 332 bus_space_read_4( \ 333 ((struct ena_bus*)bus)->reg_bar_t, \ 334 ((struct ena_bus*)bus)->reg_bar_h, \ 335 (bus_size_t)(offset)) 336 337#define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \ 338 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE) 339#define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \ 340 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD) 341#define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \ 342 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD) 343#define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle) 344 345#define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 346 347#define VLAN_HLEN sizeof(struct ether_vlan_header) 348#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) 349 350#define prefetch(x) (void)(x) 351#define prefetchw(x) (void)(x) 352 353/* DMA buffers access */ 354#define dma_unmap_addr(p, name) ((p)->dma->name) 355#define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 356#define dma_unmap_len(p, name) ((p)->name) 357#define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 358 359#define memcpy_toio memcpy 360 361#define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 362#define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 363#define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 364#define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 365 366#define barrier() __asm__ __volatile__("": : :"memory") 367#define dma_rmb() barrier() 368#define mmiowb() barrier() 369 370#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 371#define READ_ONCE(x) ({ \ 372 __typeof(x) __var; \ 373 barrier(); \ 374 __var = ACCESS_ONCE(x); \ 375 barrier(); \ 376 __var; \ 377 }) 378#define READ_ONCE8(x) READ_ONCE(x) 379#define READ_ONCE16(x) READ_ONCE(x) 380#define READ_ONCE32(x) READ_ONCE(x) 381 382#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) 383#define lower_32_bits(n) ((uint32_t)(n)) 384 385#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 386 387#include "ena_defs/ena_includes.h" 388 389#endif /* ENA_PLAT_H_ */ 390