ena_plat.h revision 368013
1/*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34#ifndef ENA_PLAT_H_ 35#define ENA_PLAT_H_ 36 37#include <sys/cdefs.h> 38__FBSDID("$FreeBSD: stable/11/sys/contrib/ena-com/ena_plat.h 368013 2020-11-25 09:37:17Z mw $"); 39 40#include <sys/param.h> 41#include <sys/systm.h> 42 43#include <sys/bus.h> 44#include <sys/condvar.h> 45#include <sys/endian.h> 46#include <sys/kernel.h> 47#include <sys/kthread.h> 48#include <sys/malloc.h> 49#include <sys/mbuf.h> 50#include <sys/module.h> 51#include <sys/rman.h> 52#include <sys/proc.h> 53#include <sys/smp.h> 54#include <sys/socket.h> 55#include <sys/sockio.h> 56#include <sys/sysctl.h> 57#include <sys/taskqueue.h> 58#include <sys/eventhandler.h> 59#include <sys/types.h> 60#include <sys/timetc.h> 61#include <sys/cdefs.h> 62 63#include <machine/atomic.h> 64#include <machine/bus.h> 65#include <machine/in_cksum.h> 66#include <machine/pcpu.h> 67#include <machine/resource.h> 68#include <machine/_inttypes.h> 69 70#include <net/bpf.h> 71#include <net/ethernet.h> 72#include <net/if.h> 73#include <net/if_var.h> 74#include <net/if_arp.h> 75#include <net/if_dl.h> 76#include <net/if_media.h> 77 78#include <net/if_types.h> 79#include <net/if_vlan_var.h> 80 81#include <netinet/in_systm.h> 82#include <netinet/in.h> 83#include <netinet/if_ether.h> 84#include <netinet/ip.h> 85#include <netinet/ip6.h> 86#include <netinet/tcp.h> 87#include <netinet/tcp_lro.h> 88#include <netinet/udp.h> 89 90#include <dev/led/led.h> 91#include <dev/pci/pcivar.h> 92#include <dev/pci/pcireg.h> 93 94extern struct ena_bus_space ebs; 95 96/* Levels */ 97#define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */ 98#define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */ 99#define ENA_INFO (1 << 2) /* Provides additional driver info. */ 100#define ENA_DBG (1 << 3) /* Driver output for debugging. */ 101/* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */ 102#define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */ 103#define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */ 104#define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */ 105#define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */ 106#define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */ 107#define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */ 108 109#define DEFAULT_ALLOC_ALIGNMENT 8 110 111extern int ena_log_level; 112 113#define container_of(ptr, type, member) \ 114 ({ \ 115 const __typeof(((type *)0)->member) *__p = (ptr); \ 116 (type *)((uintptr_t)__p - offsetof(type, member)); \ 117 }) 118 119#define ena_trace_raw(ctx, level, fmt, args...) \ 120 do { \ 121 ((void)(ctx)); \ 122 if (((level) & ena_log_level) != (level)) \ 123 break; \ 124 printf(fmt, ##args); \ 125 } while (0) 126 127#define ena_trace(ctx, level, fmt, args...) \ 128 ena_trace_raw(ctx, level, "%s() [TID:%d]: " \ 129 fmt, __func__, curthread->td_tid, ##args) 130 131 132#define ena_trc_dbg(ctx, format, arg...) \ 133 ena_trace(ctx, ENA_DBG, format, ##arg) 134#define ena_trc_info(ctx, format, arg...) \ 135 ena_trace(ctx, ENA_INFO, format, ##arg) 136#define ena_trc_warn(ctx, format, arg...) \ 137 ena_trace(ctx, ENA_WARNING, format, ##arg) 138#define ena_trc_err(ctx, format, arg...) \ 139 ena_trace(ctx, ENA_ALERT, format, ##arg) 140 141#define unlikely(x) __predict_false(!!(x)) 142#define likely(x) __predict_true(!!(x)) 143 144#define __iomem 145#define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 146 147#define MAX_ERRNO 4095 148#define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 149 150#define ENA_WARN(cond, ctx, format, arg...) \ 151 do { \ 152 if (unlikely((cond))) { \ 153 ena_trc_warn(ctx, format, ##arg); \ 154 } \ 155 } while (0) 156 157static inline long IS_ERR(const void *ptr) 158{ 159 return IS_ERR_VALUE((unsigned long)ptr); 160} 161 162static inline void *ERR_PTR(long error) 163{ 164 return (void *)error; 165} 166 167static inline long PTR_ERR(const void *ptr) 168{ 169 return (long) ptr; 170} 171 172#define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 173#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 174#define BIT(x) (1UL << (x)) 175 176#define ENA_ABORT() BUG() 177#define BUG() panic("ENA BUG") 178 179#define SZ_256 (256) 180#define SZ_4K (4096) 181 182#define ENA_COM_OK 0 183#define ENA_COM_FAULT EFAULT 184#define ENA_COM_INVAL EINVAL 185#define ENA_COM_NO_MEM ENOMEM 186#define ENA_COM_NO_SPACE ENOSPC 187#define ENA_COM_TRY_AGAIN -1 188#define ENA_COM_UNSUPPORTED EOPNOTSUPP 189#define ENA_COM_NO_DEVICE ENODEV 190#define ENA_COM_PERMISSION EPERM 191#define ENA_COM_TIMER_EXPIRED ETIMEDOUT 192#define ENA_COM_EIO EIO 193 194#define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 195#define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0) 196#define ENA_UDELAY(x) DELAY(x) 197#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 198 ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 199#define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks())) 200#define ENA_MIGHT_SLEEP() 201 202#define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 203#define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 204 205#define ENA_MIN32(x,y) MIN(x, y) 206#define ENA_MIN16(x,y) MIN(x, y) 207#define ENA_MIN8(x,y) MIN(x, y) 208 209#define ENA_MAX32(x,y) MAX(x, y) 210#define ENA_MAX16(x,y) MAX(x, y) 211#define ENA_MAX8(x,y) MAX(x, y) 212 213/* Spinlock related methods */ 214#define ena_spinlock_t struct mtx 215#define ENA_SPINLOCK_INIT(spinlock) \ 216 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 217#define ENA_SPINLOCK_DESTROY(spinlock) \ 218 do { \ 219 if (mtx_initialized(&(spinlock))) \ 220 mtx_destroy(&(spinlock)); \ 221 } while (0) 222#define ENA_SPINLOCK_LOCK(spinlock, flags) \ 223 do { \ 224 (void)(flags); \ 225 mtx_lock_spin(&(spinlock)); \ 226 } while (0) 227#define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 228 do { \ 229 (void)(flags); \ 230 mtx_unlock_spin(&(spinlock)); \ 231 } while (0) 232 233 234/* Wait queue related methods */ 235#define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 236#define ENA_WAIT_EVENT_INIT(waitqueue) \ 237 do { \ 238 cv_init(&((waitqueue).wq), "cv"); \ 239 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 240 } while (0) 241#define ENA_WAIT_EVENTS_DESTROY(admin_queue) \ 242 do { \ 243 struct ena_comp_ctx *comp_ctx; \ 244 int i; \ 245 for (i = 0; i < admin_queue->q_depth; i++) { \ 246 comp_ctx = get_comp_ctxt(admin_queue, i, false); \ 247 if (comp_ctx != NULL) { \ 248 cv_destroy(&((comp_ctx->wait_event).wq)); \ 249 mtx_destroy(&((comp_ctx->wait_event).mtx)); \ 250 } \ 251 } \ 252 } while (0) 253#define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 254 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 255#define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 256 do { \ 257 mtx_lock(&((waitqueue).mtx)); \ 258 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 259 timeout_us * hz / 1000 / 1000 ); \ 260 mtx_unlock(&((waitqueue).mtx)); \ 261 } while (0) 262#define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 263 do { \ 264 mtx_lock(&((waitqueue).mtx)); \ 265 cv_broadcast(&((waitqueue).wq)); \ 266 mtx_unlock(&((waitqueue).mtx)); \ 267 } while (0) 268 269#define dma_addr_t bus_addr_t 270#define u8 uint8_t 271#define u16 uint16_t 272#define u32 uint32_t 273#define u64 uint64_t 274 275typedef struct { 276 bus_addr_t paddr; 277 caddr_t vaddr; 278 bus_dma_tag_t tag; 279 bus_dmamap_t map; 280 bus_dma_segment_t seg; 281 int nseg; 282} ena_mem_handle_t; 283 284struct ena_bus { 285 bus_space_handle_t reg_bar_h; 286 bus_space_tag_t reg_bar_t; 287 bus_space_handle_t mem_bar_h; 288 bus_space_tag_t mem_bar_t; 289}; 290 291typedef uint32_t ena_atomic32_t; 292 293#define ENA_PRIu64 PRIu64 294 295typedef uint64_t ena_time_t; 296typedef struct ifnet ena_netdev; 297 298void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 299 int error); 300int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 301 int mapflags, bus_size_t alignment); 302 303static inline uint32_t 304ena_reg_read32(struct ena_bus *bus, bus_size_t offset) 305{ 306 uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset); 307 rmb(); 308 return v; 309} 310 311#define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 312 do { \ 313 int count, i; \ 314 volatile uint64_t *to = (volatile uint64_t *)(dst); \ 315 const uint64_t *from = (const uint64_t *)(src); \ 316 count = (size) / 8; \ 317 \ 318 for (i = 0; i < count; i++, from++, to++) \ 319 *to = *from; \ 320 } while (0) 321 322#define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 323#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL) 324#define ENA_MEM_FREE(dmadev, ptr, size) \ 325 do { \ 326 (void)(size); \ 327 free(ptr, M_DEVBUF); \ 328 } while (0) 329#define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \ 330 handle, node, dev_node, alignment) \ 331 do { \ 332 ((virt) = NULL); \ 333 (void)(dev_node); \ 334 } while (0) 335 336#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \ 337 node, dev_node) \ 338 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \ 339 phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT) 340 341#define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \ 342 alignment) \ 343 do { \ 344 ena_dma_alloc((dmadev), (size), &(dma), 0, alignment); \ 345 (virt) = (void *)(dma).vaddr; \ 346 (phys) = (dma).paddr; \ 347 } while (0) 348 349#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 350 ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \ 351 phys, dma, DEFAULT_ALLOC_ALIGNMENT) 352 353#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 354 do { \ 355 (void)size; \ 356 bus_dmamap_unload((dma).tag, (dma).map); \ 357 bus_dmamem_free((dma).tag, (virt), (dma).map); \ 358 bus_dma_tag_destroy((dma).tag); \ 359 (dma).tag = NULL; \ 360 (virt) = NULL; \ 361 } while (0) 362 363/* Register R/W methods */ 364#define ENA_REG_WRITE32(bus, value, offset) \ 365 do { \ 366 wmb(); \ 367 ENA_REG_WRITE32_RELAXED(bus, value, offset); \ 368 } while (0) 369 370#define ENA_REG_WRITE32_RELAXED(bus, value, offset) \ 371 bus_space_write_4( \ 372 ((struct ena_bus*)bus)->reg_bar_t, \ 373 ((struct ena_bus*)bus)->reg_bar_h, \ 374 (bus_size_t)(offset), (value)) 375 376#define ENA_REG_READ32(bus, offset) \ 377 ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset)) 378 379#define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \ 380 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE) 381#define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \ 382 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD) 383#define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \ 384 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD) 385#define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle) 386 387#define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 388 389#define VLAN_HLEN sizeof(struct ether_vlan_header) 390#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) 391 392#define prefetch(x) (void)(x) 393#define prefetchw(x) (void)(x) 394 395/* DMA buffers access */ 396#define dma_unmap_addr(p, name) ((p)->dma->name) 397#define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 398#define dma_unmap_len(p, name) ((p)->name) 399#define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 400 401#define memcpy_toio memcpy 402 403#define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 404#define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 405#define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 406#define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 407 408#define barrier() __asm__ __volatile__("": : :"memory") 409#define dma_rmb() barrier() 410#define mmiowb() barrier() 411 412#define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 413#define READ_ONCE(x) ({ \ 414 __typeof(x) __var; \ 415 barrier(); \ 416 __var = ACCESS_ONCE(x); \ 417 barrier(); \ 418 __var; \ 419 }) 420#define READ_ONCE8(x) READ_ONCE(x) 421#define READ_ONCE16(x) READ_ONCE(x) 422#define READ_ONCE32(x) READ_ONCE(x) 423 424#define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) 425#define lower_32_bits(n) ((uint32_t)(n)) 426 427#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 428 429#define ENA_FFS(x) ffs(x) 430 431void ena_rss_key_fill(void *key, size_t size); 432 433#define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) 434 435#include "ena_defs/ena_includes.h" 436 437#endif /* ENA_PLAT_H_ */ 438