mlx5_ib.h revision 323223
1251881Speter/*- 2251881Speter * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3251881Speter * 4251881Speter * Redistribution and use in source and binary forms, with or without 5251881Speter * modification, are permitted provided that the following conditions 6251881Speter * are met: 7251881Speter * 1. Redistributions of source code must retain the above copyright 8251881Speter * notice, this list of conditions and the following disclaimer. 9251881Speter * 2. Redistributions in binary form must reproduce the above copyright 10251881Speter * notice, this list of conditions and the following disclaimer in the 11251881Speter * documentation and/or other materials provided with the distribution. 12251881Speter * 13251881Speter * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14251881Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15251881Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16251881Speter * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17251881Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18251881Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19251881Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20251881Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21251881Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22251881Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23251881Speter * SUCH DAMAGE. 24251881Speter * 25251881Speter * $FreeBSD: stable/10/sys/dev/mlx5/mlx5_ib/mlx5_ib.h 323223 2017-09-06 15:33:23Z hselasky $ 26251881Speter */ 27251881Speter 28251881Speter#ifndef MLX5_IB_H 29251881Speter#define MLX5_IB_H 30251881Speter 31251881Speter#include <linux/kernel.h> 32251881Speter#include <linux/sched.h> 33251881Speter#include <rdma/ib_verbs.h> 34251881Speter#include <rdma/ib_smi.h> 35251881Speter#include <rdma/ib_addr.h> 36251881Speter#include <dev/mlx5/device.h> 37251881Speter#include <dev/mlx5/driver.h> 38251881Speter#include <dev/mlx5/cq.h> 39251881Speter#include <dev/mlx5/qp.h> 40251881Speter#include <dev/mlx5/srq.h> 41251881Speter#include <linux/types.h> 42251881Speter#include <dev/mlx5/mlx5_core/transobj.h> 43251881Speter 44251881Speter#define mlx5_ib_dbg(dev, format, arg...) \ 45251881Speterpr_debug("mlx5_dbg:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ 46251881Speter __LINE__, curthread->td_proc->p_pid, ##arg) 47251881Speter 48251881Speter#define mlx5_ib_err(dev, format, arg...) \ 49251881Speterprintf("mlx5_ib: ERR: ""mlx5_err:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ 50251881Speter __LINE__, curthread->td_proc->p_pid, ##arg) 51251881Speter 52251881Speter#define mlx5_ib_warn(dev, format, arg...) \ 53251881Speterprintf("mlx5_ib: WARN: ""mlx5_warn:%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \ 54251881Speter __LINE__, curthread->td_proc->p_pid, ##arg) 55251881Speter#define BF_ENABLE 0 56251881Speter 57251881Speterextern struct workqueue_struct *mlx5_ib_wq; 58251881Speter 59251881Speterenum { 60251881Speter MLX5_IB_MMAP_CMD_SHIFT = 8, 61251881Speter MLX5_IB_MMAP_CMD_MASK = 0xff, 62251881Speter}; 63251881Speter 64251881Speterenum mlx5_ib_mmap_cmd { 65251881Speter MLX5_IB_MMAP_REGULAR_PAGE = 0, 66251881Speter MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES = 1, 67251881Speter MLX5_IB_MMAP_WC_PAGE = 2, 68251881Speter MLX5_IB_MMAP_NC_PAGE = 3, 69251881Speter MLX5_IB_MMAP_MAP_DC_INFO_PAGE = 4, 70251881Speter 71251881Speter /* Use EXP mmap commands until it is pushed to upstream */ 72251881Speter MLX5_IB_EXP_MMAP_CORE_CLOCK = 0xFB, 73251881Speter MLX5_IB_EXP_MMAP_GET_CONTIGUOUS_PAGES_CPU_NUMA = 0xFC, 74251881Speter MLX5_IB_EXP_MMAP_GET_CONTIGUOUS_PAGES_DEV_NUMA = 0xFD, 75251881Speter MLX5_IB_EXP_ALLOC_N_MMAP_WC = 0xFE, 76251881Speter}; 77251881Speter 78251881Speterenum { 79251881Speter MLX5_RES_SCAT_DATA32_CQE = 0x1, 80251881Speter MLX5_RES_SCAT_DATA64_CQE = 0x2, 81251881Speter MLX5_REQ_SCAT_DATA32_CQE = 0x11, 82251881Speter MLX5_REQ_SCAT_DATA64_CQE = 0x22, 83251881Speter}; 84251881Speter 85251881Speterenum { 86251881Speter MLX5_DCT_CS_RES_64 = 2, 87251881Speter MLX5_CNAK_RX_POLL_CQ_QUOTA = 256, 88251881Speter}; 89251881Speter 90251881Speterenum mlx5_ib_latency_class { 91251881Speter MLX5_IB_LATENCY_CLASS_LOW, 92251881Speter MLX5_IB_LATENCY_CLASS_MEDIUM, 93251881Speter MLX5_IB_LATENCY_CLASS_HIGH, 94251881Speter MLX5_IB_LATENCY_CLASS_FAST_PATH 95251881Speter}; 96251881Speter 97251881Speterenum mlx5_ib_mad_ifc_flags { 98251881Speter MLX5_MAD_IFC_IGNORE_MKEY = 1, 99251881Speter MLX5_MAD_IFC_IGNORE_BKEY = 2, 100251881Speter MLX5_MAD_IFC_NET_VIEW = 4, 101251881Speter}; 102251881Speter 103251881Speterenum { 104251881Speter MLX5_CROSS_CHANNEL_UUAR = 0, 105251881Speter}; 106251881Speter 107251881Speterenum { 108251881Speter MLX5_IB_MAX_CTX_DYNAMIC_UARS = 256, 109251881Speter MLX5_IB_INVALID_UAR_INDEX = -1U 110251881Speter}; 111251881Speter 112251881Speterenum { 113251881Speter MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES = 13, 114251881Speter MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES = 6, 115251881Speter MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES = 16, 116251881Speter MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES = 9, 117251881Speter}; 118251881Speter 119251881Speterstruct mlx5_ib_ucontext { 120251881Speter struct ib_ucontext ibucontext; 121251881Speter struct list_head db_page_list; 122251881Speter 123251881Speter /* protect doorbell record alloc/free 124251881Speter */ 125251881Speter struct mutex db_page_mutex; 126251881Speter struct mlx5_uuar_info uuari; 127251881Speter u32 dynamic_wc_uar_index[MLX5_IB_MAX_CTX_DYNAMIC_UARS]; 128251881Speter /* Transport Domain number */ 129251881Speter u32 tdn; 130251881Speter}; 131251881Speter 132251881Speterstatic inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext) 133251881Speter{ 134251881Speter return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext); 135251881Speter} 136251881Speter 137251881Speterstruct mlx5_ib_pd { 138251881Speter struct ib_pd ibpd; 139251881Speter u32 pdn; 140251881Speter u32 pa_lkey; 141251881Speter}; 142251881Speter 143251881Speterstruct wr_list { 144251881Speter u16 opcode; 145251881Speter u16 next; 146251881Speter}; 147251881Speter 148251881Speterstruct mlx5_swr_ctx { 149251881Speter u64 wrid; 150251881Speter u32 wr_data; 151251881Speter struct wr_list w_list; 152251881Speter u32 wqe_head; 153251881Speter u8 sig_piped; 154251881Speter u8 rsvd[11]; 155251881Speter}; 156251881Speter 157251881Speterstruct mlx5_rwr_ctx { 158251881Speter u64 wrid; 159251881Speter}; 160251881Speter 161251881Speterstruct mlx5_ib_wq { 162251881Speter union { 163251881Speter struct mlx5_swr_ctx *swr_ctx; 164251881Speter struct mlx5_rwr_ctx *rwr_ctx; 165251881Speter }; 166251881Speter u16 unsig_count; 167251881Speter 168251881Speter /* serialize post to the work queue 169251881Speter */ 170251881Speter spinlock_t lock; 171251881Speter int wqe_cnt; 172251881Speter int max_post; 173251881Speter int max_gs; 174251881Speter int offset; 175251881Speter int wqe_shift; 176251881Speter unsigned head; 177251881Speter unsigned tail; 178251881Speter u16 cur_post; 179251881Speter u16 last_poll; 180251881Speter void *qend; 181251881Speter}; 182251881Speter 183251881Speterenum { 184251881Speter MLX5_QP_USER, 185251881Speter MLX5_QP_KERNEL, 186251881Speter MLX5_QP_EMPTY 187251881Speter}; 188251881Speter 189251881Speterenum { 190251881Speter MLX5_WQ_USER, 191251881Speter MLX5_WQ_KERNEL 192251881Speter}; 193251881Speter 194251881Speterstruct mlx5_ib_sqd { 195251881Speter struct mlx5_ib_qp *qp; 196251881Speter struct work_struct work; 197251881Speter}; 198251881Speter 199251881Speterstruct mlx5_ib_mc_flows_list { 200251881Speter struct list_head flows_list; 201251881Speter /*Protect the flows_list*/ 202251881Speter struct mutex lock; 203251881Speter}; 204251881Speter 205251881Speterstruct mlx5_ib_qp { 206251881Speter struct ib_qp ibqp; 207251881Speter struct mlx5_core_qp mqp; 208251881Speter struct mlx5_core_qp mrq; 209251881Speter struct mlx5_core_qp msq; 210251881Speter u32 tisn; 211251881Speter u32 tirn; 212251881Speter struct mlx5_buf buf; 213251881Speter 214251881Speter struct mlx5_db db; 215251881Speter struct mlx5_ib_wq rq; 216251881Speter 217251881Speter u32 doorbell_qpn; 218251881Speter u8 sq_signal_bits; 219251881Speter u8 fm_cache; 220251881Speter int sq_max_wqes_per_wr; 221251881Speter int sq_spare_wqes; 222251881Speter struct mlx5_ib_wq sq; 223251881Speter 224251881Speter struct ib_umem *umem; 225251881Speter int buf_size; 226251881Speter /* Raw Ethernet QP's SQ is allocated seperately 227251881Speter * from the RQ's buffer in user-space. 228251881Speter */ 229251881Speter struct ib_umem *sq_umem; 230251881Speter int sq_buf_size; 231251881Speter u64 sq_buf_addr; 232251881Speter int allow_mp_wqe; 233251881Speter 234251881Speter /* serialize qp state modifications 235251881Speter */ 236251881Speter struct mutex mutex; 237251881Speter u16 xrcdn; 238251881Speter u32 flags; 239251881Speter u8 port; 240251881Speter u8 alt_port; 241251881Speter u8 atomic_rd_en; 242251881Speter u8 resp_depth; 243251881Speter u8 state; 244251881Speter /* Raw Ethernet QP's SQ and RQ states */ 245251881Speter u8 rq_state; 246251881Speter u8 sq_state; 247251881Speter int mlx_type; 248251881Speter int wq_sig; 249251881Speter int scat_cqe; 250251881Speter int max_inline_data; 251251881Speter struct mlx5_bf *bf; 252251881Speter int has_rq; 253251881Speter 254251881Speter /* only for user space QPs. For kernel 255251881Speter * we have it from the bf object 256251881Speter */ 257251881Speter int uuarn; 258251881Speter 259251881Speter int create_type; 260251881Speter u32 pa_lkey; 261251881Speter 262251881Speter /* Store signature errors */ 263251881Speter bool signature_en; 264251881Speter 265251881Speter struct list_head qps_list; 266251881Speter struct list_head cq_recv_list; 267251881Speter struct list_head cq_send_list; 268251881Speter 269251881Speter struct mlx5_ib_mc_flows_list mc_flows_list; 270251881Speter}; 271251881Speter 272251881Speterstruct mlx5_ib_cq_buf { 273251881Speter struct mlx5_buf buf; 274251881Speter struct ib_umem *umem; 275251881Speter int cqe_size; 276251881Speter int nent; 277251881Speter}; 278251881Speter 279251881Speterenum mlx5_ib_qp_flags { 280251881Speter MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = 1 << 0, 281251881Speter MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 1, 282251881Speter MLX5_IB_QP_CAP_RX_END_PADDING = 1 << 5, 283251881Speter}; 284251881Speter 285251881Speterstruct mlx5_umr_wr { 286251881Speter union { 287251881Speter u64 virt_addr; 288251881Speter u64 offset; 289251881Speter } target; 290251881Speter struct ib_pd *pd; 291251881Speter unsigned int page_shift; 292251881Speter unsigned int npages; 293251881Speter u64 length; 294251881Speter int access_flags; 295251881Speter u32 mkey; 296251881Speter}; 297251881Speter 298251881Speterstruct mlx5_shared_mr_info { 299251881Speter int mr_id; 300251881Speter struct ib_umem *umem; 301251881Speter}; 302251881Speter 303251881Speterstruct mlx5_ib_cq { 304251881Speter struct ib_cq ibcq; 305251881Speter struct mlx5_core_cq mcq; 306251881Speter struct mlx5_ib_cq_buf buf; 307251881Speter struct mlx5_db db; 308251881Speter 309251881Speter /* serialize access to the CQ 310251881Speter */ 311251881Speter spinlock_t lock; 312251881Speter 313251881Speter /* protect resize cq 314251881Speter */ 315251881Speter struct mutex resize_mutex; 316251881Speter struct mlx5_ib_cq_buf *resize_buf; 317251881Speter struct ib_umem *resize_umem; 318251881Speter int cqe_size; 319251881Speter struct list_head list_send_qp; 320251881Speter struct list_head list_recv_qp; 321251881Speter}; 322251881Speter 323251881Speterstruct mlx5_ib_srq { 324251881Speter struct ib_srq ibsrq; 325251881Speter struct mlx5_core_srq msrq; 326251881Speter struct mlx5_buf buf; 327251881Speter struct mlx5_db db; 328251881Speter u64 *wrid; 329251881Speter /* protect SRQ hanlding 330251881Speter */ 331251881Speter spinlock_t lock; 332251881Speter int head; 333251881Speter int tail; 334251881Speter u16 wqe_ctr; 335251881Speter struct ib_umem *umem; 336251881Speter /* serialize arming a SRQ 337251881Speter */ 338251881Speter struct mutex mutex; 339251881Speter int wq_sig; 340251881Speter}; 341251881Speter 342251881Speterstruct mlx5_ib_xrcd { 343251881Speter struct ib_xrcd ibxrcd; 344251881Speter u32 xrcdn; 345251881Speter}; 346251881Speter 347251881Speterenum mlx5_ib_mtt_access_flags { 348251881Speter MLX5_IB_MTT_READ = (1 << 0), 349251881Speter MLX5_IB_MTT_WRITE = (1 << 1), 350251881Speter}; 351251881Speter 352251881Speter#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE) 353251881Speter 354251881Speterstruct mlx5_ib_mr { 355251881Speter struct ib_mr ibmr; 356251881Speter struct mlx5_core_mr mmr; 357251881Speter struct ib_umem *umem; 358251881Speter struct mlx5_shared_mr_info *smr_info; 359251881Speter struct list_head list; 360251881Speter int order; 361251881Speter int umred; 362251881Speter dma_addr_t dma; 363251881Speter int npages; 364251881Speter struct mlx5_ib_dev *dev; 365251881Speter struct mlx5_create_mkey_mbox_out out; 366251881Speter struct mlx5_core_sig_ctx *sig; 367251881Speter u32 max_reg_descriptors; 368251881Speter u64 size; 369251881Speter u64 page_count; 370251881Speter struct mlx5_ib_mr **children; 371251881Speter int nchild; 372251881Speter}; 373251881Speter 374251881Speterstruct mlx5_ib_fast_reg_page_list { 375251881Speter struct ib_fast_reg_page_list ibfrpl; 376251881Speter __be64 *mapped_page_list; 377251881Speter dma_addr_t map; 378251881Speter}; 379251881Speter 380251881Speterstruct mlx5_ib_umr_context { 381251881Speter enum ib_wc_status status; 382251881Speter struct completion done; 383251881Speter}; 384251881Speter 385251881Speterstatic inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 386251881Speter{ 387251881Speter context->status = -1; 388251881Speter init_completion(&context->done); 389251881Speter} 390251881Speter 391251881Speterstruct umr_common { 392251881Speter struct ib_pd *pd; 393251881Speter struct ib_mr *mr; 394251881Speter}; 395251881Speter 396251881Speterenum { 397251881Speter MLX5_FMR_INVALID, 398251881Speter MLX5_FMR_VALID, 399251881Speter MLX5_FMR_BUSY, 400251881Speter}; 401251881Speter 402251881Speterstruct mlx5_ib_fmr { 403251881Speter struct ib_fmr ibfmr; 404251881Speter struct mlx5_core_mr mr; 405251881Speter int access_flags; 406251881Speter int state; 407251881Speter /* protect fmr state 408251881Speter */ 409251881Speter spinlock_t lock; 410251881Speter u64 wrid; 411251881Speter struct ib_send_wr wr[2]; 412251881Speter u8 page_shift; 413251881Speter struct ib_fast_reg_page_list page_list; 414251881Speter}; 415251881Speter 416251881Speterstruct cache_order { 417251881Speter struct kobject kobj; 418251881Speter int order; 419251881Speter int index; 420251881Speter struct mlx5_ib_dev *dev; 421251881Speter}; 422251881Speter 423251881Speterstruct mlx5_cache_ent { 424251881Speter struct list_head head; 425251881Speter /* sync access to the cahce entry 426251881Speter */ 427251881Speter spinlock_t lock; 428251881Speter 429251881Speter 430251881Speter u32 order; 431 u32 size; 432 u32 cur; 433 u32 miss; 434 u32 limit; 435 436 struct mlx5_ib_dev *dev; 437 struct work_struct work; 438 struct delayed_work dwork; 439 int pending; 440 struct cache_order co; 441}; 442 443struct mlx5_mr_cache { 444 struct workqueue_struct *wq; 445 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES]; 446 int stopped; 447 struct dentry *root; 448 int last_add; 449 int rel_timeout; 450 int rel_imm; 451}; 452 453struct mlx5_ib_resources { 454 struct ib_cq *c0; 455 struct ib_xrcd *x0; 456 struct ib_xrcd *x1; 457 struct ib_pd *p0; 458 struct ib_srq *s0; 459 struct ib_srq *s1; 460}; 461 462struct mlx5_dc_tracer { 463 struct page *pg; 464 dma_addr_t dma; 465 int size; 466 int order; 467}; 468 469struct mlx5_dc_desc { 470 dma_addr_t dma; 471 void *buf; 472}; 473 474enum mlx5_op { 475 MLX5_WR_OP_MLX = 1, 476}; 477 478struct mlx5_mlx_wr { 479 u8 sl; 480 u16 dlid; 481 int icrc; 482}; 483 484struct mlx5_send_wr { 485 struct ib_send_wr wr; 486 union { 487 struct mlx5_mlx_wr mlx; 488 } sel; 489}; 490 491struct mlx5_dc_data { 492 struct ib_mr *mr; 493 struct ib_qp *dcqp; 494 struct ib_cq *rcq; 495 struct ib_cq *scq; 496 unsigned int rx_npages; 497 unsigned int tx_npages; 498 struct mlx5_dc_desc *rxdesc; 499 struct mlx5_dc_desc *txdesc; 500 unsigned int max_wqes; 501 unsigned int cur_send; 502 unsigned int last_send_completed; 503 int tx_pending; 504 struct mlx5_ib_dev *dev; 505 int port; 506 int initialized; 507 struct kobject kobj; 508 unsigned long connects; 509 unsigned long cnaks; 510 unsigned long discards; 511 struct ib_wc wc_tbl[MLX5_CNAK_RX_POLL_CQ_QUOTA]; 512}; 513 514struct mlx5_ib_port_sysfs_group { 515 struct kobject kobj; 516 bool enabled; 517 struct attribute_group counters; 518}; 519 520#define MLX5_IB_GID_MAX 16 521 522struct mlx5_ib_port { 523 struct mlx5_ib_dev *dev; 524 u8 port_num; /* 0 based */ 525 u8 port_gone; /* set when gone */ 526 u16 q_cnt_id; 527 struct mlx5_ib_port_sysfs_group group; 528 union ib_gid gid_table[MLX5_IB_GID_MAX]; 529}; 530 531struct mlx5_ib_dev { 532 struct ib_device ib_dev; 533 struct mlx5_core_dev *mdev; 534 MLX5_DECLARE_DOORBELL_LOCK(uar_lock); 535 int num_ports; 536 /* serialize update of capability mask 537 */ 538 struct mutex cap_mask_mutex; 539 bool ib_active; 540 struct umr_common umrc; 541 /* sync used page count stats 542 */ 543 struct mlx5_ib_resources devr; 544 struct mutex slow_path_mutex; 545 int enable_atomic_resp; 546 enum ib_atomic_cap atomic_cap; 547 struct mlx5_mr_cache cache; 548 struct kobject mr_cache; 549 /* protect resources needed as part of reset flow */ 550 spinlock_t reset_flow_resource_lock; 551 struct list_head qp_list; 552 struct timer_list delay_timer; 553 int fill_delay; 554 struct mlx5_dc_tracer dctr; 555 struct mlx5_dc_data dcd[MLX5_MAX_PORTS]; 556 struct kobject *dc_kobj; 557 /* Array with num_ports elements */ 558 struct mlx5_ib_port *port; 559 struct kobject *ports_parent; 560}; 561 562static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq) 563{ 564 return container_of(mcq, struct mlx5_ib_cq, mcq); 565} 566 567static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd) 568{ 569 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd); 570} 571 572static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev) 573{ 574 return container_of(ibdev, struct mlx5_ib_dev, ib_dev); 575} 576 577static inline struct mlx5_ib_fmr *to_mfmr(struct ib_fmr *ibfmr) 578{ 579 return container_of(ibfmr, struct mlx5_ib_fmr, ibfmr); 580} 581 582static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq) 583{ 584 return container_of(ibcq, struct mlx5_ib_cq, ibcq); 585} 586 587static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp) 588{ 589 return container_of(mqp, struct mlx5_ib_qp, mqp); 590} 591 592static inline struct mlx5_ib_qp *sq_to_mibqp(struct mlx5_core_qp *msq) 593{ 594 return container_of(msq, struct mlx5_ib_qp, msq); 595} 596 597static inline struct mlx5_ib_qp *rq_to_mibqp(struct mlx5_core_qp *mrq) 598{ 599 return container_of(mrq, struct mlx5_ib_qp, mrq); 600} 601 602static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmr) 603{ 604 return container_of(mmr, struct mlx5_ib_mr, mmr); 605} 606 607static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd) 608{ 609 return container_of(ibpd, struct mlx5_ib_pd, ibpd); 610} 611 612static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq) 613{ 614 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq); 615} 616 617static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp) 618{ 619 return container_of(ibqp, struct mlx5_ib_qp, ibqp); 620} 621 622static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq) 623{ 624 return container_of(msrq, struct mlx5_ib_srq, msrq); 625} 626 627static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr) 628{ 629 return container_of(ibmr, struct mlx5_ib_mr, ibmr); 630} 631 632static inline struct mlx5_ib_fast_reg_page_list *to_mfrpl(struct ib_fast_reg_page_list *ibfrpl) 633{ 634 return container_of(ibfrpl, struct mlx5_ib_fast_reg_page_list, ibfrpl); 635} 636 637struct mlx5_ib_ah { 638 struct ib_ah ibah; 639 struct mlx5_av av; 640}; 641 642static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah) 643{ 644 return container_of(ibah, struct mlx5_ib_ah, ibah); 645} 646 647int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, uintptr_t virt, 648 struct mlx5_db *db); 649void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db); 650void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); 651void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq); 652void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index); 653int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey, 654 u8 port, struct ib_wc *in_wc, struct ib_grh *in_grh, 655 void *in_mad, void *response_mad); 656int mlx5_ib_resolve_grh(const struct ib_ah_attr *ah_attr, u8 *mac, int *is_mcast); 657struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev, struct ib_ah_attr *ah_attr, 658 struct mlx5_ib_ah *ah, enum rdma_link_layer ll); 659struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr); 660int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr); 661int mlx5_ib_destroy_ah(struct ib_ah *ah); 662struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, 663 struct ib_srq_init_attr *init_attr, 664 struct ib_udata *udata); 665int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 666 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); 667int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); 668int mlx5_ib_destroy_srq(struct ib_srq *srq); 669int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 670 struct ib_recv_wr **bad_wr); 671struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, 672 struct ib_qp_init_attr *init_attr, 673 struct ib_udata *udata); 674int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 675 int attr_mask, struct ib_udata *udata); 676int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, 677 struct ib_qp_init_attr *qp_init_attr); 678int mlx5_ib_destroy_qp(struct ib_qp *qp); 679int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 680 struct ib_send_wr **bad_wr); 681int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, 682 struct ib_recv_wr **bad_wr); 683void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n); 684struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, 685 int entries, int vector, 686 struct ib_ucontext *context, 687 struct ib_udata *udata); 688int mlx5_ib_destroy_cq(struct ib_cq *cq); 689int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); 690int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); 691int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); 692int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); 693struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc); 694struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 695 u64 virt_addr, int access_flags, 696 struct ib_udata *udata, int mr_id); 697struct ib_mr *mlx5_ib_reg_phys_mr(struct ib_pd *pd, 698 struct ib_phys_buf *buffer_list, 699 int num_phys_buf, 700 int access_flags, 701 u64 *virt_addr); 702int mlx5_ib_dereg_mr(struct ib_mr *ibmr); 703int mlx5_ib_destroy_mr(struct ib_mr *ibmr); 704struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd, 705 int max_page_list_len); 706struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev, 707 int page_list_len); 708void mlx5_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list); 709 710struct ib_fmr *mlx5_ib_fmr_alloc(struct ib_pd *pd, int acc, 711 struct ib_fmr_attr *fmr_attr); 712int mlx5_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 713 int npages, u64 iova); 714int mlx5_ib_unmap_fmr(struct list_head *fmr_list); 715int mlx5_ib_fmr_dealloc(struct ib_fmr *ibfmr); 716int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, 717 struct ib_wc *in_wc, struct ib_grh *in_grh, 718 struct ib_mad *in_mad, struct ib_mad *out_mad); 719struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev, 720 struct ib_ucontext *context, 721 struct ib_udata *udata); 722int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd); 723int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset); 724int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port); 725int mlx5_query_smp_attr_node_info_mad_ifc(struct ib_device *ibdev, 726 struct ib_smp *out_mad); 727int mlx5_query_system_image_guid_mad_ifc(struct ib_device *ibdev, 728 __be64 *sys_image_guid); 729int mlx5_query_max_pkeys_mad_ifc(struct ib_device *ibdev, 730 u16 *max_pkeys); 731int mlx5_query_vendor_id_mad_ifc(struct ib_device *ibdev, 732 u32 *vendor_id); 733int mlx5_query_pkey_mad_ifc(struct ib_device *ibdev, u8 port, u16 index, 734 u16 *pkey); 735int mlx5_query_node_desc_mad_ifc(struct mlx5_ib_dev *dev, char *node_desc); 736int mlx5_query_node_guid_mad_ifc(struct mlx5_ib_dev *dev, u64 *node_guid); 737int mlx5_query_gids_mad_ifc(struct ib_device *ibdev, u8 port, int index, 738 union ib_gid *gid); 739int mlx5_query_port_mad_ifc(struct ib_device *ibdev, u8 port, 740 struct ib_port_attr *props); 741int mlx5_ib_query_port(struct ib_device *ibdev, u8 port, 742 struct ib_port_attr *props); 743int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev); 744void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev); 745void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift, 746 int *ncont, int *order); 747void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 748 int page_shift, __be64 *pas, int umr); 749void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num); 750int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq); 751int mlx5_mr_cache_init(struct mlx5_ib_dev *dev); 752int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); 753int mlx5_mr_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift); 754void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context); 755int mlx5_query_port_roce(struct ib_device *ibdev, u8 port, 756 struct ib_port_attr *props); 757__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port, int index, 758 __be16 ah_udp_s_port); 759int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port, 760 int index, int *gid_type); 761struct net_device *mlx5_ib_get_netdev(struct ib_device *ib_dev, u8 port); 762int modify_gid_roce(struct ib_device *ib_dev, u8 port, unsigned int index, 763 const union ib_gid *gid, struct net_device *ndev); 764int query_gid_roce(struct ib_device *ib_dev, u8 port, int index, 765 union ib_gid *gid); 766int mlx5_process_mad_mad_ifc(struct ib_device *ibdev, int mad_flags, 767 u8 port_num, struct ib_wc *in_wc, 768 struct ib_grh *in_grh, struct ib_mad *in_mad, 769 struct ib_mad *out_mad); 770 771static inline void init_query_mad(struct ib_smp *mad) 772{ 773 mad->base_version = 1; 774 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; 775 mad->class_version = 1; 776 mad->method = IB_MGMT_METHOD_GET; 777} 778 779static inline u8 convert_access(int acc) 780{ 781 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) | 782 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | 783 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | 784 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | 785 MLX5_PERM_LOCAL_READ; 786} 787 788#define MLX5_MAX_UMR_SHIFT 16 789#define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT) 790 791#endif /* MLX5_IB_H */ 792