1/* 2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#ifndef MLX5_H 34#define MLX5_H 35 36#include <stddef.h> 37#include <stdio.h> 38#include <stdatomic.h> 39#include <stdbool.h> 40 41#include <sys/queue.h> 42 43#include <infiniband/driver.h> 44#include <infiniband/udma_barrier.h> 45#include "mlx5-abi.h" 46#include "bitmap.h" 47#include "mlx5dv.h" 48 49#define PFX "mlx5: " 50 51 52enum { 53 MLX5_IB_MMAP_CMD_SHIFT = 8, 54 MLX5_IB_MMAP_CMD_MASK = 0xff, 55}; 56 57enum { 58 MLX5_MMAP_GET_REGULAR_PAGES_CMD = 0, 59 MLX5_MMAP_GET_CONTIGUOUS_PAGES_CMD = 1, 60 MLX5_MMAP_GET_CORE_CLOCK_CMD = 5 61}; 62 63enum { 64 MLX5_CQE_VERSION_V0 = 0, 65 MLX5_CQE_VERSION_V1 = 1, 66}; 67 68enum { 69 MLX5_ADAPTER_PAGE_SIZE = 4096, 70}; 71 72#define MLX5_CQ_PREFIX "MLX_CQ" 73#define MLX5_QP_PREFIX "MLX_QP" 74#define MLX5_MR_PREFIX "MLX_MR" 75#define MLX5_RWQ_PREFIX "MLX_RWQ" 76#define MLX5_MAX_LOG2_CONTIG_BLOCK_SIZE 23 77#define MLX5_MIN_LOG2_CONTIG_BLOCK_SIZE 12 78 79enum { 80 MLX5_DBG_QP = 1 << 0, 81 MLX5_DBG_CQ = 1 << 1, 82 MLX5_DBG_QP_SEND = 1 << 2, 83 MLX5_DBG_QP_SEND_ERR = 1 << 3, 84 MLX5_DBG_CQ_CQE = 1 << 4, 85 MLX5_DBG_CONTIG = 1 << 5, 86}; 87 88extern uint32_t mlx5_debug_mask; 89extern int mlx5_freeze_on_error_cqe; 90 91#ifdef MLX5_DEBUG 92#define mlx5_dbg(fp, mask, format, arg...) \ 93do { \ 94 if (mask & mlx5_debug_mask) \ 95 fprintf(fp, "%s:%d: " format, __func__, __LINE__, ##arg); \ 96} while (0) 97 98#else 99static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...) 100 __attribute__((format(printf, 3, 4))); 101static inline void mlx5_dbg(FILE *fp, uint32_t mask, const char *fmt, ...) 102{ 103} 104#endif 105 106enum { 107 MLX5_STAT_RATE_OFFSET = 5 108}; 109 110enum { 111 MLX5_QP_TABLE_SHIFT = 12, 112 MLX5_QP_TABLE_MASK = (1 << MLX5_QP_TABLE_SHIFT) - 1, 113 MLX5_QP_TABLE_SIZE = 1 << (24 - MLX5_QP_TABLE_SHIFT), 114}; 115 116enum { 117 MLX5_UIDX_TABLE_SHIFT = 12, 118 MLX5_UIDX_TABLE_MASK = (1 << MLX5_UIDX_TABLE_SHIFT) - 1, 119 MLX5_UIDX_TABLE_SIZE = 1 << (24 - MLX5_UIDX_TABLE_SHIFT), 120}; 121 122enum { 123 MLX5_SRQ_TABLE_SHIFT = 12, 124 MLX5_SRQ_TABLE_MASK = (1 << MLX5_SRQ_TABLE_SHIFT) - 1, 125 MLX5_SRQ_TABLE_SIZE = 1 << (24 - MLX5_SRQ_TABLE_SHIFT), 126}; 127 128enum { 129 MLX5_BF_OFFSET = 0x800 130}; 131 132enum { 133 MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, 134 MLX5_RECV_OPCODE_SEND = 0x01, 135 MLX5_RECV_OPCODE_SEND_IMM = 0x02, 136 MLX5_RECV_OPCODE_SEND_INVAL = 0x03, 137 138 MLX5_CQE_OPCODE_ERROR = 0x1e, 139 MLX5_CQE_OPCODE_RESIZE = 0x16, 140}; 141 142enum { 143 MLX5_SRQ_FLAG_SIGNATURE = 1 << 0, 144}; 145 146enum { 147 MLX5_MAX_PORTS_NUM = 2, 148}; 149 150enum { 151 MLX5_CSUM_SUPPORT_RAW_OVER_ETH = (1 << 0), 152 /* 153 * Only report rx checksum when the validation 154 * is valid. 155 */ 156 MLX5_RX_CSUM_VALID = (1 << 16), 157}; 158 159enum mlx5_alloc_type { 160 MLX5_ALLOC_TYPE_ANON, 161 MLX5_ALLOC_TYPE_HUGE, 162 MLX5_ALLOC_TYPE_CONTIG, 163 MLX5_ALLOC_TYPE_PREFER_HUGE, 164 MLX5_ALLOC_TYPE_PREFER_CONTIG, 165 MLX5_ALLOC_TYPE_ALL 166}; 167 168enum mlx5_rsc_type { 169 MLX5_RSC_TYPE_QP, 170 MLX5_RSC_TYPE_XSRQ, 171 MLX5_RSC_TYPE_SRQ, 172 MLX5_RSC_TYPE_RWQ, 173 MLX5_RSC_TYPE_INVAL, 174}; 175 176enum { 177 MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE = 1 << 0, 178 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH = 1 << 1, 179}; 180 181enum mlx5_vendor_cap_flags { 182 MLX5_VENDOR_CAP_FLAGS_MPW = 1 << 0, 183}; 184 185enum { 186 MLX5_FLOW_TAG_MASK = 0x000fffff, 187}; 188 189struct mlx5_resource { 190 enum mlx5_rsc_type type; 191 uint32_t rsn; 192}; 193 194struct mlx5_device { 195 struct verbs_device verbs_dev; 196 int page_size; 197 int driver_abi_ver; 198}; 199 200struct mlx5_db_page; 201 202struct mlx5_spinlock { 203 pthread_spinlock_t lock; 204 int in_use; 205}; 206 207struct mlx5_hugetlb_mem; 208 209struct mlx5_context { 210 struct ibv_context ibv_ctx; 211 int max_num_qps; 212 int bf_reg_size; 213 int tot_uuars; 214 int low_lat_uuars; 215 int num_uars_per_page; 216 int bf_regs_per_page; 217 int num_bf_regs; 218 int prefer_bf; 219 int shut_up_bf; 220 struct { 221 struct mlx5_qp **table; 222 int refcnt; 223 } qp_table[MLX5_QP_TABLE_SIZE]; 224 pthread_mutex_t qp_table_mutex; 225 226 struct { 227 struct mlx5_srq **table; 228 int refcnt; 229 } srq_table[MLX5_SRQ_TABLE_SIZE]; 230 pthread_mutex_t srq_table_mutex; 231 232 struct { 233 struct mlx5_resource **table; 234 int refcnt; 235 } uidx_table[MLX5_UIDX_TABLE_SIZE]; 236 pthread_mutex_t uidx_table_mutex; 237 238 void *uar[MLX5_MAX_UARS]; 239 struct mlx5_spinlock lock32; 240 struct mlx5_db_page *db_list; 241 pthread_mutex_t db_list_mutex; 242 int cache_line_size; 243 int max_sq_desc_sz; 244 int max_rq_desc_sz; 245 int max_send_wqebb; 246 int max_recv_wr; 247 unsigned max_srq_recv_wr; 248 int num_ports; 249 int stall_enable; 250 int stall_adaptive_enable; 251 int stall_cycles; 252 struct mlx5_bf *bfs; 253 FILE *dbg_fp; 254 char hostname[40]; 255 struct mlx5_spinlock hugetlb_lock; 256 TAILQ_HEAD(,mlx5_hugetlb_mem) hugetlb_list; 257 int cqe_version; 258 uint8_t cached_link_layer[MLX5_MAX_PORTS_NUM]; 259 int cached_device_cap_flags; 260 enum ibv_atomic_cap atomic_cap; 261 struct { 262 uint64_t offset; 263 uint64_t mask; 264 } core_clock; 265 void *hca_core_clock; 266 struct ibv_tso_caps cached_tso_caps; 267 int cmds_supp_uhw; 268 uint32_t uar_size; 269 uint64_t vendor_cap_flags; /* Use enum mlx5_vendor_cap_flags */ 270 struct mlx5dv_cqe_comp_caps cqe_comp_caps; 271}; 272 273struct mlx5_bitmap { 274 uint32_t last; 275 uint32_t top; 276 uint32_t max; 277 uint32_t avail; 278 uint32_t mask; 279 unsigned long *table; 280}; 281 282struct mlx5_hugetlb_mem { 283 int shmid; 284 void *shmaddr; 285 struct mlx5_bitmap bitmap; 286 TAILQ_ENTRY(mlx5_hugetlb_mem) entry; 287}; 288 289struct mlx5_buf { 290 void *buf; 291 size_t length; 292 int base; 293 struct mlx5_hugetlb_mem *hmem; 294 enum mlx5_alloc_type type; 295}; 296 297struct mlx5_pd { 298 struct ibv_pd ibv_pd; 299 uint32_t pdn; 300}; 301 302enum { 303 MLX5_CQ_SET_CI = 0, 304 MLX5_CQ_ARM_DB = 1, 305}; 306 307enum { 308 MLX5_CQ_FLAGS_RX_CSUM_VALID = 1 << 0, 309 MLX5_CQ_FLAGS_EMPTY_DURING_POLL = 1 << 1, 310 MLX5_CQ_FLAGS_FOUND_CQES = 1 << 2, 311 MLX5_CQ_FLAGS_EXTENDED = 1 << 3, 312 MLX5_CQ_FLAGS_SINGLE_THREADED = 1 << 4, 313 MLX5_CQ_FLAGS_DV_OWNED = 1 << 5, 314}; 315 316struct mlx5_cq { 317 /* ibv_cq should always be subset of ibv_cq_ex */ 318 struct ibv_cq_ex ibv_cq; 319 struct mlx5_buf buf_a; 320 struct mlx5_buf buf_b; 321 struct mlx5_buf *active_buf; 322 struct mlx5_buf *resize_buf; 323 int resize_cqes; 324 int active_cqes; 325 struct mlx5_spinlock lock; 326 uint32_t cqn; 327 uint32_t cons_index; 328 uint32_t *dbrec; 329 int arm_sn; 330 int cqe_sz; 331 int resize_cqe_sz; 332 int stall_next_poll; 333 int stall_enable; 334 uint64_t stall_last_count; 335 int stall_adaptive_enable; 336 int stall_cycles; 337 struct mlx5_resource *cur_rsc; 338 struct mlx5_srq *cur_srq; 339 struct mlx5_cqe64 *cqe64; 340 uint32_t flags; 341 int umr_opcode; 342}; 343 344struct mlx5_srq { 345 struct mlx5_resource rsc; /* This struct must be first */ 346 struct verbs_srq vsrq; 347 struct mlx5_buf buf; 348 struct mlx5_spinlock lock; 349 uint64_t *wrid; 350 uint32_t srqn; 351 int max; 352 int max_gs; 353 int wqe_shift; 354 int head; 355 int tail; 356 uint32_t *db; 357 uint16_t counter; 358 int wq_sig; 359}; 360 361struct wr_list { 362 uint16_t opcode; 363 uint16_t next; 364}; 365 366struct mlx5_wq { 367 uint64_t *wrid; 368 unsigned *wqe_head; 369 struct mlx5_spinlock lock; 370 unsigned wqe_cnt; 371 unsigned max_post; 372 unsigned head; 373 unsigned tail; 374 unsigned cur_post; 375 int max_gs; 376 int wqe_shift; 377 int offset; 378 void *qend; 379 uint32_t *wr_data; 380}; 381 382struct mlx5_bf { 383 void *reg; 384 int need_lock; 385 struct mlx5_spinlock lock; 386 unsigned offset; 387 unsigned buf_size; 388 unsigned uuarn; 389}; 390 391struct mlx5_mr { 392 struct ibv_mr ibv_mr; 393 struct mlx5_buf buf; 394 uint32_t alloc_flags; 395}; 396 397struct mlx5_qp { 398 struct mlx5_resource rsc; /* This struct must be first */ 399 struct verbs_qp verbs_qp; 400 struct ibv_qp *ibv_qp; 401 struct mlx5_buf buf; 402 void *sq_start; 403 int max_inline_data; 404 int buf_size; 405 /* For Raw Packet QP, use different buffers for the SQ and RQ */ 406 struct mlx5_buf sq_buf; 407 int sq_buf_size; 408 struct mlx5_bf *bf; 409 410 uint8_t fm_cache; 411 uint8_t sq_signal_bits; 412 struct mlx5_wq sq; 413 414 uint32_t *db; 415 struct mlx5_wq rq; 416 int wq_sig; 417 uint32_t qp_cap_cache; 418 int atomics_enabled; 419 uint32_t max_tso; 420 uint16_t max_tso_header; 421 int rss_qp; 422}; 423 424struct mlx5_ah { 425 struct ibv_ah ibv_ah; 426 struct mlx5_wqe_av av; 427 bool kern_ah; 428}; 429 430struct mlx5_rwq { 431 struct mlx5_resource rsc; 432 struct ibv_wq wq; 433 struct mlx5_buf buf; 434 int buf_size; 435 struct mlx5_wq rq; 436 uint32_t *db; 437 void *pbuff; 438 uint32_t *recv_db; 439 int wq_sig; 440}; 441 442static inline int mlx5_ilog2(int n) 443{ 444 int t; 445 446 if (n <= 0) 447 return -1; 448 449 t = 0; 450 while ((1 << t) < n) 451 ++t; 452 453 return t; 454} 455 456extern int mlx5_stall_num_loop; 457extern int mlx5_stall_cq_poll_min; 458extern int mlx5_stall_cq_poll_max; 459extern int mlx5_stall_cq_inc_step; 460extern int mlx5_stall_cq_dec_step; 461extern int mlx5_single_threaded; 462 463static inline unsigned DIV_ROUND_UP(unsigned n, unsigned d) 464{ 465 return (n + d - 1u) / d; 466} 467 468static inline unsigned long align(unsigned long val, unsigned long align) 469{ 470 return (val + align - 1) & ~(align - 1); 471} 472 473#define to_mxxx(xxx, type) \ 474 ((struct mlx5_##type *) \ 475 ((void *) ib##xxx - offsetof(struct mlx5_##type, ibv_##xxx))) 476 477static inline struct mlx5_device *to_mdev(struct ibv_device *ibdev) 478{ 479 struct mlx5_device *ret; 480 481 ret = (void *)ibdev - offsetof(struct mlx5_device, verbs_dev); 482 return ret; 483} 484 485static inline struct mlx5_context *to_mctx(struct ibv_context *ibctx) 486{ 487 return to_mxxx(ctx, context); 488} 489 490static inline struct mlx5_pd *to_mpd(struct ibv_pd *ibpd) 491{ 492 return to_mxxx(pd, pd); 493} 494 495static inline struct mlx5_cq *to_mcq(struct ibv_cq *ibcq) 496{ 497 return to_mxxx(cq, cq); 498} 499 500static inline struct mlx5_srq *to_msrq(struct ibv_srq *ibsrq) 501{ 502 struct verbs_srq *vsrq = (struct verbs_srq *)ibsrq; 503 504 return container_of(vsrq, struct mlx5_srq, vsrq); 505} 506 507static inline struct mlx5_qp *to_mqp(struct ibv_qp *ibqp) 508{ 509 struct verbs_qp *vqp = (struct verbs_qp *)ibqp; 510 511 return container_of(vqp, struct mlx5_qp, verbs_qp); 512} 513 514static inline struct mlx5_rwq *to_mrwq(struct ibv_wq *ibwq) 515{ 516 return container_of(ibwq, struct mlx5_rwq, wq); 517} 518 519static inline struct mlx5_mr *to_mmr(struct ibv_mr *ibmr) 520{ 521 return to_mxxx(mr, mr); 522} 523 524static inline struct mlx5_ah *to_mah(struct ibv_ah *ibah) 525{ 526 return to_mxxx(ah, ah); 527} 528 529static inline int max_int(int a, int b) 530{ 531 return a > b ? a : b; 532} 533 534static inline struct mlx5_qp *rsc_to_mqp(struct mlx5_resource *rsc) 535{ 536 return (struct mlx5_qp *)rsc; 537} 538 539static inline struct mlx5_srq *rsc_to_msrq(struct mlx5_resource *rsc) 540{ 541 return (struct mlx5_srq *)rsc; 542} 543 544static inline struct mlx5_rwq *rsc_to_mrwq(struct mlx5_resource *rsc) 545{ 546 return (struct mlx5_rwq *)rsc; 547} 548 549int mlx5_alloc_buf(struct mlx5_buf *buf, size_t size, int page_size); 550void mlx5_free_buf(struct mlx5_buf *buf); 551int mlx5_alloc_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf, 552 size_t size, int page_size, const char *component); 553void mlx5_free_buf_contig(struct mlx5_context *mctx, struct mlx5_buf *buf); 554int mlx5_alloc_prefered_buf(struct mlx5_context *mctx, 555 struct mlx5_buf *buf, 556 size_t size, int page_size, 557 enum mlx5_alloc_type alloc_type, 558 const char *component); 559int mlx5_free_actual_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); 560void mlx5_get_alloc_type(const char *component, 561 enum mlx5_alloc_type *alloc_type, 562 enum mlx5_alloc_type default_alloc_type); 563int mlx5_use_huge(const char *key); 564 565uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context); 566void mlx5_free_db(struct mlx5_context *context, uint32_t *db); 567 568int mlx5_query_device(struct ibv_context *context, 569 struct ibv_device_attr *attr); 570int mlx5_query_device_ex(struct ibv_context *context, 571 const struct ibv_query_device_ex_input *input, 572 struct ibv_device_attr_ex *attr, 573 size_t attr_size); 574int mlx5_query_rt_values(struct ibv_context *context, 575 struct ibv_values_ex *values); 576struct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context, 577 struct ibv_qp_init_attr_ex *attr); 578int mlx5_query_port(struct ibv_context *context, uint8_t port, 579 struct ibv_port_attr *attr); 580 581struct ibv_pd *mlx5_alloc_pd(struct ibv_context *context); 582int mlx5_free_pd(struct ibv_pd *pd); 583 584struct ibv_mr *mlx5_reg_mr(struct ibv_pd *pd, void *addr, 585 size_t length, int access); 586int mlx5_rereg_mr(struct ibv_mr *mr, int flags, struct ibv_pd *pd, void *addr, 587 size_t length, int access); 588int mlx5_dereg_mr(struct ibv_mr *mr); 589struct ibv_mw *mlx5_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type); 590int mlx5_dealloc_mw(struct ibv_mw *mw); 591int mlx5_bind_mw(struct ibv_qp *qp, struct ibv_mw *mw, 592 struct ibv_mw_bind *mw_bind); 593 594struct ibv_cq *mlx5_create_cq(struct ibv_context *context, int cqe, 595 struct ibv_comp_channel *channel, 596 int comp_vector); 597struct ibv_cq_ex *mlx5_create_cq_ex(struct ibv_context *context, 598 struct ibv_cq_init_attr_ex *cq_attr); 599void mlx5_cq_fill_pfns(struct mlx5_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr); 600int mlx5_alloc_cq_buf(struct mlx5_context *mctx, struct mlx5_cq *cq, 601 struct mlx5_buf *buf, int nent, int cqe_sz); 602int mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf); 603int mlx5_resize_cq(struct ibv_cq *cq, int cqe); 604int mlx5_destroy_cq(struct ibv_cq *cq); 605int mlx5_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 606int mlx5_poll_cq_v1(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 607int mlx5_arm_cq(struct ibv_cq *cq, int solicited); 608void mlx5_cq_event(struct ibv_cq *cq); 609void __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq); 610void mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq); 611void mlx5_cq_resize_copy_cqes(struct mlx5_cq *cq); 612 613struct ibv_srq *mlx5_create_srq(struct ibv_pd *pd, 614 struct ibv_srq_init_attr *attr); 615int mlx5_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, 616 int mask); 617int mlx5_query_srq(struct ibv_srq *srq, 618 struct ibv_srq_attr *attr); 619int mlx5_destroy_srq(struct ibv_srq *srq); 620int mlx5_alloc_srq_buf(struct ibv_context *context, struct mlx5_srq *srq); 621void mlx5_free_srq_wqe(struct mlx5_srq *srq, int ind); 622int mlx5_post_srq_recv(struct ibv_srq *ibsrq, 623 struct ibv_recv_wr *wr, 624 struct ibv_recv_wr **bad_wr); 625 626struct ibv_qp *mlx5_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); 627int mlx5_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 628 int attr_mask, 629 struct ibv_qp_init_attr *init_attr); 630int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 631 int attr_mask); 632int mlx5_destroy_qp(struct ibv_qp *qp); 633void mlx5_init_qp_indices(struct mlx5_qp *qp); 634void mlx5_init_rwq_indices(struct mlx5_rwq *rwq); 635int mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 636 struct ibv_send_wr **bad_wr); 637int mlx5_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 638 struct ibv_recv_wr **bad_wr); 639int mlx5_post_wq_recv(struct ibv_wq *ibwq, struct ibv_recv_wr *wr, 640 struct ibv_recv_wr **bad_wr); 641void mlx5_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type, 642 struct mlx5_qp *qp); 643void mlx5_set_sq_sizes(struct mlx5_qp *qp, struct ibv_qp_cap *cap, 644 enum ibv_qp_type type); 645struct mlx5_qp *mlx5_find_qp(struct mlx5_context *ctx, uint32_t qpn); 646int mlx5_store_qp(struct mlx5_context *ctx, uint32_t qpn, struct mlx5_qp *qp); 647void mlx5_clear_qp(struct mlx5_context *ctx, uint32_t qpn); 648int32_t mlx5_store_uidx(struct mlx5_context *ctx, void *rsc); 649void mlx5_clear_uidx(struct mlx5_context *ctx, uint32_t uidx); 650struct mlx5_srq *mlx5_find_srq(struct mlx5_context *ctx, uint32_t srqn); 651int mlx5_store_srq(struct mlx5_context *ctx, uint32_t srqn, 652 struct mlx5_srq *srq); 653void mlx5_clear_srq(struct mlx5_context *ctx, uint32_t srqn); 654struct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); 655int mlx5_destroy_ah(struct ibv_ah *ah); 656int mlx5_alloc_av(struct mlx5_pd *pd, struct ibv_ah_attr *attr, 657 struct mlx5_ah *ah); 658void mlx5_free_av(struct mlx5_ah *ah); 659int mlx5_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); 660int mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); 661int mlx5_round_up_power_of_two(long long sz); 662void *mlx5_get_atomic_laddr(struct mlx5_qp *qp, uint16_t idx, int *byte_count); 663void *mlx5_get_send_wqe(struct mlx5_qp *qp, int n); 664int mlx5_copy_to_recv_wqe(struct mlx5_qp *qp, int idx, void *buf, int size); 665int mlx5_copy_to_send_wqe(struct mlx5_qp *qp, int idx, void *buf, int size); 666int mlx5_copy_to_recv_srq(struct mlx5_srq *srq, int idx, void *buf, int size); 667struct ibv_xrcd *mlx5_open_xrcd(struct ibv_context *context, 668 struct ibv_xrcd_init_attr *xrcd_init_attr); 669int mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num); 670int mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd); 671struct ibv_wq *mlx5_create_wq(struct ibv_context *context, 672 struct ibv_wq_init_attr *attr); 673int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr); 674int mlx5_destroy_wq(struct ibv_wq *wq); 675struct ibv_rwq_ind_table *mlx5_create_rwq_ind_table(struct ibv_context *context, 676 struct ibv_rwq_ind_table_init_attr *init_attr); 677int mlx5_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table); 678struct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, 679 struct ibv_srq_init_attr_ex *attr); 680 681static inline void *mlx5_find_uidx(struct mlx5_context *ctx, uint32_t uidx) 682{ 683 int tind = uidx >> MLX5_UIDX_TABLE_SHIFT; 684 685 if (likely(ctx->uidx_table[tind].refcnt)) 686 return ctx->uidx_table[tind].table[uidx & MLX5_UIDX_TABLE_MASK]; 687 688 return NULL; 689} 690 691static inline int mlx5_spin_lock(struct mlx5_spinlock *lock) 692{ 693 if (!mlx5_single_threaded) 694 return pthread_spin_lock(&lock->lock); 695 696 if (unlikely(lock->in_use)) { 697 fprintf(stderr, "*** ERROR: multithreading vilation ***\n" 698 "You are running a multithreaded application but\n" 699 "you set MLX5_SINGLE_THREADED=1. Please unset it.\n"); 700 abort(); 701 } else { 702 lock->in_use = 1; 703 /* 704 * This fence is not at all correct, but it increases the 705 * chance that in_use is detected by another thread without 706 * much runtime cost. */ 707 atomic_thread_fence(memory_order_acq_rel); 708 } 709 710 return 0; 711} 712 713static inline int mlx5_spin_unlock(struct mlx5_spinlock *lock) 714{ 715 if (!mlx5_single_threaded) 716 return pthread_spin_unlock(&lock->lock); 717 718 lock->in_use = 0; 719 720 return 0; 721} 722 723static inline int mlx5_spinlock_init(struct mlx5_spinlock *lock) 724{ 725 lock->in_use = 0; 726 return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE); 727} 728 729static inline int mlx5_spinlock_destroy(struct mlx5_spinlock *lock) 730{ 731 return pthread_spin_destroy(&lock->lock); 732} 733 734static inline void set_command(int command, off_t *offset) 735{ 736 *offset |= (command << MLX5_IB_MMAP_CMD_SHIFT); 737} 738 739static inline void set_arg(int arg, off_t *offset) 740{ 741 *offset |= arg; 742} 743 744static inline void set_order(int order, off_t *offset) 745{ 746 set_arg(order, offset); 747} 748 749static inline void set_index(int index, off_t *offset) 750{ 751 set_arg(index, offset); 752} 753 754static inline uint8_t calc_sig(void *wqe, int size) 755{ 756 int i; 757 uint8_t *p = wqe; 758 uint8_t res = 0; 759 760 for (i = 0; i < size; ++i) 761 res ^= p[i]; 762 763 return ~res; 764} 765 766#endif /* MLX5_H */ 767