1219820Sjeff/* 2219820Sjeff * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3219820Sjeff * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#ifndef MTHCA_H 35219820Sjeff#define MTHCA_H 36219820Sjeff 37219820Sjeff#include <stddef.h> 38219820Sjeff 39219820Sjeff#include <infiniband/driver.h> 40219820Sjeff#include <infiniband/arch.h> 41219820Sjeff 42219820Sjeff#ifdef HAVE_VALGRIND_MEMCHECK_H 43219820Sjeff 44219820Sjeff# include <valgrind/memcheck.h> 45219820Sjeff 46219820Sjeff# if !defined(VALGRIND_MAKE_MEM_DEFINED) || !defined(VALGRIND_MAKE_MEM_UNDEFINED) 47219820Sjeff# warning "Valgrind support requested, but VALGRIND_MAKE_MEM_(UN)DEFINED not available" 48219820Sjeff# endif 49219820Sjeff 50219820Sjeff#endif /* HAVE_VALGRIND_MEMCHECK_H */ 51219820Sjeff 52219820Sjeff#ifndef VALGRIND_MAKE_MEM_DEFINED 53219820Sjeff# define VALGRIND_MAKE_MEM_DEFINED(addr,len) 54219820Sjeff#endif 55219820Sjeff 56219820Sjeff#ifndef VALGRIND_MAKE_MEM_UNDEFINED 57219820Sjeff# define VALGRIND_MAKE_MEM_UNDEFINED(addr,len) 58219820Sjeff#endif 59219820Sjeff 60219820Sjeff#ifndef rmb 61219820Sjeff# define rmb() mb() 62219820Sjeff#endif 63219820Sjeff 64219820Sjeff#ifndef wmb 65219820Sjeff# define wmb() mb() 66219820Sjeff#endif 67219820Sjeff 68219820Sjeff#define HIDDEN __attribute__((visibility ("hidden"))) 69219820Sjeff 70219820Sjeff#define PFX "mthca: " 71219820Sjeff 72219820Sjeffenum mthca_hca_type { 73219820Sjeff MTHCA_TAVOR, 74219820Sjeff MTHCA_ARBEL 75219820Sjeff}; 76219820Sjeff 77219820Sjeffenum { 78219820Sjeff MTHCA_CQ_ENTRY_SIZE = 0x20 79219820Sjeff}; 80219820Sjeff 81219820Sjeffenum { 82219820Sjeff MTHCA_QP_TABLE_BITS = 8, 83219820Sjeff MTHCA_QP_TABLE_SIZE = 1 << MTHCA_QP_TABLE_BITS, 84219820Sjeff MTHCA_QP_TABLE_MASK = MTHCA_QP_TABLE_SIZE - 1 85219820Sjeff}; 86219820Sjeff 87219820Sjeffenum { 88219820Sjeff MTHCA_DB_REC_PAGE_SIZE = 4096, 89219820Sjeff MTHCA_DB_REC_PER_PAGE = MTHCA_DB_REC_PAGE_SIZE / 8 90219820Sjeff}; 91219820Sjeff 92219820Sjeffenum mthca_db_type { 93219820Sjeff MTHCA_DB_TYPE_INVALID = 0x0, 94219820Sjeff MTHCA_DB_TYPE_CQ_SET_CI = 0x1, 95219820Sjeff MTHCA_DB_TYPE_CQ_ARM = 0x2, 96219820Sjeff MTHCA_DB_TYPE_SQ = 0x3, 97219820Sjeff MTHCA_DB_TYPE_RQ = 0x4, 98219820Sjeff MTHCA_DB_TYPE_SRQ = 0x5, 99219820Sjeff MTHCA_DB_TYPE_GROUP_SEP = 0x7 100219820Sjeff}; 101219820Sjeff 102219820Sjeffenum { 103219820Sjeff MTHCA_OPCODE_NOP = 0x00, 104219820Sjeff MTHCA_OPCODE_RDMA_WRITE = 0x08, 105219820Sjeff MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09, 106219820Sjeff MTHCA_OPCODE_SEND = 0x0a, 107219820Sjeff MTHCA_OPCODE_SEND_IMM = 0x0b, 108219820Sjeff MTHCA_OPCODE_RDMA_READ = 0x10, 109219820Sjeff MTHCA_OPCODE_ATOMIC_CS = 0x11, 110219820Sjeff MTHCA_OPCODE_ATOMIC_FA = 0x12, 111219820Sjeff MTHCA_OPCODE_BIND_MW = 0x18, 112219820Sjeff MTHCA_OPCODE_INVALID = 0xff 113219820Sjeff}; 114219820Sjeff 115219820Sjeffstruct mthca_ah_page; 116219820Sjeff 117219820Sjeffstruct mthca_device { 118219820Sjeff struct ibv_device ibv_dev; 119219820Sjeff enum mthca_hca_type hca_type; 120219820Sjeff int page_size; 121219820Sjeff}; 122219820Sjeff 123219820Sjeffstruct mthca_db_table; 124219820Sjeff 125219820Sjeffstruct mthca_context { 126219820Sjeff struct ibv_context ibv_ctx; 127219820Sjeff void *uar; 128219820Sjeff pthread_spinlock_t uar_lock; 129219820Sjeff struct mthca_db_table *db_tab; 130219820Sjeff struct ibv_pd *pd; 131219820Sjeff struct { 132219820Sjeff struct mthca_qp **table; 133219820Sjeff int refcnt; 134219820Sjeff } qp_table[MTHCA_QP_TABLE_SIZE]; 135219820Sjeff pthread_mutex_t qp_table_mutex; 136219820Sjeff int num_qps; 137219820Sjeff int qp_table_shift; 138219820Sjeff int qp_table_mask; 139219820Sjeff}; 140219820Sjeff 141219820Sjeffstruct mthca_buf { 142219820Sjeff void *buf; 143219820Sjeff size_t length; 144219820Sjeff}; 145219820Sjeff 146219820Sjeffstruct mthca_pd { 147219820Sjeff struct ibv_pd ibv_pd; 148219820Sjeff struct mthca_ah_page *ah_list; 149219820Sjeff pthread_mutex_t ah_mutex; 150219820Sjeff uint32_t pdn; 151219820Sjeff}; 152219820Sjeff 153219820Sjeffstruct mthca_cq { 154219820Sjeff struct ibv_cq ibv_cq; 155219820Sjeff struct mthca_buf buf; 156219820Sjeff pthread_spinlock_t lock; 157219820Sjeff struct ibv_mr *mr; 158219820Sjeff uint32_t cqn; 159219820Sjeff uint32_t cons_index; 160219820Sjeff 161219820Sjeff /* Next fields are mem-free only */ 162219820Sjeff int set_ci_db_index; 163219820Sjeff uint32_t *set_ci_db; 164219820Sjeff int arm_db_index; 165219820Sjeff uint32_t *arm_db; 166219820Sjeff int arm_sn; 167219820Sjeff}; 168219820Sjeff 169219820Sjeffstruct mthca_srq { 170219820Sjeff struct ibv_srq ibv_srq; 171219820Sjeff struct mthca_buf buf; 172219820Sjeff void *last; 173219820Sjeff pthread_spinlock_t lock; 174219820Sjeff struct ibv_mr *mr; 175219820Sjeff uint64_t *wrid; 176219820Sjeff uint32_t srqn; 177219820Sjeff int max; 178219820Sjeff int max_gs; 179219820Sjeff int wqe_shift; 180219820Sjeff int first_free; 181219820Sjeff int last_free; 182219820Sjeff int buf_size; 183219820Sjeff 184219820Sjeff /* Next fields are mem-free only */ 185219820Sjeff int db_index; 186219820Sjeff uint32_t *db; 187219820Sjeff uint16_t counter; 188219820Sjeff}; 189219820Sjeff 190219820Sjeffstruct mthca_wq { 191219820Sjeff pthread_spinlock_t lock; 192219820Sjeff int max; 193219820Sjeff unsigned next_ind; 194219820Sjeff unsigned last_comp; 195219820Sjeff unsigned head; 196219820Sjeff unsigned tail; 197219820Sjeff void *last; 198219820Sjeff int max_gs; 199219820Sjeff int wqe_shift; 200219820Sjeff 201219820Sjeff /* Next fields are mem-free only */ 202219820Sjeff int db_index; 203219820Sjeff uint32_t *db; 204219820Sjeff}; 205219820Sjeff 206219820Sjeffstruct mthca_qp { 207219820Sjeff struct ibv_qp ibv_qp; 208219820Sjeff struct mthca_buf buf; 209219820Sjeff uint64_t *wrid; 210219820Sjeff int send_wqe_offset; 211219820Sjeff int max_inline_data; 212219820Sjeff int buf_size; 213219820Sjeff struct mthca_wq sq; 214219820Sjeff struct mthca_wq rq; 215219820Sjeff struct ibv_mr *mr; 216219820Sjeff int sq_sig_all; 217219820Sjeff}; 218219820Sjeff 219219820Sjeffstruct mthca_av { 220219820Sjeff uint32_t port_pd; 221219820Sjeff uint8_t reserved1; 222219820Sjeff uint8_t g_slid; 223219820Sjeff uint16_t dlid; 224219820Sjeff uint8_t reserved2; 225219820Sjeff uint8_t gid_index; 226219820Sjeff uint8_t msg_sr; 227219820Sjeff uint8_t hop_limit; 228219820Sjeff uint32_t sl_tclass_flowlabel; 229219820Sjeff uint32_t dgid[4]; 230219820Sjeff}; 231219820Sjeff 232219820Sjeffstruct mthca_ah { 233219820Sjeff struct ibv_ah ibv_ah; 234219820Sjeff struct mthca_av *av; 235219820Sjeff struct mthca_ah_page *page; 236219820Sjeff uint32_t key; 237219820Sjeff}; 238219820Sjeff 239219820Sjeffstatic inline unsigned long align(unsigned long val, unsigned long align) 240219820Sjeff{ 241219820Sjeff return (val + align - 1) & ~(align - 1); 242219820Sjeff} 243219820Sjeff 244219820Sjeffstatic inline uintptr_t db_align(uint32_t *db) 245219820Sjeff{ 246219820Sjeff return (uintptr_t) db & ~((uintptr_t) MTHCA_DB_REC_PAGE_SIZE - 1); 247219820Sjeff} 248219820Sjeff 249219820Sjeff#define to_mxxx(xxx, type) \ 250219820Sjeff ((struct mthca_##type *) \ 251219820Sjeff ((void *) ib##xxx - offsetof(struct mthca_##type, ibv_##xxx))) 252219820Sjeff 253219820Sjeffstatic inline struct mthca_device *to_mdev(struct ibv_device *ibdev) 254219820Sjeff{ 255219820Sjeff return to_mxxx(dev, device); 256219820Sjeff} 257219820Sjeff 258219820Sjeffstatic inline struct mthca_context *to_mctx(struct ibv_context *ibctx) 259219820Sjeff{ 260219820Sjeff return to_mxxx(ctx, context); 261219820Sjeff} 262219820Sjeff 263219820Sjeffstatic inline struct mthca_pd *to_mpd(struct ibv_pd *ibpd) 264219820Sjeff{ 265219820Sjeff return to_mxxx(pd, pd); 266219820Sjeff} 267219820Sjeff 268219820Sjeffstatic inline struct mthca_cq *to_mcq(struct ibv_cq *ibcq) 269219820Sjeff{ 270219820Sjeff return to_mxxx(cq, cq); 271219820Sjeff} 272219820Sjeff 273219820Sjeffstatic inline struct mthca_srq *to_msrq(struct ibv_srq *ibsrq) 274219820Sjeff{ 275219820Sjeff return to_mxxx(srq, srq); 276219820Sjeff} 277219820Sjeff 278219820Sjeffstatic inline struct mthca_qp *to_mqp(struct ibv_qp *ibqp) 279219820Sjeff{ 280219820Sjeff return to_mxxx(qp, qp); 281219820Sjeff} 282219820Sjeff 283219820Sjeffstatic inline struct mthca_ah *to_mah(struct ibv_ah *ibah) 284219820Sjeff{ 285219820Sjeff return to_mxxx(ah, ah); 286219820Sjeff} 287219820Sjeff 288219820Sjeffstatic inline int mthca_is_memfree(struct ibv_context *ibctx) 289219820Sjeff{ 290219820Sjeff return to_mdev(ibctx->device)->hca_type == MTHCA_ARBEL; 291219820Sjeff} 292219820Sjeff 293219820Sjeffint mthca_alloc_buf(struct mthca_buf *buf, size_t size, int page_size); 294219820Sjeffvoid mthca_free_buf(struct mthca_buf *buf); 295219820Sjeff 296219820Sjeffint mthca_alloc_db(struct mthca_db_table *db_tab, enum mthca_db_type type, 297219820Sjeff uint32_t **db); 298219820Sjeffvoid mthca_set_db_qn(uint32_t *db, enum mthca_db_type type, uint32_t qn); 299219820Sjeffvoid mthca_free_db(struct mthca_db_table *db_tab, enum mthca_db_type type, int db_index); 300219820Sjeffstruct mthca_db_table *mthca_alloc_db_tab(int uarc_size); 301219820Sjeffvoid mthca_free_db_tab(struct mthca_db_table *db_tab); 302219820Sjeff 303219820Sjeffint mthca_query_device(struct ibv_context *context, 304219820Sjeff struct ibv_device_attr *attr); 305219820Sjeffint mthca_query_port(struct ibv_context *context, uint8_t port, 306219820Sjeff struct ibv_port_attr *attr); 307219820Sjeff 308219820Sjeffstruct ibv_pd *mthca_alloc_pd(struct ibv_context *context); 309219820Sjeffint mthca_free_pd(struct ibv_pd *pd); 310219820Sjeff 311219820Sjeffstruct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, 312219820Sjeff size_t length, enum ibv_access_flags access); 313219820Sjeffint mthca_dereg_mr(struct ibv_mr *mr); 314219820Sjeff 315219820Sjeffstruct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe, 316219820Sjeff struct ibv_comp_channel *channel, 317219820Sjeff int comp_vector); 318219820Sjeffint mthca_resize_cq(struct ibv_cq *cq, int cqe); 319219820Sjeffint mthca_destroy_cq(struct ibv_cq *cq); 320219820Sjeffint mthca_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); 321219820Sjeffint mthca_tavor_arm_cq(struct ibv_cq *cq, int solicited); 322219820Sjeffint mthca_arbel_arm_cq(struct ibv_cq *cq, int solicited); 323219820Sjeffvoid mthca_arbel_cq_event(struct ibv_cq *cq); 324219820Sjeffvoid __mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq); 325219820Sjeffvoid mthca_cq_clean(struct mthca_cq *cq, uint32_t qpn, struct mthca_srq *srq); 326219820Sjeffvoid mthca_cq_resize_copy_cqes(struct mthca_cq *cq, void *buf, int new_cqe); 327219820Sjeffint mthca_alloc_cq_buf(struct mthca_device *dev, struct mthca_buf *buf, int nent); 328219820Sjeff 329219820Sjeffstruct ibv_srq *mthca_create_srq(struct ibv_pd *pd, 330219820Sjeff struct ibv_srq_init_attr *attr); 331219820Sjeffint mthca_modify_srq(struct ibv_srq *srq, 332219820Sjeff struct ibv_srq_attr *attr, 333219820Sjeff enum ibv_srq_attr_mask mask); 334219820Sjeffint mthca_query_srq(struct ibv_srq *srq, 335219820Sjeff struct ibv_srq_attr *attr); 336219820Sjeffint mthca_destroy_srq(struct ibv_srq *srq); 337219820Sjeffint mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 338219820Sjeff struct mthca_srq *srq); 339219820Sjeffvoid mthca_free_srq_wqe(struct mthca_srq *srq, int ind); 340219820Sjeffint mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, 341219820Sjeff struct ibv_recv_wr *wr, 342219820Sjeff struct ibv_recv_wr **bad_wr); 343219820Sjeffint mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, 344219820Sjeff struct ibv_recv_wr *wr, 345219820Sjeff struct ibv_recv_wr **bad_wr); 346219820Sjeff 347219820Sjeffstruct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); 348219820Sjeffint mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 349219820Sjeff enum ibv_qp_attr_mask attr_mask, 350219820Sjeff struct ibv_qp_init_attr *init_attr); 351219820Sjeffint mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 352219820Sjeff enum ibv_qp_attr_mask attr_mask); 353219820Sjeffint mthca_destroy_qp(struct ibv_qp *qp); 354219820Sjeffvoid mthca_init_qp_indices(struct mthca_qp *qp); 355219820Sjeffint mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 356219820Sjeff struct ibv_send_wr **bad_wr); 357219820Sjeffint mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 358219820Sjeff struct ibv_recv_wr **bad_wr); 359219820Sjeffint mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 360219820Sjeff struct ibv_send_wr **bad_wr); 361219820Sjeffint mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 362219820Sjeff struct ibv_recv_wr **bad_wr); 363219820Sjeffint mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, 364219820Sjeff enum ibv_qp_type type, struct mthca_qp *qp); 365219820Sjeffstruct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn); 366219820Sjeffint mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp); 367219820Sjeffvoid mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn); 368219820Sjeffint mthca_free_err_wqe(struct mthca_qp *qp, int is_send, 369219820Sjeff int index, int *dbd, uint32_t *new_wqe); 370219820Sjeffstruct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr); 371219820Sjeffint mthca_destroy_ah(struct ibv_ah *ah); 372219820Sjeffint mthca_alloc_av(struct mthca_pd *pd, struct ibv_ah_attr *attr, 373219820Sjeff struct mthca_ah *ah); 374219820Sjeffvoid mthca_free_av(struct mthca_ah *ah); 375219820Sjeffint mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); 376219820Sjeffint mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid); 377219820Sjeff 378219820Sjeff#endif /* MTHCA_H */ 379