1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#if HAVE_CONFIG_H 35219820Sjeff# include <config.h> 36219820Sjeff#endif /* HAVE_CONFIG_H */ 37219820Sjeff 38219820Sjeff#include <stdlib.h> 39219820Sjeff#include <stdio.h> 40219820Sjeff#include <strings.h> 41219820Sjeff#include <pthread.h> 42219820Sjeff#include <errno.h> 43219820Sjeff#include <netinet/in.h> 44219820Sjeff 45219820Sjeff#include "mthca.h" 46219820Sjeff#include "mthca-abi.h" 47219820Sjeff 48219820Sjeffint mthca_query_device(struct ibv_context *context, struct ibv_device_attr *attr) 49219820Sjeff{ 50219820Sjeff struct ibv_query_device cmd; 51219820Sjeff uint64_t raw_fw_ver; 52219820Sjeff unsigned major, minor, sub_minor; 53219820Sjeff int ret; 54219820Sjeff 55219820Sjeff ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd); 56219820Sjeff if (ret) 57219820Sjeff return ret; 58219820Sjeff 59219820Sjeff major = (raw_fw_ver >> 32) & 0xffff; 60219820Sjeff minor = (raw_fw_ver >> 16) & 0xffff; 61219820Sjeff sub_minor = raw_fw_ver & 0xffff; 62219820Sjeff 63219820Sjeff snprintf(attr->fw_ver, sizeof attr->fw_ver, 64219820Sjeff "%d.%d.%d", major, minor, sub_minor); 65219820Sjeff 66219820Sjeff return 0; 67219820Sjeff} 68219820Sjeff 69219820Sjeffint mthca_query_port(struct ibv_context *context, uint8_t port, 70219820Sjeff struct ibv_port_attr *attr) 71219820Sjeff{ 72219820Sjeff struct ibv_query_port cmd; 73219820Sjeff 74219820Sjeff return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); 75219820Sjeff} 76219820Sjeff 77219820Sjeffstruct ibv_pd *mthca_alloc_pd(struct ibv_context *context) 78219820Sjeff{ 79219820Sjeff struct ibv_alloc_pd cmd; 80219820Sjeff struct mthca_alloc_pd_resp resp; 81219820Sjeff struct mthca_pd *pd; 82219820Sjeff 83219820Sjeff pd = malloc(sizeof *pd); 84219820Sjeff if (!pd) 85219820Sjeff return NULL; 86219820Sjeff 87219820Sjeff if (!mthca_is_memfree(context)) { 88219820Sjeff pd->ah_list = NULL; 89219820Sjeff if (pthread_mutex_init(&pd->ah_mutex, NULL)) { 90219820Sjeff free(pd); 91219820Sjeff return NULL; 92219820Sjeff } 93219820Sjeff } 94219820Sjeff 95219820Sjeff if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd, 96219820Sjeff &resp.ibv_resp, sizeof resp)) { 97219820Sjeff free(pd); 98219820Sjeff return NULL; 99219820Sjeff } 100219820Sjeff 101219820Sjeff pd->pdn = resp.pdn; 102219820Sjeff 103219820Sjeff return &pd->ibv_pd; 104219820Sjeff} 105219820Sjeff 106219820Sjeffint mthca_free_pd(struct ibv_pd *pd) 107219820Sjeff{ 108219820Sjeff int ret; 109219820Sjeff 110219820Sjeff ret = ibv_cmd_dealloc_pd(pd); 111219820Sjeff if (ret) 112219820Sjeff return ret; 113219820Sjeff 114219820Sjeff free(to_mpd(pd)); 115219820Sjeff return 0; 116219820Sjeff} 117219820Sjeff 118219820Sjeffstatic struct ibv_mr *__mthca_reg_mr(struct ibv_pd *pd, void *addr, 119219820Sjeff size_t length, uint64_t hca_va, 120219820Sjeff enum ibv_access_flags access, 121219820Sjeff int dma_sync) 122219820Sjeff{ 123219820Sjeff struct ibv_mr *mr; 124219820Sjeff struct mthca_reg_mr cmd; 125219820Sjeff int ret; 126219820Sjeff 127219820Sjeff /* 128219820Sjeff * Old kernels just ignore the extra data we pass in with the 129219820Sjeff * reg_mr command structure, so there's no need to add an ABI 130219820Sjeff * version check here (and indeed the kernel ABI was not 131219820Sjeff * incremented due to this change). 132219820Sjeff */ 133219820Sjeff cmd.mr_attrs = dma_sync ? MTHCA_MR_DMASYNC : 0; 134219820Sjeff cmd.reserved = 0; 135219820Sjeff 136219820Sjeff mr = malloc(sizeof *mr); 137219820Sjeff if (!mr) 138219820Sjeff return NULL; 139219820Sjeff 140219820Sjeff#ifdef IBV_CMD_REG_MR_HAS_RESP_PARAMS 141219820Sjeff { 142219820Sjeff struct ibv_reg_mr_resp resp; 143219820Sjeff 144219820Sjeff ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr, 145219820Sjeff &cmd.ibv_cmd, sizeof cmd, &resp, sizeof resp); 146219820Sjeff } 147219820Sjeff#else 148219820Sjeff ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, access, mr, 149219820Sjeff &cmd.ibv_cmd, sizeof cmd); 150219820Sjeff#endif 151219820Sjeff if (ret) { 152219820Sjeff free(mr); 153219820Sjeff return NULL; 154219820Sjeff } 155219820Sjeff 156219820Sjeff return mr; 157219820Sjeff} 158219820Sjeff 159219820Sjeffstruct ibv_mr *mthca_reg_mr(struct ibv_pd *pd, void *addr, 160219820Sjeff size_t length, enum ibv_access_flags access) 161219820Sjeff{ 162219820Sjeff return __mthca_reg_mr(pd, addr, length, (uintptr_t) addr, access, 0); 163219820Sjeff} 164219820Sjeff 165219820Sjeffint mthca_dereg_mr(struct ibv_mr *mr) 166219820Sjeff{ 167219820Sjeff int ret; 168219820Sjeff 169219820Sjeff ret = ibv_cmd_dereg_mr(mr); 170219820Sjeff if (ret) 171219820Sjeff return ret; 172219820Sjeff 173219820Sjeff free(mr); 174219820Sjeff return 0; 175219820Sjeff} 176219820Sjeff 177219820Sjeffstatic int align_cq_size(int cqe) 178219820Sjeff{ 179219820Sjeff int nent; 180219820Sjeff 181219820Sjeff for (nent = 1; nent <= cqe; nent <<= 1) 182219820Sjeff ; /* nothing */ 183219820Sjeff 184219820Sjeff return nent; 185219820Sjeff} 186219820Sjeff 187219820Sjeffstruct ibv_cq *mthca_create_cq(struct ibv_context *context, int cqe, 188219820Sjeff struct ibv_comp_channel *channel, 189219820Sjeff int comp_vector) 190219820Sjeff{ 191219820Sjeff struct mthca_create_cq cmd; 192219820Sjeff struct mthca_create_cq_resp resp; 193219820Sjeff struct mthca_cq *cq; 194219820Sjeff int ret; 195219820Sjeff 196219820Sjeff /* Sanity check CQ size before proceeding */ 197219820Sjeff if (cqe > 131072) 198219820Sjeff return NULL; 199219820Sjeff 200219820Sjeff cq = malloc(sizeof *cq); 201219820Sjeff if (!cq) 202219820Sjeff return NULL; 203219820Sjeff 204219820Sjeff cq->cons_index = 0; 205219820Sjeff 206219820Sjeff if (pthread_spin_init(&cq->lock, PTHREAD_PROCESS_PRIVATE)) 207219820Sjeff goto err; 208219820Sjeff 209219820Sjeff cqe = align_cq_size(cqe); 210219820Sjeff if (mthca_alloc_cq_buf(to_mdev(context->device), &cq->buf, cqe)) 211219820Sjeff goto err; 212219820Sjeff 213219820Sjeff cq->mr = __mthca_reg_mr(to_mctx(context)->pd, cq->buf.buf, 214219820Sjeff cqe * MTHCA_CQ_ENTRY_SIZE, 215219820Sjeff 0, IBV_ACCESS_LOCAL_WRITE, 1); 216219820Sjeff if (!cq->mr) 217219820Sjeff goto err_buf; 218219820Sjeff 219219820Sjeff cq->mr->context = context; 220219820Sjeff 221219820Sjeff if (mthca_is_memfree(context)) { 222219820Sjeff cq->arm_sn = 1; 223219820Sjeff cq->set_ci_db_index = mthca_alloc_db(to_mctx(context)->db_tab, 224219820Sjeff MTHCA_DB_TYPE_CQ_SET_CI, 225219820Sjeff &cq->set_ci_db); 226219820Sjeff if (cq->set_ci_db_index < 0) 227219820Sjeff goto err_unreg; 228219820Sjeff 229219820Sjeff cq->arm_db_index = mthca_alloc_db(to_mctx(context)->db_tab, 230219820Sjeff MTHCA_DB_TYPE_CQ_ARM, 231219820Sjeff &cq->arm_db); 232219820Sjeff if (cq->arm_db_index < 0) 233219820Sjeff goto err_set_db; 234219820Sjeff 235219820Sjeff cmd.arm_db_page = db_align(cq->arm_db); 236219820Sjeff cmd.set_db_page = db_align(cq->set_ci_db); 237219820Sjeff cmd.arm_db_index = cq->arm_db_index; 238219820Sjeff cmd.set_db_index = cq->set_ci_db_index; 239219820Sjeff } else { 240219820Sjeff cmd.arm_db_page = cmd.set_db_page = 241219820Sjeff cmd.arm_db_index = cmd.set_db_index = 0; 242219820Sjeff } 243219820Sjeff 244219820Sjeff cmd.lkey = cq->mr->lkey; 245219820Sjeff cmd.pdn = to_mpd(to_mctx(context)->pd)->pdn; 246219820Sjeff ret = ibv_cmd_create_cq(context, cqe - 1, channel, comp_vector, 247219820Sjeff &cq->ibv_cq, &cmd.ibv_cmd, sizeof cmd, 248219820Sjeff &resp.ibv_resp, sizeof resp); 249219820Sjeff if (ret) 250219820Sjeff goto err_arm_db; 251219820Sjeff 252219820Sjeff cq->cqn = resp.cqn; 253219820Sjeff 254219820Sjeff if (mthca_is_memfree(context)) { 255219820Sjeff mthca_set_db_qn(cq->set_ci_db, MTHCA_DB_TYPE_CQ_SET_CI, cq->cqn); 256219820Sjeff mthca_set_db_qn(cq->arm_db, MTHCA_DB_TYPE_CQ_ARM, cq->cqn); 257219820Sjeff } 258219820Sjeff 259219820Sjeff return &cq->ibv_cq; 260219820Sjeff 261219820Sjefferr_arm_db: 262219820Sjeff if (mthca_is_memfree(context)) 263219820Sjeff mthca_free_db(to_mctx(context)->db_tab, MTHCA_DB_TYPE_CQ_ARM, 264219820Sjeff cq->arm_db_index); 265219820Sjeff 266219820Sjefferr_set_db: 267219820Sjeff if (mthca_is_memfree(context)) 268219820Sjeff mthca_free_db(to_mctx(context)->db_tab, MTHCA_DB_TYPE_CQ_SET_CI, 269219820Sjeff cq->set_ci_db_index); 270219820Sjeff 271219820Sjefferr_unreg: 272219820Sjeff mthca_dereg_mr(cq->mr); 273219820Sjeff 274219820Sjefferr_buf: 275219820Sjeff mthca_free_buf(&cq->buf); 276219820Sjeff 277219820Sjefferr: 278219820Sjeff free(cq); 279219820Sjeff 280219820Sjeff return NULL; 281219820Sjeff} 282219820Sjeff 283219820Sjeffint mthca_resize_cq(struct ibv_cq *ibcq, int cqe) 284219820Sjeff{ 285219820Sjeff struct mthca_cq *cq = to_mcq(ibcq); 286219820Sjeff struct mthca_resize_cq cmd; 287219820Sjeff struct ibv_mr *mr; 288219820Sjeff struct mthca_buf buf; 289219820Sjeff int old_cqe; 290219820Sjeff int ret; 291219820Sjeff 292219820Sjeff /* Sanity check CQ size before proceeding */ 293219820Sjeff if (cqe > 131072) 294219820Sjeff return EINVAL; 295219820Sjeff 296219820Sjeff pthread_spin_lock(&cq->lock); 297219820Sjeff 298219820Sjeff cqe = align_cq_size(cqe); 299219820Sjeff if (cqe == ibcq->cqe + 1) { 300219820Sjeff ret = 0; 301219820Sjeff goto out; 302219820Sjeff } 303219820Sjeff 304219820Sjeff ret = mthca_alloc_cq_buf(to_mdev(ibcq->context->device), &buf, cqe); 305219820Sjeff if (ret) 306219820Sjeff goto out; 307219820Sjeff 308219820Sjeff mr = __mthca_reg_mr(to_mctx(ibcq->context)->pd, buf.buf, 309219820Sjeff cqe * MTHCA_CQ_ENTRY_SIZE, 310219820Sjeff 0, IBV_ACCESS_LOCAL_WRITE, 1); 311219820Sjeff if (!mr) { 312219820Sjeff mthca_free_buf(&buf); 313219820Sjeff ret = ENOMEM; 314219820Sjeff goto out; 315219820Sjeff } 316219820Sjeff 317219820Sjeff mr->context = ibcq->context; 318219820Sjeff 319219820Sjeff old_cqe = ibcq->cqe; 320219820Sjeff 321219820Sjeff cmd.lkey = mr->lkey; 322219820Sjeff#ifdef IBV_CMD_RESIZE_CQ_HAS_RESP_PARAMS 323219820Sjeff { 324219820Sjeff struct ibv_resize_cq_resp resp; 325219820Sjeff ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd, 326219820Sjeff &resp, sizeof resp); 327219820Sjeff } 328219820Sjeff#else 329219820Sjeff ret = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof cmd); 330219820Sjeff#endif 331219820Sjeff if (ret) { 332219820Sjeff mthca_dereg_mr(mr); 333219820Sjeff mthca_free_buf(&buf); 334219820Sjeff goto out; 335219820Sjeff } 336219820Sjeff 337219820Sjeff mthca_cq_resize_copy_cqes(cq, buf.buf, old_cqe); 338219820Sjeff 339219820Sjeff mthca_dereg_mr(cq->mr); 340219820Sjeff mthca_free_buf(&cq->buf); 341219820Sjeff 342219820Sjeff cq->buf = buf; 343219820Sjeff cq->mr = mr; 344219820Sjeff 345219820Sjeffout: 346219820Sjeff pthread_spin_unlock(&cq->lock); 347219820Sjeff return ret; 348219820Sjeff} 349219820Sjeff 350219820Sjeffint mthca_destroy_cq(struct ibv_cq *cq) 351219820Sjeff{ 352219820Sjeff int ret; 353219820Sjeff 354219820Sjeff ret = ibv_cmd_destroy_cq(cq); 355219820Sjeff if (ret) 356219820Sjeff return ret; 357219820Sjeff 358219820Sjeff if (mthca_is_memfree(cq->context)) { 359219820Sjeff mthca_free_db(to_mctx(cq->context)->db_tab, MTHCA_DB_TYPE_CQ_SET_CI, 360219820Sjeff to_mcq(cq)->set_ci_db_index); 361219820Sjeff mthca_free_db(to_mctx(cq->context)->db_tab, MTHCA_DB_TYPE_CQ_ARM, 362219820Sjeff to_mcq(cq)->arm_db_index); 363219820Sjeff } 364219820Sjeff 365219820Sjeff mthca_dereg_mr(to_mcq(cq)->mr); 366219820Sjeff mthca_free_buf(&to_mcq(cq)->buf); 367219820Sjeff free(to_mcq(cq)); 368219820Sjeff 369219820Sjeff return 0; 370219820Sjeff} 371219820Sjeff 372219820Sjeffstatic int align_queue_size(struct ibv_context *context, int size, int spare) 373219820Sjeff{ 374219820Sjeff int ret; 375219820Sjeff 376219820Sjeff /* 377219820Sjeff * If someone asks for a 0-sized queue, presumably they're not 378219820Sjeff * going to use it. So don't mess with their size. 379219820Sjeff */ 380219820Sjeff if (!size) 381219820Sjeff return 0; 382219820Sjeff 383219820Sjeff if (mthca_is_memfree(context)) { 384219820Sjeff for (ret = 1; ret < size + spare; ret <<= 1) 385219820Sjeff ; /* nothing */ 386219820Sjeff 387219820Sjeff return ret; 388219820Sjeff } else 389219820Sjeff return size + spare; 390219820Sjeff} 391219820Sjeff 392219820Sjeffstruct ibv_srq *mthca_create_srq(struct ibv_pd *pd, 393219820Sjeff struct ibv_srq_init_attr *attr) 394219820Sjeff{ 395219820Sjeff struct mthca_create_srq cmd; 396219820Sjeff struct mthca_create_srq_resp resp; 397219820Sjeff struct mthca_srq *srq; 398219820Sjeff int ret; 399219820Sjeff 400219820Sjeff /* Sanity check SRQ size before proceeding */ 401219820Sjeff if (attr->attr.max_wr > 1 << 16 || attr->attr.max_sge > 64) 402219820Sjeff return NULL; 403219820Sjeff 404219820Sjeff srq = malloc(sizeof *srq); 405219820Sjeff if (!srq) 406219820Sjeff return NULL; 407219820Sjeff 408219820Sjeff if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) 409219820Sjeff goto err; 410219820Sjeff 411219820Sjeff srq->max = align_queue_size(pd->context, attr->attr.max_wr, 1); 412219820Sjeff srq->max_gs = attr->attr.max_sge; 413219820Sjeff srq->counter = 0; 414219820Sjeff 415219820Sjeff if (mthca_alloc_srq_buf(pd, &attr->attr, srq)) 416219820Sjeff goto err; 417219820Sjeff 418219820Sjeff srq->mr = __mthca_reg_mr(pd, srq->buf.buf, srq->buf_size, 0, 0, 0); 419219820Sjeff if (!srq->mr) 420219820Sjeff goto err_free; 421219820Sjeff 422219820Sjeff srq->mr->context = pd->context; 423219820Sjeff 424219820Sjeff if (mthca_is_memfree(pd->context)) { 425219820Sjeff srq->db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab, 426219820Sjeff MTHCA_DB_TYPE_SRQ, &srq->db); 427219820Sjeff if (srq->db_index < 0) 428219820Sjeff goto err_unreg; 429219820Sjeff 430219820Sjeff cmd.db_page = db_align(srq->db); 431219820Sjeff cmd.db_index = srq->db_index; 432219820Sjeff } else { 433219820Sjeff cmd.db_page = cmd.db_index = 0; 434219820Sjeff } 435219820Sjeff 436219820Sjeff cmd.lkey = srq->mr->lkey; 437219820Sjeff 438219820Sjeff ret = ibv_cmd_create_srq(pd, &srq->ibv_srq, attr, 439219820Sjeff &cmd.ibv_cmd, sizeof cmd, 440219820Sjeff &resp.ibv_resp, sizeof resp); 441219820Sjeff if (ret) 442219820Sjeff goto err_db; 443219820Sjeff 444219820Sjeff srq->srqn = resp.srqn; 445219820Sjeff 446219820Sjeff if (mthca_is_memfree(pd->context)) 447219820Sjeff mthca_set_db_qn(srq->db, MTHCA_DB_TYPE_SRQ, srq->srqn); 448219820Sjeff 449219820Sjeff return &srq->ibv_srq; 450219820Sjeff 451219820Sjefferr_db: 452219820Sjeff if (mthca_is_memfree(pd->context)) 453219820Sjeff mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_SRQ, 454219820Sjeff srq->db_index); 455219820Sjeff 456219820Sjefferr_unreg: 457219820Sjeff mthca_dereg_mr(srq->mr); 458219820Sjeff 459219820Sjefferr_free: 460219820Sjeff free(srq->wrid); 461219820Sjeff mthca_free_buf(&srq->buf); 462219820Sjeff 463219820Sjefferr: 464219820Sjeff free(srq); 465219820Sjeff 466219820Sjeff return NULL; 467219820Sjeff} 468219820Sjeff 469219820Sjeffint mthca_modify_srq(struct ibv_srq *srq, 470219820Sjeff struct ibv_srq_attr *attr, 471219820Sjeff enum ibv_srq_attr_mask attr_mask) 472219820Sjeff{ 473219820Sjeff struct ibv_modify_srq cmd; 474219820Sjeff 475219820Sjeff return ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof cmd); 476219820Sjeff} 477219820Sjeff 478219820Sjeffint mthca_query_srq(struct ibv_srq *srq, 479219820Sjeff struct ibv_srq_attr *attr) 480219820Sjeff{ 481219820Sjeff struct ibv_query_srq cmd; 482219820Sjeff 483219820Sjeff return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd); 484219820Sjeff} 485219820Sjeff 486219820Sjeffint mthca_destroy_srq(struct ibv_srq *srq) 487219820Sjeff{ 488219820Sjeff int ret; 489219820Sjeff 490219820Sjeff ret = ibv_cmd_destroy_srq(srq); 491219820Sjeff if (ret) 492219820Sjeff return ret; 493219820Sjeff 494219820Sjeff if (mthca_is_memfree(srq->context)) 495219820Sjeff mthca_free_db(to_mctx(srq->context)->db_tab, MTHCA_DB_TYPE_SRQ, 496219820Sjeff to_msrq(srq)->db_index); 497219820Sjeff 498219820Sjeff mthca_dereg_mr(to_msrq(srq)->mr); 499219820Sjeff 500219820Sjeff mthca_free_buf(&to_msrq(srq)->buf); 501219820Sjeff free(to_msrq(srq)->wrid); 502219820Sjeff free(to_msrq(srq)); 503219820Sjeff 504219820Sjeff return 0; 505219820Sjeff} 506219820Sjeff 507219820Sjeffstruct ibv_qp *mthca_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) 508219820Sjeff{ 509219820Sjeff struct mthca_create_qp cmd; 510219820Sjeff struct ibv_create_qp_resp resp; 511219820Sjeff struct mthca_qp *qp; 512219820Sjeff int ret; 513219820Sjeff 514219820Sjeff /* Sanity check QP size before proceeding */ 515219820Sjeff if (attr->cap.max_send_wr > 65536 || 516219820Sjeff attr->cap.max_recv_wr > 65536 || 517219820Sjeff attr->cap.max_send_sge > 64 || 518219820Sjeff attr->cap.max_recv_sge > 64 || 519219820Sjeff attr->cap.max_inline_data > 1024) 520219820Sjeff return NULL; 521219820Sjeff 522219820Sjeff qp = malloc(sizeof *qp); 523219820Sjeff if (!qp) 524219820Sjeff return NULL; 525219820Sjeff 526219820Sjeff qp->sq.max = align_queue_size(pd->context, attr->cap.max_send_wr, 0); 527219820Sjeff qp->rq.max = align_queue_size(pd->context, attr->cap.max_recv_wr, 0); 528219820Sjeff 529219820Sjeff if (mthca_alloc_qp_buf(pd, &attr->cap, attr->qp_type, qp)) 530219820Sjeff goto err; 531219820Sjeff 532219820Sjeff mthca_init_qp_indices(qp); 533219820Sjeff 534219820Sjeff if (pthread_spin_init(&qp->sq.lock, PTHREAD_PROCESS_PRIVATE) || 535219820Sjeff pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE)) 536219820Sjeff goto err_free; 537219820Sjeff 538219820Sjeff qp->mr = __mthca_reg_mr(pd, qp->buf.buf, qp->buf_size, 0, 0, 0); 539219820Sjeff if (!qp->mr) 540219820Sjeff goto err_free; 541219820Sjeff 542219820Sjeff qp->mr->context = pd->context; 543219820Sjeff 544219820Sjeff cmd.lkey = qp->mr->lkey; 545219820Sjeff cmd.reserved = 0; 546219820Sjeff 547219820Sjeff if (mthca_is_memfree(pd->context)) { 548219820Sjeff qp->sq.db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab, 549219820Sjeff MTHCA_DB_TYPE_SQ, 550219820Sjeff &qp->sq.db); 551219820Sjeff if (qp->sq.db_index < 0) 552219820Sjeff goto err_unreg; 553219820Sjeff 554219820Sjeff qp->rq.db_index = mthca_alloc_db(to_mctx(pd->context)->db_tab, 555219820Sjeff MTHCA_DB_TYPE_RQ, 556219820Sjeff &qp->rq.db); 557219820Sjeff if (qp->rq.db_index < 0) 558219820Sjeff goto err_sq_db; 559219820Sjeff 560219820Sjeff cmd.sq_db_page = db_align(qp->sq.db); 561219820Sjeff cmd.rq_db_page = db_align(qp->rq.db); 562219820Sjeff cmd.sq_db_index = qp->sq.db_index; 563219820Sjeff cmd.rq_db_index = qp->rq.db_index; 564219820Sjeff } else { 565219820Sjeff cmd.sq_db_page = cmd.rq_db_page = 566219820Sjeff cmd.sq_db_index = cmd.rq_db_index = 0; 567219820Sjeff } 568219820Sjeff 569219820Sjeff pthread_mutex_lock(&to_mctx(pd->context)->qp_table_mutex); 570219820Sjeff ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd, 571219820Sjeff &resp, sizeof resp); 572219820Sjeff if (ret) 573219820Sjeff goto err_rq_db; 574219820Sjeff 575219820Sjeff if (mthca_is_memfree(pd->context)) { 576219820Sjeff mthca_set_db_qn(qp->sq.db, MTHCA_DB_TYPE_SQ, qp->ibv_qp.qp_num); 577219820Sjeff mthca_set_db_qn(qp->rq.db, MTHCA_DB_TYPE_RQ, qp->ibv_qp.qp_num); 578219820Sjeff } 579219820Sjeff 580219820Sjeff ret = mthca_store_qp(to_mctx(pd->context), qp->ibv_qp.qp_num, qp); 581219820Sjeff if (ret) 582219820Sjeff goto err_destroy; 583219820Sjeff pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex); 584219820Sjeff 585219820Sjeff qp->sq.max = attr->cap.max_send_wr; 586219820Sjeff qp->rq.max = attr->cap.max_recv_wr; 587219820Sjeff qp->sq.max_gs = attr->cap.max_send_sge; 588219820Sjeff qp->rq.max_gs = attr->cap.max_recv_sge; 589219820Sjeff qp->max_inline_data = attr->cap.max_inline_data; 590219820Sjeff 591219820Sjeff return &qp->ibv_qp; 592219820Sjeff 593219820Sjefferr_destroy: 594219820Sjeff ibv_cmd_destroy_qp(&qp->ibv_qp); 595219820Sjeff 596219820Sjefferr_rq_db: 597219820Sjeff pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex); 598219820Sjeff if (mthca_is_memfree(pd->context)) 599219820Sjeff mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_RQ, 600219820Sjeff qp->rq.db_index); 601219820Sjeff 602219820Sjefferr_sq_db: 603219820Sjeff if (mthca_is_memfree(pd->context)) 604219820Sjeff mthca_free_db(to_mctx(pd->context)->db_tab, MTHCA_DB_TYPE_SQ, 605219820Sjeff qp->sq.db_index); 606219820Sjeff 607219820Sjefferr_unreg: 608219820Sjeff mthca_dereg_mr(qp->mr); 609219820Sjeff 610219820Sjefferr_free: 611219820Sjeff free(qp->wrid); 612219820Sjeff mthca_free_buf(&qp->buf); 613219820Sjeff 614219820Sjefferr: 615219820Sjeff free(qp); 616219820Sjeff 617219820Sjeff return NULL; 618219820Sjeff} 619219820Sjeff 620219820Sjeffint mthca_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 621219820Sjeff enum ibv_qp_attr_mask attr_mask, 622219820Sjeff struct ibv_qp_init_attr *init_attr) 623219820Sjeff{ 624219820Sjeff struct ibv_query_qp cmd; 625219820Sjeff 626219820Sjeff return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof cmd); 627219820Sjeff} 628219820Sjeff 629219820Sjeffint mthca_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 630219820Sjeff enum ibv_qp_attr_mask attr_mask) 631219820Sjeff{ 632219820Sjeff struct ibv_modify_qp cmd; 633219820Sjeff int ret; 634219820Sjeff 635219820Sjeff ret = ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd); 636219820Sjeff 637219820Sjeff if (!ret && 638219820Sjeff (attr_mask & IBV_QP_STATE) && 639219820Sjeff attr->qp_state == IBV_QPS_RESET) { 640219820Sjeff mthca_cq_clean(to_mcq(qp->recv_cq), qp->qp_num, 641219820Sjeff qp->srq ? to_msrq(qp->srq) : NULL); 642219820Sjeff if (qp->send_cq != qp->recv_cq) 643219820Sjeff mthca_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL); 644219820Sjeff 645219820Sjeff mthca_init_qp_indices(to_mqp(qp)); 646219820Sjeff 647219820Sjeff if (mthca_is_memfree(qp->context)) { 648219820Sjeff *to_mqp(qp)->sq.db = 0; 649219820Sjeff *to_mqp(qp)->rq.db = 0; 650219820Sjeff } 651219820Sjeff } 652219820Sjeff 653219820Sjeff return ret; 654219820Sjeff} 655219820Sjeff 656219820Sjeffstatic void mthca_lock_cqs(struct ibv_qp *qp) 657219820Sjeff{ 658219820Sjeff struct mthca_cq *send_cq = to_mcq(qp->send_cq); 659219820Sjeff struct mthca_cq *recv_cq = to_mcq(qp->recv_cq); 660219820Sjeff 661219820Sjeff if (send_cq == recv_cq) 662219820Sjeff pthread_spin_lock(&send_cq->lock); 663219820Sjeff else if (send_cq->cqn < recv_cq->cqn) { 664219820Sjeff pthread_spin_lock(&send_cq->lock); 665219820Sjeff pthread_spin_lock(&recv_cq->lock); 666219820Sjeff } else { 667219820Sjeff pthread_spin_lock(&recv_cq->lock); 668219820Sjeff pthread_spin_lock(&send_cq->lock); 669219820Sjeff } 670219820Sjeff} 671219820Sjeff 672219820Sjeffstatic void mthca_unlock_cqs(struct ibv_qp *qp) 673219820Sjeff{ 674219820Sjeff struct mthca_cq *send_cq = to_mcq(qp->send_cq); 675219820Sjeff struct mthca_cq *recv_cq = to_mcq(qp->recv_cq); 676219820Sjeff 677219820Sjeff if (send_cq == recv_cq) 678219820Sjeff pthread_spin_unlock(&send_cq->lock); 679219820Sjeff else if (send_cq->cqn < recv_cq->cqn) { 680219820Sjeff pthread_spin_unlock(&recv_cq->lock); 681219820Sjeff pthread_spin_unlock(&send_cq->lock); 682219820Sjeff } else { 683219820Sjeff pthread_spin_unlock(&send_cq->lock); 684219820Sjeff pthread_spin_unlock(&recv_cq->lock); 685219820Sjeff } 686219820Sjeff} 687219820Sjeff 688219820Sjeffint mthca_destroy_qp(struct ibv_qp *qp) 689219820Sjeff{ 690219820Sjeff int ret; 691219820Sjeff 692219820Sjeff pthread_mutex_lock(&to_mctx(qp->context)->qp_table_mutex); 693219820Sjeff ret = ibv_cmd_destroy_qp(qp); 694219820Sjeff if (ret) { 695219820Sjeff pthread_mutex_unlock(&to_mctx(qp->context)->qp_table_mutex); 696219820Sjeff return ret; 697219820Sjeff } 698219820Sjeff 699219820Sjeff mthca_lock_cqs(qp); 700219820Sjeff 701219820Sjeff __mthca_cq_clean(to_mcq(qp->recv_cq), qp->qp_num, 702219820Sjeff qp->srq ? to_msrq(qp->srq) : NULL); 703219820Sjeff if (qp->send_cq != qp->recv_cq) 704219820Sjeff __mthca_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL); 705219820Sjeff 706219820Sjeff mthca_clear_qp(to_mctx(qp->context), qp->qp_num); 707219820Sjeff 708219820Sjeff mthca_unlock_cqs(qp); 709219820Sjeff pthread_mutex_unlock(&to_mctx(qp->context)->qp_table_mutex); 710219820Sjeff 711219820Sjeff if (mthca_is_memfree(qp->context)) { 712219820Sjeff mthca_free_db(to_mctx(qp->context)->db_tab, MTHCA_DB_TYPE_RQ, 713219820Sjeff to_mqp(qp)->rq.db_index); 714219820Sjeff mthca_free_db(to_mctx(qp->context)->db_tab, MTHCA_DB_TYPE_SQ, 715219820Sjeff to_mqp(qp)->sq.db_index); 716219820Sjeff } 717219820Sjeff 718219820Sjeff mthca_dereg_mr(to_mqp(qp)->mr); 719219820Sjeff mthca_free_buf(&to_mqp(qp)->buf); 720219820Sjeff free(to_mqp(qp)->wrid); 721219820Sjeff free(to_mqp(qp)); 722219820Sjeff 723219820Sjeff return 0; 724219820Sjeff} 725219820Sjeff 726219820Sjeffstruct ibv_ah *mthca_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 727219820Sjeff{ 728219820Sjeff struct mthca_ah *ah; 729219820Sjeff 730219820Sjeff ah = malloc(sizeof *ah); 731219820Sjeff if (!ah) 732219820Sjeff return NULL; 733219820Sjeff 734219820Sjeff if (mthca_alloc_av(to_mpd(pd), attr, ah)) { 735219820Sjeff free(ah); 736219820Sjeff return NULL; 737219820Sjeff } 738219820Sjeff 739219820Sjeff return &ah->ibv_ah; 740219820Sjeff} 741219820Sjeff 742219820Sjeffint mthca_destroy_ah(struct ibv_ah *ah) 743219820Sjeff{ 744219820Sjeff mthca_free_av(to_mah(ah)); 745219820Sjeff free(to_mah(ah)); 746219820Sjeff 747219820Sjeff return 0; 748219820Sjeff} 749219820Sjeff 750219820Sjeffint mthca_attach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) 751219820Sjeff{ 752219820Sjeff return ibv_cmd_attach_mcast(qp, gid, lid); 753219820Sjeff} 754219820Sjeff 755219820Sjeffint mthca_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid) 756219820Sjeff{ 757219820Sjeff return ibv_cmd_detach_mcast(qp, gid, lid); 758219820Sjeff} 759