1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#if HAVE_CONFIG_H 35219820Sjeff# include <config.h> 36219820Sjeff#endif /* HAVE_CONFIG_H */ 37219820Sjeff 38219820Sjeff#include <stdlib.h> 39219820Sjeff#include <netinet/in.h> 40219820Sjeff#include <pthread.h> 41219820Sjeff#include <string.h> 42219820Sjeff 43219820Sjeff#include "mthca.h" 44219820Sjeff#include "doorbell.h" 45219820Sjeff#include "wqe.h" 46219820Sjeff 47219820Sjeffenum { 48219820Sjeff MTHCA_SEND_DOORBELL_FENCE = 1 << 5 49219820Sjeff}; 50219820Sjeff 51219820Sjeffstatic const uint8_t mthca_opcode[] = { 52219820Sjeff [IBV_WR_SEND] = MTHCA_OPCODE_SEND, 53219820Sjeff [IBV_WR_SEND_WITH_IMM] = MTHCA_OPCODE_SEND_IMM, 54219820Sjeff [IBV_WR_RDMA_WRITE] = MTHCA_OPCODE_RDMA_WRITE, 55219820Sjeff [IBV_WR_RDMA_WRITE_WITH_IMM] = MTHCA_OPCODE_RDMA_WRITE_IMM, 56219820Sjeff [IBV_WR_RDMA_READ] = MTHCA_OPCODE_RDMA_READ, 57219820Sjeff [IBV_WR_ATOMIC_CMP_AND_SWP] = MTHCA_OPCODE_ATOMIC_CS, 58219820Sjeff [IBV_WR_ATOMIC_FETCH_AND_ADD] = MTHCA_OPCODE_ATOMIC_FA, 59219820Sjeff}; 60219820Sjeff 61219820Sjeffstatic void *get_recv_wqe(struct mthca_qp *qp, int n) 62219820Sjeff{ 63219820Sjeff return qp->buf.buf + (n << qp->rq.wqe_shift); 64219820Sjeff} 65219820Sjeff 66219820Sjeffstatic void *get_send_wqe(struct mthca_qp *qp, int n) 67219820Sjeff{ 68219820Sjeff return qp->buf.buf + qp->send_wqe_offset + (n << qp->sq.wqe_shift); 69219820Sjeff} 70219820Sjeff 71219820Sjeffvoid mthca_init_qp_indices(struct mthca_qp *qp) 72219820Sjeff{ 73219820Sjeff qp->sq.next_ind = 0; 74219820Sjeff qp->sq.last_comp = qp->sq.max - 1; 75219820Sjeff qp->sq.head = 0; 76219820Sjeff qp->sq.tail = 0; 77219820Sjeff qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); 78219820Sjeff 79219820Sjeff qp->rq.next_ind = 0; 80219820Sjeff qp->rq.last_comp = qp->rq.max - 1; 81219820Sjeff qp->rq.head = 0; 82219820Sjeff qp->rq.tail = 0; 83219820Sjeff qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); 84219820Sjeff} 85219820Sjeff 86219820Sjeffstatic inline int wq_overflow(struct mthca_wq *wq, int nreq, struct mthca_cq *cq) 87219820Sjeff{ 88219820Sjeff unsigned cur; 89219820Sjeff 90219820Sjeff cur = wq->head - wq->tail; 91219820Sjeff if (cur + nreq < wq->max) 92219820Sjeff return 0; 93219820Sjeff 94219820Sjeff pthread_spin_lock(&cq->lock); 95219820Sjeff cur = wq->head - wq->tail; 96219820Sjeff pthread_spin_unlock(&cq->lock); 97219820Sjeff 98219820Sjeff return cur + nreq >= wq->max; 99219820Sjeff} 100219820Sjeff 101219820Sjeffint mthca_tavor_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 102219820Sjeff struct ibv_send_wr **bad_wr) 103219820Sjeff{ 104219820Sjeff struct mthca_qp *qp = to_mqp(ibqp); 105219820Sjeff void *wqe, *prev_wqe; 106219820Sjeff int ind; 107219820Sjeff int nreq; 108219820Sjeff int ret = 0; 109219820Sjeff int size; 110219820Sjeff int size0 = 0; 111219820Sjeff int i; 112219820Sjeff /* 113219820Sjeff * f0 and op0 cannot be used unless nreq > 0, which means this 114219820Sjeff * function makes it through the loop at least once. So the 115219820Sjeff * code inside the if (!size0) will be executed, and f0 and 116219820Sjeff * op0 will be initialized. So any gcc warning about "may be 117219820Sjeff * used unitialized" is bogus. 118219820Sjeff */ 119219820Sjeff uint32_t f0; 120219820Sjeff uint32_t op0; 121219820Sjeff 122219820Sjeff pthread_spin_lock(&qp->sq.lock); 123219820Sjeff 124219820Sjeff ind = qp->sq.next_ind; 125219820Sjeff 126219820Sjeff for (nreq = 0; wr; ++nreq, wr = wr->next) { 127219820Sjeff if (wq_overflow(&qp->sq, nreq, to_mcq(qp->ibv_qp.send_cq))) { 128219820Sjeff ret = -1; 129219820Sjeff *bad_wr = wr; 130219820Sjeff goto out; 131219820Sjeff } 132219820Sjeff 133219820Sjeff wqe = get_send_wqe(qp, ind); 134219820Sjeff prev_wqe = qp->sq.last; 135219820Sjeff qp->sq.last = wqe; 136219820Sjeff 137219820Sjeff ((struct mthca_next_seg *) wqe)->nda_op = 0; 138219820Sjeff ((struct mthca_next_seg *) wqe)->ee_nds = 0; 139219820Sjeff ((struct mthca_next_seg *) wqe)->flags = 140219820Sjeff ((wr->send_flags & IBV_SEND_SIGNALED) ? 141219820Sjeff htonl(MTHCA_NEXT_CQ_UPDATE) : 0) | 142219820Sjeff ((wr->send_flags & IBV_SEND_SOLICITED) ? 143219820Sjeff htonl(MTHCA_NEXT_SOLICIT) : 0) | 144219820Sjeff htonl(1); 145219820Sjeff if (wr->opcode == IBV_WR_SEND_WITH_IMM || 146219820Sjeff wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) 147219820Sjeff ((struct mthca_next_seg *) wqe)->imm = wr->imm_data; 148219820Sjeff 149219820Sjeff wqe += sizeof (struct mthca_next_seg); 150219820Sjeff size = sizeof (struct mthca_next_seg) / 16; 151219820Sjeff 152219820Sjeff switch (ibqp->qp_type) { 153219820Sjeff case IBV_QPT_RC: 154219820Sjeff switch (wr->opcode) { 155219820Sjeff case IBV_WR_ATOMIC_CMP_AND_SWP: 156219820Sjeff case IBV_WR_ATOMIC_FETCH_AND_ADD: 157219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 158219820Sjeff htonll(wr->wr.atomic.remote_addr); 159219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 160219820Sjeff htonl(wr->wr.atomic.rkey); 161219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 162219820Sjeff 163219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 164219820Sjeff 165219820Sjeff if (wr->opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { 166219820Sjeff ((struct mthca_atomic_seg *) wqe)->swap_add = 167219820Sjeff htonll(wr->wr.atomic.swap); 168219820Sjeff ((struct mthca_atomic_seg *) wqe)->compare = 169219820Sjeff htonll(wr->wr.atomic.compare_add); 170219820Sjeff } else { 171219820Sjeff ((struct mthca_atomic_seg *) wqe)->swap_add = 172219820Sjeff htonll(wr->wr.atomic.compare_add); 173219820Sjeff ((struct mthca_atomic_seg *) wqe)->compare = 0; 174219820Sjeff } 175219820Sjeff 176219820Sjeff wqe += sizeof (struct mthca_atomic_seg); 177219820Sjeff size += (sizeof (struct mthca_raddr_seg) + 178219820Sjeff sizeof (struct mthca_atomic_seg)) / 16; 179219820Sjeff break; 180219820Sjeff 181219820Sjeff case IBV_WR_RDMA_WRITE: 182219820Sjeff case IBV_WR_RDMA_WRITE_WITH_IMM: 183219820Sjeff case IBV_WR_RDMA_READ: 184219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 185219820Sjeff htonll(wr->wr.rdma.remote_addr); 186219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 187219820Sjeff htonl(wr->wr.rdma.rkey); 188219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 189219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 190219820Sjeff size += sizeof (struct mthca_raddr_seg) / 16; 191219820Sjeff break; 192219820Sjeff 193219820Sjeff default: 194219820Sjeff /* No extra segments required for sends */ 195219820Sjeff break; 196219820Sjeff } 197219820Sjeff 198219820Sjeff break; 199219820Sjeff 200219820Sjeff case IBV_QPT_UC: 201219820Sjeff switch (wr->opcode) { 202219820Sjeff case IBV_WR_RDMA_WRITE: 203219820Sjeff case IBV_WR_RDMA_WRITE_WITH_IMM: 204219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 205219820Sjeff htonll(wr->wr.rdma.remote_addr); 206219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 207219820Sjeff htonl(wr->wr.rdma.rkey); 208219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 209219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 210219820Sjeff size += sizeof (struct mthca_raddr_seg) / 16; 211219820Sjeff break; 212219820Sjeff 213219820Sjeff default: 214219820Sjeff /* No extra segments required for sends */ 215219820Sjeff break; 216219820Sjeff } 217219820Sjeff 218219820Sjeff break; 219219820Sjeff 220219820Sjeff case IBV_QPT_UD: 221219820Sjeff ((struct mthca_tavor_ud_seg *) wqe)->lkey = 222219820Sjeff htonl(to_mah(wr->wr.ud.ah)->key); 223219820Sjeff ((struct mthca_tavor_ud_seg *) wqe)->av_addr = 224219820Sjeff htonll((uintptr_t) to_mah(wr->wr.ud.ah)->av); 225219820Sjeff ((struct mthca_tavor_ud_seg *) wqe)->dqpn = 226219820Sjeff htonl(wr->wr.ud.remote_qpn); 227219820Sjeff ((struct mthca_tavor_ud_seg *) wqe)->qkey = 228219820Sjeff htonl(wr->wr.ud.remote_qkey); 229219820Sjeff 230219820Sjeff wqe += sizeof (struct mthca_tavor_ud_seg); 231219820Sjeff size += sizeof (struct mthca_tavor_ud_seg) / 16; 232219820Sjeff break; 233219820Sjeff 234219820Sjeff default: 235219820Sjeff break; 236219820Sjeff } 237219820Sjeff 238219820Sjeff if (wr->num_sge > qp->sq.max_gs) { 239219820Sjeff ret = -1; 240219820Sjeff *bad_wr = wr; 241219820Sjeff goto out; 242219820Sjeff } 243219820Sjeff 244219820Sjeff if (wr->send_flags & IBV_SEND_INLINE) { 245219820Sjeff if (wr->num_sge) { 246219820Sjeff struct mthca_inline_seg *seg = wqe; 247219820Sjeff int s = 0; 248219820Sjeff 249219820Sjeff wqe += sizeof *seg; 250219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 251219820Sjeff struct ibv_sge *sge = &wr->sg_list[i]; 252219820Sjeff 253219820Sjeff s += sge->length; 254219820Sjeff 255219820Sjeff if (s > qp->max_inline_data) { 256219820Sjeff ret = -1; 257219820Sjeff *bad_wr = wr; 258219820Sjeff goto out; 259219820Sjeff } 260219820Sjeff 261219820Sjeff memcpy(wqe, (void *) (intptr_t) sge->addr, 262219820Sjeff sge->length); 263219820Sjeff wqe += sge->length; 264219820Sjeff } 265219820Sjeff 266219820Sjeff seg->byte_count = htonl(MTHCA_INLINE_SEG | s); 267219820Sjeff size += align(s + sizeof *seg, 16) / 16; 268219820Sjeff } 269219820Sjeff } else { 270219820Sjeff struct mthca_data_seg *seg; 271219820Sjeff 272219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 273219820Sjeff seg = wqe; 274219820Sjeff seg->byte_count = htonl(wr->sg_list[i].length); 275219820Sjeff seg->lkey = htonl(wr->sg_list[i].lkey); 276219820Sjeff seg->addr = htonll(wr->sg_list[i].addr); 277219820Sjeff wqe += sizeof *seg; 278219820Sjeff } 279219820Sjeff 280219820Sjeff size += wr->num_sge * (sizeof *seg / 16); 281219820Sjeff } 282219820Sjeff 283219820Sjeff qp->wrid[ind + qp->rq.max] = wr->wr_id; 284219820Sjeff 285219820Sjeff if (wr->opcode >= sizeof mthca_opcode / sizeof mthca_opcode[0]) { 286219820Sjeff ret = -1; 287219820Sjeff *bad_wr = wr; 288219820Sjeff goto out; 289219820Sjeff } 290219820Sjeff 291219820Sjeff ((struct mthca_next_seg *) prev_wqe)->nda_op = 292219820Sjeff htonl(((ind << qp->sq.wqe_shift) + 293219820Sjeff qp->send_wqe_offset) | 294219820Sjeff mthca_opcode[wr->opcode]); 295219820Sjeff /* 296219820Sjeff * Make sure that nda_op is written before setting ee_nds. 297219820Sjeff */ 298219820Sjeff wmb(); 299219820Sjeff ((struct mthca_next_seg *) prev_wqe)->ee_nds = 300219820Sjeff htonl((size0 ? 0 : MTHCA_NEXT_DBD) | size | 301219820Sjeff ((wr->send_flags & IBV_SEND_FENCE) ? 302219820Sjeff MTHCA_NEXT_FENCE : 0)); 303219820Sjeff 304219820Sjeff if (!size0) { 305219820Sjeff size0 = size; 306219820Sjeff op0 = mthca_opcode[wr->opcode]; 307219820Sjeff f0 = wr->send_flags & IBV_SEND_FENCE ? 308219820Sjeff MTHCA_SEND_DOORBELL_FENCE : 0; 309219820Sjeff } 310219820Sjeff 311219820Sjeff ++ind; 312219820Sjeff if (ind >= qp->sq.max) 313219820Sjeff ind -= qp->sq.max; 314219820Sjeff } 315219820Sjeff 316219820Sjeffout: 317219820Sjeff if (nreq) { 318219820Sjeff uint32_t doorbell[2]; 319219820Sjeff 320219820Sjeff doorbell[0] = htonl(((qp->sq.next_ind << qp->sq.wqe_shift) + 321219820Sjeff qp->send_wqe_offset) | f0 | op0); 322219820Sjeff doorbell[1] = htonl((ibqp->qp_num << 8) | size0); 323219820Sjeff 324219820Sjeff mthca_write64(doorbell, to_mctx(ibqp->context), MTHCA_SEND_DOORBELL); 325219820Sjeff } 326219820Sjeff 327219820Sjeff qp->sq.next_ind = ind; 328219820Sjeff qp->sq.head += nreq; 329219820Sjeff 330219820Sjeff pthread_spin_unlock(&qp->sq.lock); 331219820Sjeff return ret; 332219820Sjeff} 333219820Sjeff 334219820Sjeffint mthca_tavor_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 335219820Sjeff struct ibv_recv_wr **bad_wr) 336219820Sjeff{ 337219820Sjeff struct mthca_qp *qp = to_mqp(ibqp); 338219820Sjeff uint32_t doorbell[2]; 339219820Sjeff int ret = 0; 340219820Sjeff int nreq; 341219820Sjeff int i; 342219820Sjeff int size; 343219820Sjeff int size0 = 0; 344219820Sjeff int ind; 345219820Sjeff void *wqe; 346219820Sjeff void *prev_wqe; 347219820Sjeff 348219820Sjeff pthread_spin_lock(&qp->rq.lock); 349219820Sjeff 350219820Sjeff ind = qp->rq.next_ind; 351219820Sjeff 352219820Sjeff for (nreq = 0; wr; wr = wr->next) { 353219820Sjeff if (wq_overflow(&qp->rq, nreq, to_mcq(qp->ibv_qp.recv_cq))) { 354219820Sjeff ret = -1; 355219820Sjeff *bad_wr = wr; 356219820Sjeff goto out; 357219820Sjeff } 358219820Sjeff 359219820Sjeff wqe = get_recv_wqe(qp, ind); 360219820Sjeff prev_wqe = qp->rq.last; 361219820Sjeff qp->rq.last = wqe; 362219820Sjeff 363219820Sjeff ((struct mthca_next_seg *) wqe)->ee_nds = 364219820Sjeff htonl(MTHCA_NEXT_DBD); 365219820Sjeff ((struct mthca_next_seg *) wqe)->flags = 366219820Sjeff htonl(MTHCA_NEXT_CQ_UPDATE); 367219820Sjeff 368219820Sjeff wqe += sizeof (struct mthca_next_seg); 369219820Sjeff size = sizeof (struct mthca_next_seg) / 16; 370219820Sjeff 371219820Sjeff if (wr->num_sge > qp->rq.max_gs) { 372219820Sjeff ret = -1; 373219820Sjeff *bad_wr = wr; 374219820Sjeff goto out; 375219820Sjeff } 376219820Sjeff 377219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 378219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 379219820Sjeff htonl(wr->sg_list[i].length); 380219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = 381219820Sjeff htonl(wr->sg_list[i].lkey); 382219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 383219820Sjeff htonll(wr->sg_list[i].addr); 384219820Sjeff wqe += sizeof (struct mthca_data_seg); 385219820Sjeff size += sizeof (struct mthca_data_seg) / 16; 386219820Sjeff } 387219820Sjeff 388219820Sjeff qp->wrid[ind] = wr->wr_id; 389219820Sjeff 390219820Sjeff ((struct mthca_next_seg *) prev_wqe)->ee_nds = 391219820Sjeff htonl(MTHCA_NEXT_DBD | size); 392219820Sjeff 393219820Sjeff if (!size0) 394219820Sjeff size0 = size; 395219820Sjeff 396219820Sjeff ++ind; 397219820Sjeff if (ind >= qp->rq.max) 398219820Sjeff ind -= qp->rq.max; 399219820Sjeff 400219820Sjeff ++nreq; 401219820Sjeff if (nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) { 402219820Sjeff nreq = 0; 403219820Sjeff 404219820Sjeff doorbell[0] = htonl((qp->rq.next_ind << qp->rq.wqe_shift) | size0); 405219820Sjeff doorbell[1] = htonl(ibqp->qp_num << 8); 406219820Sjeff 407219820Sjeff /* 408219820Sjeff * Make sure that descriptors are written 409219820Sjeff * before doorbell is rung. 410219820Sjeff */ 411219820Sjeff wmb(); 412219820Sjeff 413219820Sjeff mthca_write64(doorbell, to_mctx(ibqp->context), MTHCA_RECV_DOORBELL); 414219820Sjeff 415219820Sjeff qp->rq.next_ind = ind; 416219820Sjeff qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB; 417219820Sjeff size0 = 0; 418219820Sjeff } 419219820Sjeff } 420219820Sjeff 421219820Sjeffout: 422219820Sjeff if (nreq) { 423219820Sjeff doorbell[0] = htonl((qp->rq.next_ind << qp->rq.wqe_shift) | size0); 424219820Sjeff doorbell[1] = htonl((ibqp->qp_num << 8) | nreq); 425219820Sjeff 426219820Sjeff /* 427219820Sjeff * Make sure that descriptors are written before 428219820Sjeff * doorbell is rung. 429219820Sjeff */ 430219820Sjeff wmb(); 431219820Sjeff 432219820Sjeff mthca_write64(doorbell, to_mctx(ibqp->context), MTHCA_RECV_DOORBELL); 433219820Sjeff } 434219820Sjeff 435219820Sjeff qp->rq.next_ind = ind; 436219820Sjeff qp->rq.head += nreq; 437219820Sjeff 438219820Sjeff pthread_spin_unlock(&qp->rq.lock); 439219820Sjeff return ret; 440219820Sjeff} 441219820Sjeff 442219820Sjeffint mthca_arbel_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 443219820Sjeff struct ibv_send_wr **bad_wr) 444219820Sjeff{ 445219820Sjeff struct mthca_qp *qp = to_mqp(ibqp); 446219820Sjeff uint32_t doorbell[2]; 447219820Sjeff void *wqe, *prev_wqe; 448219820Sjeff int ind; 449219820Sjeff int nreq; 450219820Sjeff int ret = 0; 451219820Sjeff int size; 452219820Sjeff int size0 = 0; 453219820Sjeff int i; 454219820Sjeff /* 455219820Sjeff * f0 and op0 cannot be used unless nreq > 0, which means this 456219820Sjeff * function makes it through the loop at least once. So the 457219820Sjeff * code inside the if (!size0) will be executed, and f0 and 458219820Sjeff * op0 will be initialized. So any gcc warning about "may be 459219820Sjeff * used unitialized" is bogus. 460219820Sjeff */ 461219820Sjeff uint32_t f0; 462219820Sjeff uint32_t op0; 463219820Sjeff 464219820Sjeff pthread_spin_lock(&qp->sq.lock); 465219820Sjeff 466219820Sjeff /* XXX check that state is OK to post send */ 467219820Sjeff 468219820Sjeff ind = qp->sq.head & (qp->sq.max - 1); 469219820Sjeff 470219820Sjeff for (nreq = 0; wr; ++nreq, wr = wr->next) { 471219820Sjeff if (nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB) { 472219820Sjeff nreq = 0; 473219820Sjeff 474219820Sjeff doorbell[0] = htonl((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) | 475219820Sjeff ((qp->sq.head & 0xffff) << 8) | f0 | op0); 476219820Sjeff doorbell[1] = htonl((ibqp->qp_num << 8) | size0); 477219820Sjeff 478219820Sjeff qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB; 479219820Sjeff 480219820Sjeff /* 481219820Sjeff * Make sure that descriptors are written before 482219820Sjeff * doorbell record. 483219820Sjeff */ 484219820Sjeff wmb(); 485219820Sjeff *qp->sq.db = htonl(qp->sq.head & 0xffff); 486219820Sjeff 487219820Sjeff /* 488219820Sjeff * Make sure doorbell record is written before we 489219820Sjeff * write MMIO send doorbell. 490219820Sjeff */ 491219820Sjeff wmb(); 492219820Sjeff mthca_write64(doorbell, to_mctx(ibqp->context), MTHCA_SEND_DOORBELL); 493219820Sjeff 494219820Sjeff size0 = 0; 495219820Sjeff } 496219820Sjeff 497219820Sjeff if (wq_overflow(&qp->sq, nreq, to_mcq(qp->ibv_qp.send_cq))) { 498219820Sjeff ret = -1; 499219820Sjeff *bad_wr = wr; 500219820Sjeff goto out; 501219820Sjeff } 502219820Sjeff 503219820Sjeff wqe = get_send_wqe(qp, ind); 504219820Sjeff prev_wqe = qp->sq.last; 505219820Sjeff qp->sq.last = wqe; 506219820Sjeff 507219820Sjeff ((struct mthca_next_seg *) wqe)->flags = 508219820Sjeff ((wr->send_flags & IBV_SEND_SIGNALED) ? 509219820Sjeff htonl(MTHCA_NEXT_CQ_UPDATE) : 0) | 510219820Sjeff ((wr->send_flags & IBV_SEND_SOLICITED) ? 511219820Sjeff htonl(MTHCA_NEXT_SOLICIT) : 0) | 512219820Sjeff htonl(1); 513219820Sjeff if (wr->opcode == IBV_WR_SEND_WITH_IMM || 514219820Sjeff wr->opcode == IBV_WR_RDMA_WRITE_WITH_IMM) 515219820Sjeff ((struct mthca_next_seg *) wqe)->imm = wr->imm_data; 516219820Sjeff 517219820Sjeff wqe += sizeof (struct mthca_next_seg); 518219820Sjeff size = sizeof (struct mthca_next_seg) / 16; 519219820Sjeff 520219820Sjeff switch (ibqp->qp_type) { 521219820Sjeff case IBV_QPT_RC: 522219820Sjeff switch (wr->opcode) { 523219820Sjeff case IBV_WR_ATOMIC_CMP_AND_SWP: 524219820Sjeff case IBV_WR_ATOMIC_FETCH_AND_ADD: 525219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 526219820Sjeff htonll(wr->wr.atomic.remote_addr); 527219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 528219820Sjeff htonl(wr->wr.atomic.rkey); 529219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 530219820Sjeff 531219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 532219820Sjeff 533219820Sjeff if (wr->opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { 534219820Sjeff ((struct mthca_atomic_seg *) wqe)->swap_add = 535219820Sjeff htonll(wr->wr.atomic.swap); 536219820Sjeff ((struct mthca_atomic_seg *) wqe)->compare = 537219820Sjeff htonll(wr->wr.atomic.compare_add); 538219820Sjeff } else { 539219820Sjeff ((struct mthca_atomic_seg *) wqe)->swap_add = 540219820Sjeff htonll(wr->wr.atomic.compare_add); 541219820Sjeff ((struct mthca_atomic_seg *) wqe)->compare = 0; 542219820Sjeff } 543219820Sjeff 544219820Sjeff wqe += sizeof (struct mthca_atomic_seg); 545219820Sjeff size += (sizeof (struct mthca_raddr_seg) + 546219820Sjeff sizeof (struct mthca_atomic_seg)) / 16; 547219820Sjeff break; 548219820Sjeff 549219820Sjeff case IBV_WR_RDMA_WRITE: 550219820Sjeff case IBV_WR_RDMA_WRITE_WITH_IMM: 551219820Sjeff case IBV_WR_RDMA_READ: 552219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 553219820Sjeff htonll(wr->wr.rdma.remote_addr); 554219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 555219820Sjeff htonl(wr->wr.rdma.rkey); 556219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 557219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 558219820Sjeff size += sizeof (struct mthca_raddr_seg) / 16; 559219820Sjeff break; 560219820Sjeff 561219820Sjeff default: 562219820Sjeff /* No extra segments required for sends */ 563219820Sjeff break; 564219820Sjeff } 565219820Sjeff 566219820Sjeff break; 567219820Sjeff 568219820Sjeff case IBV_QPT_UC: 569219820Sjeff switch (wr->opcode) { 570219820Sjeff case IBV_WR_RDMA_WRITE: 571219820Sjeff case IBV_WR_RDMA_WRITE_WITH_IMM: 572219820Sjeff ((struct mthca_raddr_seg *) wqe)->raddr = 573219820Sjeff htonll(wr->wr.rdma.remote_addr); 574219820Sjeff ((struct mthca_raddr_seg *) wqe)->rkey = 575219820Sjeff htonl(wr->wr.rdma.rkey); 576219820Sjeff ((struct mthca_raddr_seg *) wqe)->reserved = 0; 577219820Sjeff wqe += sizeof (struct mthca_raddr_seg); 578219820Sjeff size += sizeof (struct mthca_raddr_seg) / 16; 579219820Sjeff break; 580219820Sjeff 581219820Sjeff default: 582219820Sjeff /* No extra segments required for sends */ 583219820Sjeff break; 584219820Sjeff } 585219820Sjeff 586219820Sjeff break; 587219820Sjeff 588219820Sjeff case IBV_QPT_UD: 589219820Sjeff memcpy(((struct mthca_arbel_ud_seg *) wqe)->av, 590219820Sjeff to_mah(wr->wr.ud.ah)->av, sizeof (struct mthca_av)); 591219820Sjeff ((struct mthca_arbel_ud_seg *) wqe)->dqpn = 592219820Sjeff htonl(wr->wr.ud.remote_qpn); 593219820Sjeff ((struct mthca_arbel_ud_seg *) wqe)->qkey = 594219820Sjeff htonl(wr->wr.ud.remote_qkey); 595219820Sjeff 596219820Sjeff wqe += sizeof (struct mthca_arbel_ud_seg); 597219820Sjeff size += sizeof (struct mthca_arbel_ud_seg) / 16; 598219820Sjeff break; 599219820Sjeff 600219820Sjeff default: 601219820Sjeff break; 602219820Sjeff } 603219820Sjeff 604219820Sjeff if (wr->num_sge > qp->sq.max_gs) { 605219820Sjeff ret = -1; 606219820Sjeff *bad_wr = wr; 607219820Sjeff goto out; 608219820Sjeff } 609219820Sjeff 610219820Sjeff if (wr->send_flags & IBV_SEND_INLINE) { 611219820Sjeff if (wr->num_sge) { 612219820Sjeff struct mthca_inline_seg *seg = wqe; 613219820Sjeff int s = 0; 614219820Sjeff 615219820Sjeff wqe += sizeof *seg; 616219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 617219820Sjeff struct ibv_sge *sge = &wr->sg_list[i]; 618219820Sjeff 619219820Sjeff s += sge->length; 620219820Sjeff 621219820Sjeff if (s > qp->max_inline_data) { 622219820Sjeff ret = -1; 623219820Sjeff *bad_wr = wr; 624219820Sjeff goto out; 625219820Sjeff } 626219820Sjeff 627219820Sjeff memcpy(wqe, (void *) (uintptr_t) sge->addr, 628219820Sjeff sge->length); 629219820Sjeff wqe += sge->length; 630219820Sjeff } 631219820Sjeff 632219820Sjeff seg->byte_count = htonl(MTHCA_INLINE_SEG | s); 633219820Sjeff size += align(s + sizeof *seg, 16) / 16; 634219820Sjeff } 635219820Sjeff } else { 636219820Sjeff struct mthca_data_seg *seg; 637219820Sjeff 638219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 639219820Sjeff seg = wqe; 640219820Sjeff seg->byte_count = htonl(wr->sg_list[i].length); 641219820Sjeff seg->lkey = htonl(wr->sg_list[i].lkey); 642219820Sjeff seg->addr = htonll(wr->sg_list[i].addr); 643219820Sjeff wqe += sizeof *seg; 644219820Sjeff } 645219820Sjeff 646219820Sjeff size += wr->num_sge * (sizeof *seg / 16); 647219820Sjeff } 648219820Sjeff 649219820Sjeff qp->wrid[ind + qp->rq.max] = wr->wr_id; 650219820Sjeff 651219820Sjeff if (wr->opcode >= sizeof mthca_opcode / sizeof mthca_opcode[0]) { 652219820Sjeff ret = -1; 653219820Sjeff *bad_wr = wr; 654219820Sjeff goto out; 655219820Sjeff } 656219820Sjeff 657219820Sjeff ((struct mthca_next_seg *) prev_wqe)->nda_op = 658219820Sjeff htonl(((ind << qp->sq.wqe_shift) + 659219820Sjeff qp->send_wqe_offset) | 660219820Sjeff mthca_opcode[wr->opcode]); 661219820Sjeff wmb(); 662219820Sjeff ((struct mthca_next_seg *) prev_wqe)->ee_nds = 663219820Sjeff htonl(MTHCA_NEXT_DBD | size | 664219820Sjeff ((wr->send_flags & IBV_SEND_FENCE) ? 665219820Sjeff MTHCA_NEXT_FENCE : 0)); 666219820Sjeff 667219820Sjeff if (!size0) { 668219820Sjeff size0 = size; 669219820Sjeff op0 = mthca_opcode[wr->opcode]; 670219820Sjeff f0 = wr->send_flags & IBV_SEND_FENCE ? 671219820Sjeff MTHCA_SEND_DOORBELL_FENCE : 0; 672219820Sjeff } 673219820Sjeff 674219820Sjeff ++ind; 675219820Sjeff if (ind >= qp->sq.max) 676219820Sjeff ind -= qp->sq.max; 677219820Sjeff } 678219820Sjeff 679219820Sjeffout: 680219820Sjeff if (nreq) { 681219820Sjeff doorbell[0] = htonl((nreq << 24) | 682219820Sjeff ((qp->sq.head & 0xffff) << 8) | 683219820Sjeff f0 | op0); 684219820Sjeff doorbell[1] = htonl((ibqp->qp_num << 8) | size0); 685219820Sjeff 686219820Sjeff qp->sq.head += nreq; 687219820Sjeff 688219820Sjeff /* 689219820Sjeff * Make sure that descriptors are written before 690219820Sjeff * doorbell record. 691219820Sjeff */ 692219820Sjeff wmb(); 693219820Sjeff *qp->sq.db = htonl(qp->sq.head & 0xffff); 694219820Sjeff 695219820Sjeff /* 696219820Sjeff * Make sure doorbell record is written before we 697219820Sjeff * write MMIO send doorbell. 698219820Sjeff */ 699219820Sjeff wmb(); 700219820Sjeff mthca_write64(doorbell, to_mctx(ibqp->context), MTHCA_SEND_DOORBELL); 701219820Sjeff } 702219820Sjeff 703219820Sjeff pthread_spin_unlock(&qp->sq.lock); 704219820Sjeff return ret; 705219820Sjeff} 706219820Sjeff 707219820Sjeffint mthca_arbel_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 708219820Sjeff struct ibv_recv_wr **bad_wr) 709219820Sjeff{ 710219820Sjeff struct mthca_qp *qp = to_mqp(ibqp); 711219820Sjeff int ret = 0; 712219820Sjeff int nreq; 713219820Sjeff int ind; 714219820Sjeff int i; 715219820Sjeff void *wqe; 716219820Sjeff 717219820Sjeff pthread_spin_lock(&qp->rq.lock); 718219820Sjeff 719219820Sjeff /* XXX check that state is OK to post receive */ 720219820Sjeff 721219820Sjeff ind = qp->rq.head & (qp->rq.max - 1); 722219820Sjeff 723219820Sjeff for (nreq = 0; wr; ++nreq, wr = wr->next) { 724219820Sjeff if (wq_overflow(&qp->rq, nreq, to_mcq(qp->ibv_qp.recv_cq))) { 725219820Sjeff ret = -1; 726219820Sjeff *bad_wr = wr; 727219820Sjeff goto out; 728219820Sjeff } 729219820Sjeff 730219820Sjeff wqe = get_recv_wqe(qp, ind); 731219820Sjeff 732219820Sjeff ((struct mthca_next_seg *) wqe)->flags = 0; 733219820Sjeff 734219820Sjeff wqe += sizeof (struct mthca_next_seg); 735219820Sjeff 736219820Sjeff if (wr->num_sge > qp->rq.max_gs) { 737219820Sjeff ret = -1; 738219820Sjeff *bad_wr = wr; 739219820Sjeff goto out; 740219820Sjeff } 741219820Sjeff 742219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 743219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 744219820Sjeff htonl(wr->sg_list[i].length); 745219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = 746219820Sjeff htonl(wr->sg_list[i].lkey); 747219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 748219820Sjeff htonll(wr->sg_list[i].addr); 749219820Sjeff wqe += sizeof (struct mthca_data_seg); 750219820Sjeff } 751219820Sjeff 752219820Sjeff if (i < qp->rq.max_gs) { 753219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 0; 754219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); 755219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 0; 756219820Sjeff } 757219820Sjeff 758219820Sjeff qp->wrid[ind] = wr->wr_id; 759219820Sjeff 760219820Sjeff ++ind; 761219820Sjeff if (ind >= qp->rq.max) 762219820Sjeff ind -= qp->rq.max; 763219820Sjeff } 764219820Sjeffout: 765219820Sjeff if (nreq) { 766219820Sjeff qp->rq.head += nreq; 767219820Sjeff 768219820Sjeff /* 769219820Sjeff * Make sure that descriptors are written before 770219820Sjeff * doorbell record. 771219820Sjeff */ 772219820Sjeff wmb(); 773219820Sjeff *qp->rq.db = htonl(qp->rq.head & 0xffff); 774219820Sjeff } 775219820Sjeff 776219820Sjeff pthread_spin_unlock(&qp->rq.lock); 777219820Sjeff return ret; 778219820Sjeff} 779219820Sjeff 780219820Sjeffint mthca_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap, 781219820Sjeff enum ibv_qp_type type, struct mthca_qp *qp) 782219820Sjeff{ 783219820Sjeff int size; 784219820Sjeff int max_sq_sge; 785219820Sjeff struct mthca_next_seg *next; 786219820Sjeff int i; 787219820Sjeff 788219820Sjeff qp->rq.max_gs = cap->max_recv_sge; 789219820Sjeff qp->sq.max_gs = cap->max_send_sge; 790219820Sjeff max_sq_sge = align(cap->max_inline_data + sizeof (struct mthca_inline_seg), 791219820Sjeff sizeof (struct mthca_data_seg)) / sizeof (struct mthca_data_seg); 792219820Sjeff if (max_sq_sge < cap->max_send_sge) 793219820Sjeff max_sq_sge = cap->max_send_sge; 794219820Sjeff 795219820Sjeff qp->wrid = malloc((qp->rq.max + qp->sq.max) * sizeof (uint64_t)); 796219820Sjeff if (!qp->wrid) 797219820Sjeff return -1; 798219820Sjeff 799219820Sjeff size = sizeof (struct mthca_next_seg) + 800219820Sjeff qp->rq.max_gs * sizeof (struct mthca_data_seg); 801219820Sjeff 802219820Sjeff for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size; 803219820Sjeff qp->rq.wqe_shift++) 804219820Sjeff ; /* nothing */ 805219820Sjeff 806219820Sjeff size = max_sq_sge * sizeof (struct mthca_data_seg); 807219820Sjeff switch (type) { 808219820Sjeff case IBV_QPT_UD: 809219820Sjeff size += mthca_is_memfree(pd->context) ? 810219820Sjeff sizeof (struct mthca_arbel_ud_seg) : 811219820Sjeff sizeof (struct mthca_tavor_ud_seg); 812219820Sjeff break; 813219820Sjeff 814219820Sjeff case IBV_QPT_UC: 815219820Sjeff size += sizeof (struct mthca_raddr_seg); 816219820Sjeff break; 817219820Sjeff 818219820Sjeff case IBV_QPT_RC: 819219820Sjeff size += sizeof (struct mthca_raddr_seg); 820219820Sjeff /* 821219820Sjeff * An atomic op will require an atomic segment, a 822219820Sjeff * remote address segment and one scatter entry. 823219820Sjeff */ 824219820Sjeff if (size < (sizeof (struct mthca_atomic_seg) + 825219820Sjeff sizeof (struct mthca_raddr_seg) + 826219820Sjeff sizeof (struct mthca_data_seg))) 827219820Sjeff size = (sizeof (struct mthca_atomic_seg) + 828219820Sjeff sizeof (struct mthca_raddr_seg) + 829219820Sjeff sizeof (struct mthca_data_seg)); 830219820Sjeff break; 831219820Sjeff 832219820Sjeff default: 833219820Sjeff break; 834219820Sjeff } 835219820Sjeff 836219820Sjeff /* Make sure that we have enough space for a bind request */ 837219820Sjeff if (size < sizeof (struct mthca_bind_seg)) 838219820Sjeff size = sizeof (struct mthca_bind_seg); 839219820Sjeff 840219820Sjeff size += sizeof (struct mthca_next_seg); 841219820Sjeff 842219820Sjeff for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size; 843219820Sjeff qp->sq.wqe_shift++) 844219820Sjeff ; /* nothing */ 845219820Sjeff 846219820Sjeff qp->send_wqe_offset = align(qp->rq.max << qp->rq.wqe_shift, 847219820Sjeff 1 << qp->sq.wqe_shift); 848219820Sjeff 849219820Sjeff qp->buf_size = qp->send_wqe_offset + (qp->sq.max << qp->sq.wqe_shift); 850219820Sjeff 851219820Sjeff if (mthca_alloc_buf(&qp->buf, 852219820Sjeff align(qp->buf_size, to_mdev(pd->context->device)->page_size), 853219820Sjeff to_mdev(pd->context->device)->page_size)) { 854219820Sjeff free(qp->wrid); 855219820Sjeff return -1; 856219820Sjeff } 857219820Sjeff 858219820Sjeff memset(qp->buf.buf, 0, qp->buf_size); 859219820Sjeff 860219820Sjeff if (mthca_is_memfree(pd->context)) { 861219820Sjeff struct mthca_data_seg *scatter; 862219820Sjeff uint32_t sz; 863219820Sjeff 864219820Sjeff sz = htonl((sizeof (struct mthca_next_seg) + 865219820Sjeff qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16); 866219820Sjeff 867219820Sjeff for (i = 0; i < qp->rq.max; ++i) { 868219820Sjeff next = get_recv_wqe(qp, i); 869219820Sjeff next->nda_op = htonl(((i + 1) & (qp->rq.max - 1)) << 870219820Sjeff qp->rq.wqe_shift); 871219820Sjeff next->ee_nds = sz; 872219820Sjeff 873219820Sjeff for (scatter = (void *) (next + 1); 874219820Sjeff (void *) scatter < (void *) next + (1 << qp->rq.wqe_shift); 875219820Sjeff ++scatter) 876219820Sjeff scatter->lkey = htonl(MTHCA_INVAL_LKEY); 877219820Sjeff } 878219820Sjeff 879219820Sjeff for (i = 0; i < qp->sq.max; ++i) { 880219820Sjeff next = get_send_wqe(qp, i); 881219820Sjeff next->nda_op = htonl((((i + 1) & (qp->sq.max - 1)) << 882219820Sjeff qp->sq.wqe_shift) + 883219820Sjeff qp->send_wqe_offset); 884219820Sjeff } 885219820Sjeff } else { 886219820Sjeff for (i = 0; i < qp->rq.max; ++i) { 887219820Sjeff next = get_recv_wqe(qp, i); 888219820Sjeff next->nda_op = htonl((((i + 1) % qp->rq.max) << 889219820Sjeff qp->rq.wqe_shift) | 1); 890219820Sjeff } 891219820Sjeff } 892219820Sjeff 893219820Sjeff qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); 894219820Sjeff qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); 895219820Sjeff 896219820Sjeff return 0; 897219820Sjeff} 898219820Sjeff 899219820Sjeffstruct mthca_qp *mthca_find_qp(struct mthca_context *ctx, uint32_t qpn) 900219820Sjeff{ 901219820Sjeff int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift; 902219820Sjeff 903219820Sjeff if (ctx->qp_table[tind].refcnt) 904219820Sjeff return ctx->qp_table[tind].table[qpn & ctx->qp_table_mask]; 905219820Sjeff else 906219820Sjeff return NULL; 907219820Sjeff} 908219820Sjeff 909219820Sjeffint mthca_store_qp(struct mthca_context *ctx, uint32_t qpn, struct mthca_qp *qp) 910219820Sjeff{ 911219820Sjeff int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift; 912219820Sjeff 913219820Sjeff if (!ctx->qp_table[tind].refcnt) { 914219820Sjeff ctx->qp_table[tind].table = calloc(ctx->qp_table_mask + 1, 915219820Sjeff sizeof (struct mthca_qp *)); 916219820Sjeff if (!ctx->qp_table[tind].table) 917219820Sjeff return -1; 918219820Sjeff } 919219820Sjeff 920219820Sjeff ++ctx->qp_table[tind].refcnt; 921219820Sjeff ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = qp; 922219820Sjeff return 0; 923219820Sjeff} 924219820Sjeff 925219820Sjeffvoid mthca_clear_qp(struct mthca_context *ctx, uint32_t qpn) 926219820Sjeff{ 927219820Sjeff int tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift; 928219820Sjeff 929219820Sjeff if (!--ctx->qp_table[tind].refcnt) 930219820Sjeff free(ctx->qp_table[tind].table); 931219820Sjeff else 932219820Sjeff ctx->qp_table[tind].table[qpn & ctx->qp_table_mask] = NULL; 933219820Sjeff} 934219820Sjeff 935219820Sjeffint mthca_free_err_wqe(struct mthca_qp *qp, int is_send, 936219820Sjeff int index, int *dbd, uint32_t *new_wqe) 937219820Sjeff{ 938219820Sjeff struct mthca_next_seg *next; 939219820Sjeff 940219820Sjeff /* 941219820Sjeff * For SRQs, all receive WQEs generate a CQE, so we're always 942219820Sjeff * at the end of the doorbell chain. 943219820Sjeff */ 944219820Sjeff if (qp->ibv_qp.srq && !is_send) { 945219820Sjeff *new_wqe = 0; 946219820Sjeff return 0; 947219820Sjeff } 948219820Sjeff 949219820Sjeff if (is_send) 950219820Sjeff next = get_send_wqe(qp, index); 951219820Sjeff else 952219820Sjeff next = get_recv_wqe(qp, index); 953219820Sjeff 954219820Sjeff *dbd = !!(next->ee_nds & htonl(MTHCA_NEXT_DBD)); 955219820Sjeff if (next->ee_nds & htonl(0x3f)) 956219820Sjeff *new_wqe = (next->nda_op & htonl(~0x3f)) | 957219820Sjeff (next->ee_nds & htonl(0x3f)); 958219820Sjeff else 959219820Sjeff *new_wqe = 0; 960219820Sjeff 961219820Sjeff return 0; 962219820Sjeff} 963219820Sjeff 964