1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Topspin Communications. All rights reserved. 3219820Sjeff * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. 4219820Sjeff * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. 5219820Sjeff * 6219820Sjeff * This software is available to you under a choice of one of two 7219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 8219820Sjeff * General Public License (GPL) Version 2, available from the file 9219820Sjeff * COPYING in the main directory of this source tree, or the 10219820Sjeff * OpenIB.org BSD license below: 11219820Sjeff * 12219820Sjeff * Redistribution and use in source and binary forms, with or 13219820Sjeff * without modification, are permitted provided that the following 14219820Sjeff * conditions are met: 15219820Sjeff * 16219820Sjeff * - Redistributions of source code must retain the above 17219820Sjeff * copyright notice, this list of conditions and the following 18219820Sjeff * disclaimer. 19219820Sjeff * 20219820Sjeff * - Redistributions in binary form must reproduce the above 21219820Sjeff * copyright notice, this list of conditions and the following 22219820Sjeff * disclaimer in the documentation and/or other materials 23219820Sjeff * provided with the distribution. 24219820Sjeff * 25219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32219820Sjeff * SOFTWARE. 33219820Sjeff */ 34219820Sjeff 35219820Sjeff#if HAVE_CONFIG_H 36219820Sjeff# include <config.h> 37219820Sjeff#endif /* HAVE_CONFIG_H */ 38219820Sjeff 39219820Sjeff#include <stdio.h> 40219820Sjeff#include <stdlib.h> 41219820Sjeff#include <pthread.h> 42219820Sjeff#include <netinet/in.h> 43219820Sjeff#include <string.h> 44219820Sjeff 45219820Sjeff#include <infiniband/opcode.h> 46219820Sjeff 47219820Sjeff#include "mlx4.h" 48219820Sjeff#include "doorbell.h" 49219820Sjeff 50219820Sjeffenum { 51219820Sjeff MLX4_CQ_DOORBELL = 0x20 52219820Sjeff}; 53219820Sjeff 54219820Sjeffenum { 55219820Sjeff CQ_OK = 0, 56219820Sjeff CQ_EMPTY = -1, 57219820Sjeff CQ_POLL_ERR = -2 58219820Sjeff}; 59219820Sjeff 60219820Sjeff#define MLX4_CQ_DB_REQ_NOT_SOL (1 << 24) 61219820Sjeff#define MLX4_CQ_DB_REQ_NOT (2 << 24) 62219820Sjeff 63219820Sjeffenum { 64219820Sjeff MLX4_CQE_OWNER_MASK = 0x80, 65219820Sjeff MLX4_CQE_IS_SEND_MASK = 0x40, 66219820Sjeff MLX4_CQE_OPCODE_MASK = 0x1f 67219820Sjeff}; 68219820Sjeff 69219820Sjeffenum { 70219820Sjeff MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, 71219820Sjeff MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, 72219820Sjeff MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, 73219820Sjeff MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, 74219820Sjeff MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06, 75219820Sjeff MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10, 76219820Sjeff MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, 77219820Sjeff MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, 78219820Sjeff MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, 79219820Sjeff MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, 80219820Sjeff MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, 81219820Sjeff MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, 82219820Sjeff MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, 83219820Sjeff}; 84219820Sjeff 85219820Sjeffstruct mlx4_cqe { 86219820Sjeff uint32_t my_qpn; 87219820Sjeff uint32_t immed_rss_invalid; 88219820Sjeff uint32_t g_mlpath_rqpn; 89219820Sjeff uint8_t sl; 90219820Sjeff uint8_t reserved1; 91219820Sjeff uint16_t rlid; 92219820Sjeff uint32_t reserved2; 93219820Sjeff uint32_t byte_cnt; 94219820Sjeff uint16_t wqe_index; 95219820Sjeff uint16_t checksum; 96219820Sjeff uint8_t reserved3[3]; 97219820Sjeff uint8_t owner_sr_opcode; 98219820Sjeff}; 99219820Sjeff 100219820Sjeffstruct mlx4_err_cqe { 101219820Sjeff uint32_t my_qpn; 102219820Sjeff uint32_t reserved1[5]; 103219820Sjeff uint16_t wqe_index; 104219820Sjeff uint8_t vendor_err; 105219820Sjeff uint8_t syndrome; 106219820Sjeff uint8_t reserved2[3]; 107219820Sjeff uint8_t owner_sr_opcode; 108219820Sjeff}; 109219820Sjeff 110219820Sjeffstatic struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry) 111219820Sjeff{ 112275228Shselasky return cq->buf.buf + entry * cq->cqe_size; 113219820Sjeff} 114219820Sjeff 115219820Sjeffstatic void *get_sw_cqe(struct mlx4_cq *cq, int n) 116219820Sjeff{ 117219820Sjeff struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe); 118275228Shselasky struct mlx4_cqe *tcqe = cq->cqe_size == 64 ? cqe + 1 : cqe; 119219820Sjeff 120275228Shselasky return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ 121275228Shselasky !!(n & (cq->ibv_cq.cqe + 1))) ? NULL : tcqe; 122219820Sjeff} 123219820Sjeff 124219820Sjeffstatic struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq) 125219820Sjeff{ 126219820Sjeff return get_sw_cqe(cq, cq->cons_index); 127219820Sjeff} 128219820Sjeff 129219820Sjeffstatic void update_cons_index(struct mlx4_cq *cq) 130219820Sjeff{ 131219820Sjeff *cq->set_ci_db = htonl(cq->cons_index & 0xffffff); 132219820Sjeff} 133219820Sjeff 134219820Sjeffstatic void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, struct ibv_wc *wc) 135219820Sjeff{ 136219820Sjeff if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) 137219820Sjeff printf(PFX "local QP operation err " 138219820Sjeff "(QPN %06x, WQE index %x, vendor syndrome %02x, " 139219820Sjeff "opcode = %02x)\n", 140219820Sjeff htonl(cqe->my_qpn), htonl(cqe->wqe_index), 141219820Sjeff cqe->vendor_err, 142219820Sjeff cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); 143219820Sjeff 144219820Sjeff switch (cqe->syndrome) { 145219820Sjeff case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: 146219820Sjeff wc->status = IBV_WC_LOC_LEN_ERR; 147219820Sjeff break; 148219820Sjeff case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: 149219820Sjeff wc->status = IBV_WC_LOC_QP_OP_ERR; 150219820Sjeff break; 151219820Sjeff case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: 152219820Sjeff wc->status = IBV_WC_LOC_PROT_ERR; 153219820Sjeff break; 154219820Sjeff case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: 155219820Sjeff wc->status = IBV_WC_WR_FLUSH_ERR; 156219820Sjeff break; 157219820Sjeff case MLX4_CQE_SYNDROME_MW_BIND_ERR: 158219820Sjeff wc->status = IBV_WC_MW_BIND_ERR; 159219820Sjeff break; 160219820Sjeff case MLX4_CQE_SYNDROME_BAD_RESP_ERR: 161219820Sjeff wc->status = IBV_WC_BAD_RESP_ERR; 162219820Sjeff break; 163219820Sjeff case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: 164219820Sjeff wc->status = IBV_WC_LOC_ACCESS_ERR; 165219820Sjeff break; 166219820Sjeff case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: 167219820Sjeff wc->status = IBV_WC_REM_INV_REQ_ERR; 168219820Sjeff break; 169219820Sjeff case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: 170219820Sjeff wc->status = IBV_WC_REM_ACCESS_ERR; 171219820Sjeff break; 172219820Sjeff case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: 173219820Sjeff wc->status = IBV_WC_REM_OP_ERR; 174219820Sjeff break; 175219820Sjeff case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: 176219820Sjeff wc->status = IBV_WC_RETRY_EXC_ERR; 177219820Sjeff break; 178219820Sjeff case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: 179219820Sjeff wc->status = IBV_WC_RNR_RETRY_EXC_ERR; 180219820Sjeff break; 181219820Sjeff case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: 182219820Sjeff wc->status = IBV_WC_REM_ABORT_ERR; 183219820Sjeff break; 184219820Sjeff default: 185219820Sjeff wc->status = IBV_WC_GENERAL_ERR; 186219820Sjeff break; 187219820Sjeff } 188219820Sjeff 189219820Sjeff wc->vendor_err = cqe->vendor_err; 190219820Sjeff} 191219820Sjeff 192219820Sjeffstatic int mlx4_poll_one(struct mlx4_cq *cq, 193219820Sjeff struct mlx4_qp **cur_qp, 194219820Sjeff struct ibv_wc *wc) 195219820Sjeff{ 196219820Sjeff struct mlx4_wq *wq; 197219820Sjeff struct mlx4_cqe *cqe; 198219820Sjeff struct mlx4_srq *srq = NULL; 199219820Sjeff uint32_t qpn; 200219820Sjeff uint32_t srqn; 201219820Sjeff uint32_t g_mlpath_rqpn; 202219820Sjeff uint16_t wqe_index; 203219820Sjeff int is_error; 204219820Sjeff int is_send; 205219820Sjeff 206219820Sjeff cqe = next_cqe_sw(cq); 207219820Sjeff if (!cqe) 208219820Sjeff return CQ_EMPTY; 209219820Sjeff 210219820Sjeff ++cq->cons_index; 211219820Sjeff 212219820Sjeff VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof *cqe); 213219820Sjeff 214219820Sjeff /* 215219820Sjeff * Make sure we read CQ entry contents after we've checked the 216219820Sjeff * ownership bit. 217219820Sjeff */ 218219820Sjeff rmb(); 219219820Sjeff 220219820Sjeff qpn = ntohl(cqe->my_qpn); 221219820Sjeff 222219820Sjeff is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; 223219820Sjeff is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 224219820Sjeff MLX4_CQE_OPCODE_ERROR; 225219820Sjeff 226219820Sjeff if (qpn & MLX4_XRC_QPN_BIT && !is_send) { 227219820Sjeff srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff; 228219820Sjeff /* 229219820Sjeff * We do not have to take the XRC SRQ table lock here, 230219820Sjeff * because CQs will be locked while XRC SRQs are removed 231219820Sjeff * from the table. 232219820Sjeff */ 233219820Sjeff srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn); 234219820Sjeff if (!srq) 235219820Sjeff return CQ_POLL_ERR; 236219820Sjeff } else if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) { 237219820Sjeff /* 238219820Sjeff * We do not have to take the QP table lock here, 239219820Sjeff * because CQs will be locked while QPs are removed 240219820Sjeff * from the table. 241219820Sjeff */ 242219820Sjeff *cur_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context), 243219820Sjeff qpn & 0xffffff); 244219820Sjeff if (!*cur_qp) 245219820Sjeff return CQ_POLL_ERR; 246219820Sjeff } 247219820Sjeff 248219820Sjeff wc->qp_num = qpn & 0xffffff; 249219820Sjeff 250219820Sjeff if (is_send) { 251219820Sjeff wq = &(*cur_qp)->sq; 252219820Sjeff wqe_index = ntohs(cqe->wqe_index); 253219820Sjeff wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail); 254219820Sjeff wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 255219820Sjeff ++wq->tail; 256219820Sjeff } else if (srq) { 257219820Sjeff wqe_index = htons(cqe->wqe_index); 258219820Sjeff wc->wr_id = srq->wrid[wqe_index]; 259219820Sjeff mlx4_free_srq_wqe(srq, wqe_index); 260219820Sjeff } else if ((*cur_qp)->ibv_qp.srq) { 261219820Sjeff srq = to_msrq((*cur_qp)->ibv_qp.srq); 262219820Sjeff wqe_index = htons(cqe->wqe_index); 263219820Sjeff wc->wr_id = srq->wrid[wqe_index]; 264219820Sjeff mlx4_free_srq_wqe(srq, wqe_index); 265219820Sjeff } else { 266219820Sjeff wq = &(*cur_qp)->rq; 267219820Sjeff wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 268219820Sjeff ++wq->tail; 269219820Sjeff } 270219820Sjeff 271219820Sjeff if (is_error) { 272219820Sjeff mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); 273219820Sjeff return CQ_OK; 274219820Sjeff } 275219820Sjeff 276219820Sjeff wc->status = IBV_WC_SUCCESS; 277219820Sjeff 278219820Sjeff if (is_send) { 279219820Sjeff wc->wc_flags = 0; 280219820Sjeff switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { 281219820Sjeff case MLX4_OPCODE_RDMA_WRITE_IMM: 282219820Sjeff wc->wc_flags |= IBV_WC_WITH_IMM; 283219820Sjeff case MLX4_OPCODE_RDMA_WRITE: 284219820Sjeff wc->opcode = IBV_WC_RDMA_WRITE; 285219820Sjeff break; 286219820Sjeff case MLX4_OPCODE_SEND_IMM: 287219820Sjeff wc->wc_flags |= IBV_WC_WITH_IMM; 288219820Sjeff case MLX4_OPCODE_SEND: 289219820Sjeff wc->opcode = IBV_WC_SEND; 290219820Sjeff break; 291219820Sjeff case MLX4_OPCODE_RDMA_READ: 292219820Sjeff wc->opcode = IBV_WC_RDMA_READ; 293219820Sjeff wc->byte_len = ntohl(cqe->byte_cnt); 294219820Sjeff break; 295219820Sjeff case MLX4_OPCODE_ATOMIC_CS: 296219820Sjeff wc->opcode = IBV_WC_COMP_SWAP; 297219820Sjeff wc->byte_len = 8; 298219820Sjeff break; 299219820Sjeff case MLX4_OPCODE_ATOMIC_FA: 300219820Sjeff wc->opcode = IBV_WC_FETCH_ADD; 301219820Sjeff wc->byte_len = 8; 302219820Sjeff break; 303219820Sjeff case MLX4_OPCODE_BIND_MW: 304219820Sjeff wc->opcode = IBV_WC_BIND_MW; 305219820Sjeff break; 306219820Sjeff default: 307219820Sjeff /* assume it's a send completion */ 308219820Sjeff wc->opcode = IBV_WC_SEND; 309219820Sjeff break; 310219820Sjeff } 311219820Sjeff } else { 312219820Sjeff wc->byte_len = ntohl(cqe->byte_cnt); 313219820Sjeff 314219820Sjeff switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { 315219820Sjeff case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: 316219820Sjeff wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; 317219820Sjeff wc->wc_flags = IBV_WC_WITH_IMM; 318219820Sjeff wc->imm_data = cqe->immed_rss_invalid; 319219820Sjeff break; 320219820Sjeff case MLX4_RECV_OPCODE_SEND: 321219820Sjeff wc->opcode = IBV_WC_RECV; 322219820Sjeff wc->wc_flags = 0; 323219820Sjeff break; 324219820Sjeff case MLX4_RECV_OPCODE_SEND_IMM: 325219820Sjeff wc->opcode = IBV_WC_RECV; 326219820Sjeff wc->wc_flags = IBV_WC_WITH_IMM; 327219820Sjeff wc->imm_data = cqe->immed_rss_invalid; 328219820Sjeff break; 329219820Sjeff } 330219820Sjeff 331219820Sjeff wc->slid = ntohs(cqe->rlid); 332219820Sjeff wc->sl = cqe->sl >> 4; 333219820Sjeff g_mlpath_rqpn = ntohl(cqe->g_mlpath_rqpn); 334219820Sjeff wc->src_qp = g_mlpath_rqpn & 0xffffff; 335219820Sjeff wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f; 336219820Sjeff wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IBV_WC_GRH : 0; 337219820Sjeff wc->pkey_index = ntohl(cqe->immed_rss_invalid) & 0x7f; 338219820Sjeff } 339219820Sjeff 340219820Sjeff return CQ_OK; 341219820Sjeff} 342219820Sjeff 343219820Sjeffint mlx4_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 344219820Sjeff{ 345219820Sjeff struct mlx4_cq *cq = to_mcq(ibcq); 346219820Sjeff struct mlx4_qp *qp = NULL; 347219820Sjeff int npolled; 348219820Sjeff int err = CQ_OK; 349219820Sjeff 350219820Sjeff pthread_spin_lock(&cq->lock); 351219820Sjeff 352219820Sjeff for (npolled = 0; npolled < ne; ++npolled) { 353219820Sjeff err = mlx4_poll_one(cq, &qp, wc + npolled); 354219820Sjeff if (err != CQ_OK) 355219820Sjeff break; 356219820Sjeff } 357219820Sjeff 358219820Sjeff if (npolled) 359219820Sjeff update_cons_index(cq); 360219820Sjeff 361219820Sjeff pthread_spin_unlock(&cq->lock); 362219820Sjeff 363219820Sjeff return err == CQ_POLL_ERR ? err : npolled; 364219820Sjeff} 365219820Sjeff 366219820Sjeffint mlx4_arm_cq(struct ibv_cq *ibvcq, int solicited) 367219820Sjeff{ 368219820Sjeff struct mlx4_cq *cq = to_mcq(ibvcq); 369219820Sjeff uint32_t doorbell[2]; 370219820Sjeff uint32_t sn; 371219820Sjeff uint32_t ci; 372219820Sjeff uint32_t cmd; 373219820Sjeff 374219820Sjeff sn = cq->arm_sn & 3; 375219820Sjeff ci = cq->cons_index & 0xffffff; 376219820Sjeff cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT; 377219820Sjeff 378219820Sjeff *cq->arm_db = htonl(sn << 28 | cmd | ci); 379219820Sjeff 380219820Sjeff /* 381219820Sjeff * Make sure that the doorbell record in host memory is 382219820Sjeff * written before ringing the doorbell via PCI MMIO. 383219820Sjeff */ 384219820Sjeff wmb(); 385219820Sjeff 386219820Sjeff doorbell[0] = htonl(sn << 28 | cmd | cq->cqn); 387219820Sjeff doorbell[1] = htonl(ci); 388219820Sjeff 389219820Sjeff mlx4_write64(doorbell, to_mctx(ibvcq->context), MLX4_CQ_DOORBELL); 390219820Sjeff 391219820Sjeff return 0; 392219820Sjeff} 393219820Sjeff 394219820Sjeffvoid mlx4_cq_event(struct ibv_cq *cq) 395219820Sjeff{ 396219820Sjeff to_mcq(cq)->arm_sn++; 397219820Sjeff} 398219820Sjeff 399219820Sjeffvoid __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq) 400219820Sjeff{ 401219820Sjeff struct mlx4_cqe *cqe, *dest; 402219820Sjeff uint32_t prod_index; 403219820Sjeff uint8_t owner_bit; 404219820Sjeff int nfreed = 0; 405219820Sjeff int is_xrc_srq = 0; 406275228Shselasky int cqe_inc = cq->cqe_size == 64 ? 1 : 0; 407219820Sjeff 408219820Sjeff if (srq && srq->ibv_srq.xrc_cq) 409219820Sjeff is_xrc_srq = 1; 410219820Sjeff 411219820Sjeff /* 412219820Sjeff * First we need to find the current producer index, so we 413219820Sjeff * know where to start cleaning from. It doesn't matter if HW 414219820Sjeff * adds new entries after this loop -- the QP we're worried 415219820Sjeff * about is already in RESET, so the new entries won't come 416219820Sjeff * from our QP and therefore don't need to be checked. 417219820Sjeff */ 418219820Sjeff for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index) 419219820Sjeff if (prod_index == cq->cons_index + cq->ibv_cq.cqe) 420219820Sjeff break; 421219820Sjeff 422219820Sjeff /* 423219820Sjeff * Now sweep backwards through the CQ, removing CQ entries 424219820Sjeff * that match our QP by copying older entries on top of them. 425219820Sjeff */ 426219820Sjeff while ((int) --prod_index - (int) cq->cons_index >= 0) { 427219820Sjeff cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); 428275228Shselasky cqe += cqe_inc; 429219820Sjeff if (is_xrc_srq && 430219820Sjeff (ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) && 431219820Sjeff !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) { 432219820Sjeff mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index)); 433219820Sjeff ++nfreed; 434219820Sjeff } else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) { 435219820Sjeff if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) 436219820Sjeff mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index)); 437219820Sjeff ++nfreed; 438219820Sjeff } else if (nfreed) { 439219820Sjeff dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe); 440275228Shselasky dest += cqe_inc; 441219820Sjeff owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; 442219820Sjeff memcpy(dest, cqe, sizeof *cqe); 443219820Sjeff dest->owner_sr_opcode = owner_bit | 444219820Sjeff (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); 445219820Sjeff } 446219820Sjeff } 447219820Sjeff 448219820Sjeff if (nfreed) { 449219820Sjeff cq->cons_index += nfreed; 450219820Sjeff /* 451219820Sjeff * Make sure update of buffer contents is done before 452219820Sjeff * updating consumer index. 453219820Sjeff */ 454219820Sjeff wmb(); 455219820Sjeff update_cons_index(cq); 456219820Sjeff } 457219820Sjeff} 458219820Sjeff 459219820Sjeffvoid mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq) 460219820Sjeff{ 461219820Sjeff pthread_spin_lock(&cq->lock); 462219820Sjeff __mlx4_cq_clean(cq, qpn, srq); 463219820Sjeff pthread_spin_unlock(&cq->lock); 464219820Sjeff} 465219820Sjeff 466219820Sjeffint mlx4_get_outstanding_cqes(struct mlx4_cq *cq) 467219820Sjeff{ 468219820Sjeff uint32_t i; 469219820Sjeff 470219820Sjeff for (i = cq->cons_index; get_sw_cqe(cq, (i & cq->ibv_cq.cqe)); ++i) 471219820Sjeff ; 472219820Sjeff 473219820Sjeff return i - cq->cons_index; 474219820Sjeff} 475219820Sjeff 476219820Sjeffvoid mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe) 477219820Sjeff{ 478219820Sjeff struct mlx4_cqe *cqe; 479219820Sjeff int i; 480275228Shselasky int cqe_inc = cq->cqe_size == 64 ? 1 : 0; 481219820Sjeff 482219820Sjeff i = cq->cons_index; 483219820Sjeff cqe = get_cqe(cq, (i & old_cqe)); 484275228Shselasky cqe += cqe_inc; 485219820Sjeff 486219820Sjeff while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) { 487219820Sjeff cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) | 488219820Sjeff (((i + 1) & (cq->ibv_cq.cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); 489275228Shselasky memcpy(buf + ((i + 1) & cq->ibv_cq.cqe) * cq->cqe_size, 490275228Shselasky cqe - cqe_inc, cq->cqe_size); 491219820Sjeff ++i; 492219820Sjeff cqe = get_cqe(cq, (i & old_cqe)); 493275228Shselasky cqe += cqe_inc; 494219820Sjeff } 495219820Sjeff 496219820Sjeff ++cq->cons_index; 497219820Sjeff} 498219820Sjeff 499275228Shselaskyint mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent, 500275228Shselasky int entry_size) 501219820Sjeff{ 502275228Shselasky if (mlx4_alloc_buf(buf, align(nent * entry_size, dev->page_size), 503219820Sjeff dev->page_size)) 504219820Sjeff return -1; 505275228Shselasky memset(buf->buf, 0, nent * entry_size); 506219820Sjeff 507219820Sjeff return 0; 508219820Sjeff} 509