cq.c revision 285830
1193323Sed/* 2193323Sed * Copyright (c) 2005 Topspin Communications. All rights reserved. 3193323Sed * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved. 4193323Sed * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved. 5193323Sed * 6193323Sed * This software is available to you under a choice of one of two 7193323Sed * licenses. You may choose to be licensed under the terms of the GNU 8193323Sed * General Public License (GPL) Version 2, available from the file 9193323Sed * COPYING in the main directory of this source tree, or the 10193323Sed * OpenIB.org BSD license below: 11193323Sed * 12193323Sed * Redistribution and use in source and binary forms, with or 13193323Sed * without modification, are permitted provided that the following 14193323Sed * conditions are met: 15193323Sed * 16193323Sed * - Redistributions of source code must retain the above 17193323Sed * copyright notice, this list of conditions and the following 18193323Sed * disclaimer. 19193323Sed * 20193323Sed * - Redistributions in binary form must reproduce the above 21193323Sed * copyright notice, this list of conditions and the following 22193323Sed * disclaimer in the documentation and/or other materials 23249423Sdim * provided with the distribution. 24193323Sed * 25193323Sed * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26193323Sed * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27193323Sed * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28193323Sed * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29193323Sed * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30193323Sed * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31193323Sed * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32193323Sed * SOFTWARE. 33193323Sed */ 34193323Sed 35193323Sed#if HAVE_CONFIG_H 36193323Sed# include <config.h> 37193323Sed#endif /* HAVE_CONFIG_H */ 38193323Sed 39193323Sed#include <stdio.h> 40193323Sed#include <stdlib.h> 41193323Sed#include <pthread.h> 42218893Sdim#include <netinet/in.h> 43218893Sdim#include <string.h> 44193323Sed 45193323Sed#include <infiniband/opcode.h> 46218893Sdim 47218893Sdim#include "mlx4.h" 48193323Sed#include "doorbell.h" 49193323Sed 50193323Sedenum { 51193323Sed MLX4_CQ_DOORBELL = 0x20 52193323Sed}; 53193323Sed 54193323Sedenum { 55193323Sed CQ_OK = 0, 56193323Sed CQ_EMPTY = -1, 57193323Sed CQ_POLL_ERR = -2 58193323Sed}; 59218893Sdim 60193323Sed#define MLX4_CQ_DB_REQ_NOT_SOL (1 << 24) 61218893Sdim#define MLX4_CQ_DB_REQ_NOT (2 << 24) 62218893Sdim 63218893Sdimenum { 64218893Sdim MLX4_CQE_OWNER_MASK = 0x80, 65218893Sdim MLX4_CQE_IS_SEND_MASK = 0x40, 66218893Sdim MLX4_CQE_OPCODE_MASK = 0x1f 67218893Sdim}; 68193323Sed 69193323Sedenum { 70218893Sdim MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01, 71193323Sed MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02, 72218893Sdim MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04, 73218893Sdim MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05, 74218893Sdim MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06, 75218893Sdim MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10, 76218893Sdim MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11, 77218893Sdim MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, 78218893Sdim MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13, 79218893Sdim MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14, 80218893Sdim MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15, 81218893Sdim MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16, 82193323Sed MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22, 83193323Sed}; 84193323Sed 85193323Sedstruct mlx4_cqe { 86193323Sed uint32_t my_qpn; 87193323Sed uint32_t immed_rss_invalid; 88193323Sed uint32_t g_mlpath_rqpn; 89193323Sed uint8_t sl; 90193323Sed uint8_t reserved1; 91193323Sed uint16_t rlid; 92193323Sed uint32_t reserved2; 93193323Sed uint32_t byte_cnt; 94193323Sed uint16_t wqe_index; 95193323Sed uint16_t checksum; 96193323Sed uint8_t reserved3[3]; 97193323Sed uint8_t owner_sr_opcode; 98193323Sed}; 99193323Sed 100193323Sedstruct mlx4_err_cqe { 101193323Sed uint32_t my_qpn; 102193323Sed uint32_t reserved1[5]; 103193323Sed uint16_t wqe_index; 104193323Sed uint8_t vendor_err; 105193323Sed uint8_t syndrome; 106193323Sed uint8_t reserved2[3]; 107193323Sed uint8_t owner_sr_opcode; 108193323Sed}; 109193323Sed 110193323Sedstatic struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry) 111193323Sed{ 112226633Sdim return cq->buf.buf + entry * cq->cqe_size; 113226633Sdim} 114193323Sed 115193323Sedstatic void *get_sw_cqe(struct mlx4_cq *cq, int n) 116193323Sed{ 117193323Sed struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe); 118193323Sed struct mlx4_cqe *tcqe = cq->cqe_size == 64 ? cqe + 1 : cqe; 119193323Sed 120193323Sed return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^ 121193323Sed !!(n & (cq->ibv_cq.cqe + 1))) ? NULL : tcqe; 122193323Sed} 123193323Sed 124193323Sedstatic struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq) 125193323Sed{ 126193323Sed return get_sw_cqe(cq, cq->cons_index); 127193323Sed} 128193323Sed 129193323Sedstatic void update_cons_index(struct mlx4_cq *cq) 130193323Sed{ 131193323Sed *cq->set_ci_db = htonl(cq->cons_index & 0xffffff); 132193323Sed} 133193323Sed 134193323Sedstatic void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, struct ibv_wc *wc) 135193323Sed{ 136193323Sed if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) 137193323Sed printf(PFX "local QP operation err " 138193323Sed "(QPN %06x, WQE index %x, vendor syndrome %02x, " 139193323Sed "opcode = %02x)\n", 140193323Sed htonl(cqe->my_qpn), htonl(cqe->wqe_index), 141193323Sed cqe->vendor_err, 142193323Sed cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); 143193323Sed 144193323Sed switch (cqe->syndrome) { 145193323Sed case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR: 146193323Sed wc->status = IBV_WC_LOC_LEN_ERR; 147193323Sed break; 148226633Sdim case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR: 149226633Sdim wc->status = IBV_WC_LOC_QP_OP_ERR; 150226633Sdim break; 151212904Sdim case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR: 152212904Sdim wc->status = IBV_WC_LOC_PROT_ERR; 153193323Sed break; 154193323Sed case MLX4_CQE_SYNDROME_WR_FLUSH_ERR: 155193323Sed wc->status = IBV_WC_WR_FLUSH_ERR; 156193323Sed break; 157193323Sed case MLX4_CQE_SYNDROME_MW_BIND_ERR: 158193323Sed wc->status = IBV_WC_MW_BIND_ERR; 159193323Sed break; 160193323Sed case MLX4_CQE_SYNDROME_BAD_RESP_ERR: 161193323Sed wc->status = IBV_WC_BAD_RESP_ERR; 162193323Sed break; 163193323Sed case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR: 164193323Sed wc->status = IBV_WC_LOC_ACCESS_ERR; 165193323Sed break; 166193323Sed case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: 167193323Sed wc->status = IBV_WC_REM_INV_REQ_ERR; 168193323Sed break; 169193323Sed case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR: 170193323Sed wc->status = IBV_WC_REM_ACCESS_ERR; 171193323Sed break; 172193323Sed case MLX4_CQE_SYNDROME_REMOTE_OP_ERR: 173193323Sed wc->status = IBV_WC_REM_OP_ERR; 174193323Sed break; 175193323Sed case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: 176193323Sed wc->status = IBV_WC_RETRY_EXC_ERR; 177193323Sed break; 178193323Sed case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR: 179198090Srdivacky wc->status = IBV_WC_RNR_RETRY_EXC_ERR; 180193323Sed break; 181193323Sed case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR: 182193323Sed wc->status = IBV_WC_REM_ABORT_ERR; 183198090Srdivacky break; 184198090Srdivacky default: 185193323Sed wc->status = IBV_WC_GENERAL_ERR; 186193323Sed break; 187193323Sed } 188193323Sed 189193323Sed wc->vendor_err = cqe->vendor_err; 190193323Sed} 191193323Sed 192193323Sedstatic int mlx4_poll_one(struct mlx4_cq *cq, 193193323Sed struct mlx4_qp **cur_qp, 194193323Sed struct ibv_wc *wc) 195193323Sed{ 196193323Sed struct mlx4_wq *wq; 197193323Sed struct mlx4_cqe *cqe; 198193323Sed struct mlx4_srq *srq = NULL; 199193323Sed uint32_t qpn; 200193323Sed uint32_t srqn; 201193323Sed uint32_t g_mlpath_rqpn; 202193323Sed uint16_t wqe_index; 203193323Sed int is_error; 204193323Sed int is_send; 205193323Sed 206218893Sdim cqe = next_cqe_sw(cq); 207218893Sdim if (!cqe) 208193323Sed return CQ_EMPTY; 209193323Sed 210193323Sed ++cq->cons_index; 211193323Sed 212193323Sed VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof *cqe); 213193323Sed 214193323Sed /* 215193323Sed * Make sure we read CQ entry contents after we've checked the 216193323Sed * ownership bit. 217193323Sed */ 218193323Sed rmb(); 219193323Sed 220193323Sed qpn = ntohl(cqe->my_qpn); 221193323Sed 222193323Sed is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK; 223193323Sed is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == 224193323Sed MLX4_CQE_OPCODE_ERROR; 225193323Sed 226193323Sed if (qpn & MLX4_XRC_QPN_BIT && !is_send) { 227243830Sdim srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff; 228243830Sdim /* 229193323Sed * We do not have to take the XRC SRQ table lock here, 230193323Sed * because CQs will be locked while XRC SRQs are removed 231193323Sed * from the table. 232193323Sed */ 233193323Sed srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn); 234193323Sed if (!srq) 235193323Sed return CQ_POLL_ERR; 236193323Sed } else if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) { 237193323Sed /* 238193323Sed * We do not have to take the QP table lock here, 239193323Sed * because CQs will be locked while QPs are removed 240193323Sed * from the table. 241193323Sed */ 242193323Sed *cur_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context), 243193323Sed qpn & 0xffffff); 244193323Sed if (!*cur_qp) 245193323Sed return CQ_POLL_ERR; 246193323Sed } 247193323Sed 248193323Sed wc->qp_num = qpn & 0xffffff; 249193323Sed 250193323Sed if (is_send) { 251218893Sdim wq = &(*cur_qp)->sq; 252218893Sdim wqe_index = ntohs(cqe->wqe_index); 253193323Sed wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail); 254226633Sdim wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 255226633Sdim ++wq->tail; 256226633Sdim } else if (srq) { 257226633Sdim wqe_index = htons(cqe->wqe_index); 258226633Sdim wc->wr_id = srq->wrid[wqe_index]; 259226633Sdim mlx4_free_srq_wqe(srq, wqe_index); 260221345Sdim } else if ((*cur_qp)->ibv_qp.srq) { 261193323Sed srq = to_msrq((*cur_qp)->ibv_qp.srq); 262193323Sed wqe_index = htons(cqe->wqe_index); 263193323Sed wc->wr_id = srq->wrid[wqe_index]; 264193323Sed mlx4_free_srq_wqe(srq, wqe_index); 265234353Sdim } else { 266193323Sed wq = &(*cur_qp)->rq; 267193323Sed wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 268193323Sed ++wq->tail; 269218893Sdim } 270218893Sdim 271226633Sdim if (is_error) { 272193323Sed mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc); 273193323Sed return CQ_OK; 274198090Srdivacky } 275193323Sed 276193323Sed wc->status = IBV_WC_SUCCESS; 277193323Sed 278193323Sed if (is_send) { 279193323Sed wc->wc_flags = 0; 280193323Sed switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { 281198090Srdivacky case MLX4_OPCODE_RDMA_WRITE_IMM: 282198090Srdivacky wc->wc_flags |= IBV_WC_WITH_IMM; 283198090Srdivacky case MLX4_OPCODE_RDMA_WRITE: 284198090Srdivacky wc->opcode = IBV_WC_RDMA_WRITE; 285198090Srdivacky break; 286221345Sdim case MLX4_OPCODE_SEND_IMM: 287198090Srdivacky wc->wc_flags |= IBV_WC_WITH_IMM; 288198090Srdivacky case MLX4_OPCODE_SEND: 289198090Srdivacky wc->opcode = IBV_WC_SEND; 290198090Srdivacky break; 291200581Srdivacky case MLX4_OPCODE_RDMA_READ: 292200581Srdivacky wc->opcode = IBV_WC_RDMA_READ; 293200581Srdivacky wc->byte_len = ntohl(cqe->byte_cnt); 294198090Srdivacky break; 295193323Sed case MLX4_OPCODE_ATOMIC_CS: 296193323Sed wc->opcode = IBV_WC_COMP_SWAP; 297193323Sed wc->byte_len = 8; 298198090Srdivacky break; 299198090Srdivacky case MLX4_OPCODE_ATOMIC_FA: 300198090Srdivacky wc->opcode = IBV_WC_FETCH_ADD; 301198090Srdivacky wc->byte_len = 8; 302193323Sed break; 303198090Srdivacky case MLX4_OPCODE_BIND_MW: 304198090Srdivacky wc->opcode = IBV_WC_BIND_MW; 305193323Sed break; 306193323Sed default: 307193323Sed /* assume it's a send completion */ 308193323Sed wc->opcode = IBV_WC_SEND; 309193323Sed break; 310193323Sed } 311193323Sed } else { 312193323Sed wc->byte_len = ntohl(cqe->byte_cnt); 313193323Sed 314193323Sed switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) { 315193323Sed case MLX4_RECV_OPCODE_RDMA_WRITE_IMM: 316193323Sed wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; 317193323Sed wc->wc_flags = IBV_WC_WITH_IMM; 318193323Sed wc->imm_data = cqe->immed_rss_invalid; 319193323Sed break; 320193323Sed case MLX4_RECV_OPCODE_SEND: 321193323Sed wc->opcode = IBV_WC_RECV; 322193323Sed wc->wc_flags = 0; 323193323Sed break; 324218893Sdim case MLX4_RECV_OPCODE_SEND_IMM: 325193323Sed wc->opcode = IBV_WC_RECV; 326193323Sed wc->wc_flags = IBV_WC_WITH_IMM; 327193323Sed wc->imm_data = cqe->immed_rss_invalid; 328193323Sed break; 329193323Sed } 330193323Sed 331226633Sdim wc->slid = ntohs(cqe->rlid); 332193323Sed wc->sl = cqe->sl >> 4; 333193323Sed g_mlpath_rqpn = ntohl(cqe->g_mlpath_rqpn); 334193323Sed wc->src_qp = g_mlpath_rqpn & 0xffffff; 335193323Sed wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f; 336218893Sdim wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IBV_WC_GRH : 0; 337218893Sdim wc->pkey_index = ntohl(cqe->immed_rss_invalid) & 0x7f; 338193323Sed } 339193323Sed 340193323Sed return CQ_OK; 341193323Sed} 342193323Sed 343226633Sdimint mlx4_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 344193323Sed{ 345193323Sed struct mlx4_cq *cq = to_mcq(ibcq); 346193323Sed struct mlx4_qp *qp = NULL; 347193323Sed int npolled; 348193323Sed int err = CQ_OK; 349193323Sed 350193323Sed pthread_spin_lock(&cq->lock); 351193323Sed 352193323Sed for (npolled = 0; npolled < ne; ++npolled) { 353193323Sed err = mlx4_poll_one(cq, &qp, wc + npolled); 354193323Sed if (err != CQ_OK) 355218893Sdim break; 356218893Sdim } 357218893Sdim 358193323Sed if (npolled) 359193323Sed update_cons_index(cq); 360193323Sed 361218893Sdim pthread_spin_unlock(&cq->lock); 362218893Sdim 363218893Sdim return err == CQ_POLL_ERR ? err : npolled; 364193323Sed} 365193323Sed 366193323Sedint mlx4_arm_cq(struct ibv_cq *ibvcq, int solicited) 367193323Sed{ 368193323Sed struct mlx4_cq *cq = to_mcq(ibvcq); 369218893Sdim uint32_t doorbell[2]; 370193323Sed uint32_t sn; 371193323Sed uint32_t ci; 372193323Sed uint32_t cmd; 373193323Sed 374193323Sed sn = cq->arm_sn & 3; 375193323Sed ci = cq->cons_index & 0xffffff; 376193323Sed cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT; 377193323Sed 378193323Sed *cq->arm_db = htonl(sn << 28 | cmd | ci); 379193323Sed 380193323Sed /* 381193323Sed * Make sure that the doorbell record in host memory is 382193323Sed * written before ringing the doorbell via PCI MMIO. 383193323Sed */ 384193323Sed wmb(); 385193323Sed 386193323Sed doorbell[0] = htonl(sn << 28 | cmd | cq->cqn); 387193323Sed doorbell[1] = htonl(ci); 388193323Sed 389193323Sed mlx4_write64(doorbell, to_mctx(ibvcq->context), MLX4_CQ_DOORBELL); 390193323Sed 391193323Sed return 0; 392193323Sed} 393193323Sed 394193323Sedvoid mlx4_cq_event(struct ibv_cq *cq) 395193323Sed{ 396193323Sed to_mcq(cq)->arm_sn++; 397193323Sed} 398193323Sed 399193323Sedvoid __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq) 400198090Srdivacky{ 401193323Sed struct mlx4_cqe *cqe, *dest; 402193323Sed uint32_t prod_index; 403193323Sed uint8_t owner_bit; 404193323Sed int nfreed = 0; 405193323Sed int is_xrc_srq = 0; 406193323Sed int cqe_inc = cq->cqe_size == 64 ? 1 : 0; 407193323Sed 408193323Sed if (srq && srq->ibv_srq.xrc_cq) 409193323Sed is_xrc_srq = 1; 410198090Srdivacky 411193323Sed /* 412193323Sed * First we need to find the current producer index, so we 413193323Sed * know where to start cleaning from. It doesn't matter if HW 414193323Sed * adds new entries after this loop -- the QP we're worried 415193323Sed * about is already in RESET, so the new entries won't come 416218893Sdim * from our QP and therefore don't need to be checked. 417218893Sdim */ 418193323Sed for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index) 419193323Sed if (prod_index == cq->cons_index + cq->ibv_cq.cqe) 420218893Sdim break; 421193323Sed 422193323Sed /* 423193323Sed * Now sweep backwards through the CQ, removing CQ entries 424218893Sdim * that match our QP by copying older entries on top of them. 425218893Sdim */ 426193323Sed while ((int) --prod_index - (int) cq->cons_index >= 0) { 427226633Sdim cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); 428193323Sed cqe += cqe_inc; 429193323Sed if (is_xrc_srq && 430198090Srdivacky (ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) && 431193323Sed !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) { 432193323Sed mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index)); 433193323Sed ++nfreed; 434193323Sed } else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) { 435193323Sed if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) 436193323Sed mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index)); 437193323Sed ++nfreed; 438 } else if (nfreed) { 439 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe); 440 dest += cqe_inc; 441 owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK; 442 memcpy(dest, cqe, sizeof *cqe); 443 dest->owner_sr_opcode = owner_bit | 444 (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK); 445 } 446 } 447 448 if (nfreed) { 449 cq->cons_index += nfreed; 450 /* 451 * Make sure update of buffer contents is done before 452 * updating consumer index. 453 */ 454 wmb(); 455 update_cons_index(cq); 456 } 457} 458 459void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq) 460{ 461 pthread_spin_lock(&cq->lock); 462 __mlx4_cq_clean(cq, qpn, srq); 463 pthread_spin_unlock(&cq->lock); 464} 465 466int mlx4_get_outstanding_cqes(struct mlx4_cq *cq) 467{ 468 uint32_t i; 469 470 for (i = cq->cons_index; get_sw_cqe(cq, (i & cq->ibv_cq.cqe)); ++i) 471 ; 472 473 return i - cq->cons_index; 474} 475 476void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe) 477{ 478 struct mlx4_cqe *cqe; 479 int i; 480 int cqe_inc = cq->cqe_size == 64 ? 1 : 0; 481 482 i = cq->cons_index; 483 cqe = get_cqe(cq, (i & old_cqe)); 484 cqe += cqe_inc; 485 486 while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) { 487 cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) | 488 (((i + 1) & (cq->ibv_cq.cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); 489 memcpy(buf + ((i + 1) & cq->ibv_cq.cqe) * cq->cqe_size, 490 cqe - cqe_inc, cq->cqe_size); 491 ++i; 492 cqe = get_cqe(cq, (i & old_cqe)); 493 cqe += cqe_inc; 494 } 495 496 ++cq->cons_index; 497} 498 499int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent, 500 int entry_size) 501{ 502 if (mlx4_alloc_buf(buf, align(nent * entry_size, dev->page_size), 503 dev->page_size)) 504 return -1; 505 memset(buf->buf, 0, nent * entry_size); 506 507 return 0; 508} 509