1321936Shselasky/* 2321936Shselasky * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3321936Shselasky * 4321936Shselasky * This software is available to you under a choice of one of two 5321936Shselasky * licenses. You may choose to be licensed under the terms of the GNU 6321936Shselasky * General Public License (GPL) Version 2, available from the file 7321936Shselasky * COPYING in the main directory of this source tree, or the 8321936Shselasky * OpenIB.org BSD license below: 9321936Shselasky * 10321936Shselasky * Redistribution and use in source and binary forms, with or 11321936Shselasky * without modification, are permitted provided that the following 12321936Shselasky * conditions are met: 13321936Shselasky * 14321936Shselasky * - Redistributions of source code must retain the above 15321936Shselasky * copyright notice, this list of conditions and the following 16321936Shselasky * disclaimer. 17321936Shselasky * 18321936Shselasky * - Redistributions in binary form must reproduce the above 19321936Shselasky * copyright notice, this list of conditions and the following 20321936Shselasky * disclaimer in the documentation and/or other materials 21321936Shselasky * provided with the distribution. 22321936Shselasky * 23321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30321936Shselasky * SOFTWARE. 31321936Shselasky */ 32321936Shselasky 33321936Shselasky#include <config.h> 34321936Shselasky 35321936Shselasky#include <stdio.h> 36321936Shselasky#include <stdlib.h> 37321936Shselasky#include <pthread.h> 38321936Shselasky#include <string.h> 39321936Shselasky#include <errno.h> 40321936Shselasky#include <unistd.h> 41321936Shselasky 42321936Shselasky#include <infiniband/opcode.h> 43321936Shselasky 44321936Shselasky#include "mlx5.h" 45321936Shselasky#include "wqe.h" 46321936Shselasky#include "doorbell.h" 47321936Shselasky 48321936Shselaskyenum { 49321936Shselasky CQ_OK = 0, 50321936Shselasky CQ_EMPTY = -1, 51321936Shselasky CQ_POLL_ERR = -2 52321936Shselasky}; 53321936Shselasky 54321936Shselaskyenum { 55321936Shselasky MLX5_CQ_MODIFY_RESEIZE = 0, 56321936Shselasky MLX5_CQ_MODIFY_MODER = 1, 57321936Shselasky MLX5_CQ_MODIFY_MAPPING = 2, 58321936Shselasky}; 59321936Shselasky 60321936Shselaskyint mlx5_stall_num_loop = 60; 61321936Shselaskyint mlx5_stall_cq_poll_min = 60; 62321936Shselaskyint mlx5_stall_cq_poll_max = 100000; 63321936Shselaskyint mlx5_stall_cq_inc_step = 100; 64321936Shselaskyint mlx5_stall_cq_dec_step = 10; 65321936Shselasky 66321936Shselaskystatic inline uint8_t get_cqe_l3_hdr_type(struct mlx5_cqe64 *cqe) 67321936Shselasky{ 68321936Shselasky return (cqe->l4_hdr_type_etc >> 2) & 0x3; 69321936Shselasky} 70321936Shselasky 71321936Shselaskystatic void *get_buf_cqe(struct mlx5_buf *buf, int n, int cqe_sz) 72321936Shselasky{ 73321936Shselasky return buf->buf + n * cqe_sz; 74321936Shselasky} 75321936Shselasky 76321936Shselaskystatic void *get_cqe(struct mlx5_cq *cq, int n) 77321936Shselasky{ 78321936Shselasky return cq->active_buf->buf + n * cq->cqe_sz; 79321936Shselasky} 80321936Shselasky 81321936Shselaskystatic void *get_sw_cqe(struct mlx5_cq *cq, int n) 82321936Shselasky{ 83321936Shselasky void *cqe = get_cqe(cq, n & cq->ibv_cq.cqe); 84321936Shselasky struct mlx5_cqe64 *cqe64; 85321936Shselasky 86321936Shselasky cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; 87321936Shselasky 88321936Shselasky if (likely(mlx5dv_get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && 89321936Shselasky !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ibv_cq.cqe + 1)))) { 90321936Shselasky return cqe; 91321936Shselasky } else { 92321936Shselasky return NULL; 93321936Shselasky } 94321936Shselasky} 95321936Shselasky 96321936Shselaskystatic void *next_cqe_sw(struct mlx5_cq *cq) 97321936Shselasky{ 98321936Shselasky return get_sw_cqe(cq, cq->cons_index); 99321936Shselasky} 100321936Shselasky 101321936Shselaskystatic void update_cons_index(struct mlx5_cq *cq) 102321936Shselasky{ 103321936Shselasky cq->dbrec[MLX5_CQ_SET_CI] = htobe32(cq->cons_index & 0xffffff); 104321936Shselasky} 105321936Shselasky 106321936Shselaskystatic inline void handle_good_req(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_wq *wq, int idx) 107321936Shselasky{ 108321936Shselasky switch (be32toh(cqe->sop_drop_qpn) >> 24) { 109321936Shselasky case MLX5_OPCODE_RDMA_WRITE_IMM: 110321936Shselasky wc->wc_flags |= IBV_WC_WITH_IMM; 111321936Shselasky SWITCH_FALLTHROUGH; 112321936Shselasky case MLX5_OPCODE_RDMA_WRITE: 113321936Shselasky wc->opcode = IBV_WC_RDMA_WRITE; 114321936Shselasky break; 115321936Shselasky case MLX5_OPCODE_SEND_IMM: 116321936Shselasky wc->wc_flags |= IBV_WC_WITH_IMM; 117321936Shselasky SWITCH_FALLTHROUGH; 118321936Shselasky case MLX5_OPCODE_SEND: 119321936Shselasky case MLX5_OPCODE_SEND_INVAL: 120321936Shselasky wc->opcode = IBV_WC_SEND; 121321936Shselasky break; 122321936Shselasky case MLX5_OPCODE_RDMA_READ: 123321936Shselasky wc->opcode = IBV_WC_RDMA_READ; 124321936Shselasky wc->byte_len = be32toh(cqe->byte_cnt); 125321936Shselasky break; 126321936Shselasky case MLX5_OPCODE_ATOMIC_CS: 127321936Shselasky wc->opcode = IBV_WC_COMP_SWAP; 128321936Shselasky wc->byte_len = 8; 129321936Shselasky break; 130321936Shselasky case MLX5_OPCODE_ATOMIC_FA: 131321936Shselasky wc->opcode = IBV_WC_FETCH_ADD; 132321936Shselasky wc->byte_len = 8; 133321936Shselasky break; 134321936Shselasky case MLX5_OPCODE_UMR: 135321936Shselasky wc->opcode = wq->wr_data[idx]; 136321936Shselasky break; 137321936Shselasky case MLX5_OPCODE_TSO: 138321936Shselasky wc->opcode = IBV_WC_TSO; 139321936Shselasky break; 140321936Shselasky } 141321936Shselasky} 142321936Shselasky 143321936Shselaskystatic inline int handle_responder_lazy(struct mlx5_cq *cq, struct mlx5_cqe64 *cqe, 144321936Shselasky struct mlx5_resource *cur_rsc, struct mlx5_srq *srq) 145321936Shselasky{ 146321936Shselasky uint16_t wqe_ctr; 147321936Shselasky struct mlx5_wq *wq; 148321936Shselasky struct mlx5_qp *qp = rsc_to_mqp(cur_rsc); 149321936Shselasky int err = IBV_WC_SUCCESS; 150321936Shselasky 151321936Shselasky if (srq) { 152321936Shselasky wqe_ctr = be16toh(cqe->wqe_counter); 153321936Shselasky cq->ibv_cq.wr_id = srq->wrid[wqe_ctr]; 154321936Shselasky mlx5_free_srq_wqe(srq, wqe_ctr); 155321936Shselasky if (cqe->op_own & MLX5_INLINE_SCATTER_32) 156321936Shselasky err = mlx5_copy_to_recv_srq(srq, wqe_ctr, cqe, 157321936Shselasky be32toh(cqe->byte_cnt)); 158321936Shselasky else if (cqe->op_own & MLX5_INLINE_SCATTER_64) 159321936Shselasky err = mlx5_copy_to_recv_srq(srq, wqe_ctr, cqe - 1, 160321936Shselasky be32toh(cqe->byte_cnt)); 161321936Shselasky } else { 162321936Shselasky if (likely(cur_rsc->type == MLX5_RSC_TYPE_QP)) { 163321936Shselasky wq = &qp->rq; 164321936Shselasky if (qp->qp_cap_cache & MLX5_RX_CSUM_VALID) 165321936Shselasky cq->flags |= MLX5_CQ_FLAGS_RX_CSUM_VALID; 166321936Shselasky } else { 167321936Shselasky wq = &(rsc_to_mrwq(cur_rsc)->rq); 168321936Shselasky } 169321936Shselasky 170321936Shselasky wqe_ctr = wq->tail & (wq->wqe_cnt - 1); 171321936Shselasky cq->ibv_cq.wr_id = wq->wrid[wqe_ctr]; 172321936Shselasky ++wq->tail; 173321936Shselasky if (cqe->op_own & MLX5_INLINE_SCATTER_32) 174321936Shselasky err = mlx5_copy_to_recv_wqe(qp, wqe_ctr, cqe, 175321936Shselasky be32toh(cqe->byte_cnt)); 176321936Shselasky else if (cqe->op_own & MLX5_INLINE_SCATTER_64) 177321936Shselasky err = mlx5_copy_to_recv_wqe(qp, wqe_ctr, cqe - 1, 178321936Shselasky be32toh(cqe->byte_cnt)); 179321936Shselasky } 180321936Shselasky 181321936Shselasky return err; 182321936Shselasky} 183321936Shselasky 184321936Shselaskystatic inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, 185321936Shselasky struct mlx5_resource *cur_rsc, struct mlx5_srq *srq) 186321936Shselasky{ 187321936Shselasky uint16_t wqe_ctr; 188321936Shselasky struct mlx5_wq *wq; 189321936Shselasky struct mlx5_qp *qp = rsc_to_mqp(cur_rsc); 190321936Shselasky uint8_t g; 191321936Shselasky int err = 0; 192321936Shselasky 193321936Shselasky wc->byte_len = be32toh(cqe->byte_cnt); 194321936Shselasky if (srq) { 195321936Shselasky wqe_ctr = be16toh(cqe->wqe_counter); 196321936Shselasky wc->wr_id = srq->wrid[wqe_ctr]; 197321936Shselasky mlx5_free_srq_wqe(srq, wqe_ctr); 198321936Shselasky if (cqe->op_own & MLX5_INLINE_SCATTER_32) 199321936Shselasky err = mlx5_copy_to_recv_srq(srq, wqe_ctr, cqe, 200321936Shselasky wc->byte_len); 201321936Shselasky else if (cqe->op_own & MLX5_INLINE_SCATTER_64) 202321936Shselasky err = mlx5_copy_to_recv_srq(srq, wqe_ctr, cqe - 1, 203321936Shselasky wc->byte_len); 204321936Shselasky } else { 205321936Shselasky if (likely(cur_rsc->type == MLX5_RSC_TYPE_QP)) { 206321936Shselasky wq = &qp->rq; 207321936Shselasky if (qp->qp_cap_cache & MLX5_RX_CSUM_VALID) 208321936Shselasky wc->wc_flags |= (!!(cqe->hds_ip_ext & MLX5_CQE_L4_OK) & 209321936Shselasky !!(cqe->hds_ip_ext & MLX5_CQE_L3_OK) & 210321936Shselasky (get_cqe_l3_hdr_type(cqe) == 211321936Shselasky MLX5_CQE_L3_HDR_TYPE_IPV4)) << 212321936Shselasky IBV_WC_IP_CSUM_OK_SHIFT; 213321936Shselasky } else { 214321936Shselasky wq = &(rsc_to_mrwq(cur_rsc)->rq); 215321936Shselasky } 216321936Shselasky 217321936Shselasky wqe_ctr = wq->tail & (wq->wqe_cnt - 1); 218321936Shselasky wc->wr_id = wq->wrid[wqe_ctr]; 219321936Shselasky ++wq->tail; 220321936Shselasky if (cqe->op_own & MLX5_INLINE_SCATTER_32) 221321936Shselasky err = mlx5_copy_to_recv_wqe(qp, wqe_ctr, cqe, 222321936Shselasky wc->byte_len); 223321936Shselasky else if (cqe->op_own & MLX5_INLINE_SCATTER_64) 224321936Shselasky err = mlx5_copy_to_recv_wqe(qp, wqe_ctr, cqe - 1, 225321936Shselasky wc->byte_len); 226321936Shselasky } 227321936Shselasky if (err) 228321936Shselasky return err; 229321936Shselasky 230321936Shselasky switch (cqe->op_own >> 4) { 231321936Shselasky case MLX5_CQE_RESP_WR_IMM: 232321936Shselasky wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; 233321936Shselasky wc->wc_flags |= IBV_WC_WITH_IMM; 234321936Shselasky wc->imm_data = cqe->imm_inval_pkey; 235321936Shselasky break; 236321936Shselasky case MLX5_CQE_RESP_SEND: 237321936Shselasky wc->opcode = IBV_WC_RECV; 238321936Shselasky break; 239321936Shselasky case MLX5_CQE_RESP_SEND_IMM: 240321936Shselasky wc->opcode = IBV_WC_RECV; 241321936Shselasky wc->wc_flags |= IBV_WC_WITH_IMM; 242321936Shselasky wc->imm_data = cqe->imm_inval_pkey; 243321936Shselasky break; 244321936Shselasky case MLX5_CQE_RESP_SEND_INV: 245321936Shselasky wc->opcode = IBV_WC_RECV; 246321936Shselasky wc->wc_flags |= IBV_WC_WITH_INV; 247321936Shselasky wc->imm_data = be32toh(cqe->imm_inval_pkey); 248321936Shselasky break; 249321936Shselasky } 250321936Shselasky wc->slid = be16toh(cqe->slid); 251321936Shselasky wc->sl = (be32toh(cqe->flags_rqpn) >> 24) & 0xf; 252321936Shselasky wc->src_qp = be32toh(cqe->flags_rqpn) & 0xffffff; 253321936Shselasky wc->dlid_path_bits = cqe->ml_path & 0x7f; 254321936Shselasky g = (be32toh(cqe->flags_rqpn) >> 28) & 3; 255321936Shselasky wc->wc_flags |= g ? IBV_WC_GRH : 0; 256321936Shselasky wc->pkey_index = be32toh(cqe->imm_inval_pkey) & 0xffff; 257321936Shselasky 258321936Shselasky return IBV_WC_SUCCESS; 259321936Shselasky} 260321936Shselasky 261321936Shselaskystatic void dump_cqe(FILE *fp, void *buf) 262321936Shselasky{ 263321936Shselasky uint32_t *p = buf; 264321936Shselasky int i; 265321936Shselasky 266321936Shselasky for (i = 0; i < 16; i += 4) 267321936Shselasky fprintf(fp, "%08x %08x %08x %08x\n", be32toh(p[i]), be32toh(p[i + 1]), 268321936Shselasky be32toh(p[i + 2]), be32toh(p[i + 3])); 269321936Shselasky} 270321936Shselasky 271321936Shselaskystatic enum ibv_wc_status mlx5_handle_error_cqe(struct mlx5_err_cqe *cqe) 272321936Shselasky{ 273321936Shselasky switch (cqe->syndrome) { 274321936Shselasky case MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR: 275321936Shselasky return IBV_WC_LOC_LEN_ERR; 276321936Shselasky case MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR: 277321936Shselasky return IBV_WC_LOC_QP_OP_ERR; 278321936Shselasky case MLX5_CQE_SYNDROME_LOCAL_PROT_ERR: 279321936Shselasky return IBV_WC_LOC_PROT_ERR; 280321936Shselasky case MLX5_CQE_SYNDROME_WR_FLUSH_ERR: 281321936Shselasky return IBV_WC_WR_FLUSH_ERR; 282321936Shselasky case MLX5_CQE_SYNDROME_MW_BIND_ERR: 283321936Shselasky return IBV_WC_MW_BIND_ERR; 284321936Shselasky case MLX5_CQE_SYNDROME_BAD_RESP_ERR: 285321936Shselasky return IBV_WC_BAD_RESP_ERR; 286321936Shselasky case MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR: 287321936Shselasky return IBV_WC_LOC_ACCESS_ERR; 288321936Shselasky case MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR: 289321936Shselasky return IBV_WC_REM_INV_REQ_ERR; 290321936Shselasky case MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR: 291321936Shselasky return IBV_WC_REM_ACCESS_ERR; 292321936Shselasky case MLX5_CQE_SYNDROME_REMOTE_OP_ERR: 293321936Shselasky return IBV_WC_REM_OP_ERR; 294321936Shselasky case MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR: 295321936Shselasky return IBV_WC_RETRY_EXC_ERR; 296321936Shselasky case MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR: 297321936Shselasky return IBV_WC_RNR_RETRY_EXC_ERR; 298321936Shselasky case MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR: 299321936Shselasky return IBV_WC_REM_ABORT_ERR; 300321936Shselasky default: 301321936Shselasky return IBV_WC_GENERAL_ERR; 302321936Shselasky } 303321936Shselasky} 304321936Shselasky 305321936Shselasky#if defined(__x86_64__) || defined (__i386__) 306321936Shselaskystatic inline unsigned long get_cycles(void) 307321936Shselasky{ 308321936Shselasky uint32_t low, high; 309321936Shselasky uint64_t val; 310321936Shselasky asm volatile ("rdtsc" : "=a" (low), "=d" (high)); 311321936Shselasky val = high; 312321936Shselasky val = (val << 32) | low; 313321936Shselasky return val; 314321936Shselasky} 315321936Shselasky 316321936Shselaskystatic void mlx5_stall_poll_cq(void) 317321936Shselasky{ 318321936Shselasky int i; 319321936Shselasky 320321936Shselasky for (i = 0; i < mlx5_stall_num_loop; i++) 321321936Shselasky (void)get_cycles(); 322321936Shselasky} 323321936Shselaskystatic void mlx5_stall_cycles_poll_cq(uint64_t cycles) 324321936Shselasky{ 325321936Shselasky while (get_cycles() < cycles) 326321936Shselasky ; /* Nothing */ 327321936Shselasky} 328321936Shselaskystatic void mlx5_get_cycles(uint64_t *cycles) 329321936Shselasky{ 330321936Shselasky *cycles = get_cycles(); 331321936Shselasky} 332321936Shselasky#else 333321936Shselaskystatic void mlx5_stall_poll_cq(void) 334321936Shselasky{ 335321936Shselasky} 336321936Shselaskystatic void mlx5_stall_cycles_poll_cq(uint64_t cycles) 337321936Shselasky{ 338321936Shselasky} 339321936Shselaskystatic void mlx5_get_cycles(uint64_t *cycles) 340321936Shselasky{ 341321936Shselasky} 342321936Shselasky#endif 343321936Shselasky 344321936Shselaskystatic inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, 345321936Shselasky struct mlx5_resource **cur_rsc, 346321936Shselasky uint32_t rsn, int cqe_ver) 347321936Shselasky ALWAYS_INLINE; 348321936Shselaskystatic inline struct mlx5_qp *get_req_context(struct mlx5_context *mctx, 349321936Shselasky struct mlx5_resource **cur_rsc, 350321936Shselasky uint32_t rsn, int cqe_ver) 351321936Shselasky{ 352321936Shselasky if (!*cur_rsc || (rsn != (*cur_rsc)->rsn)) 353321936Shselasky *cur_rsc = cqe_ver ? mlx5_find_uidx(mctx, rsn) : 354321936Shselasky (struct mlx5_resource *)mlx5_find_qp(mctx, rsn); 355321936Shselasky 356321936Shselasky return rsc_to_mqp(*cur_rsc); 357321936Shselasky} 358321936Shselasky 359321936Shselaskystatic inline int get_resp_ctx_v1(struct mlx5_context *mctx, 360321936Shselasky struct mlx5_resource **cur_rsc, 361321936Shselasky struct mlx5_srq **cur_srq, 362321936Shselasky uint32_t uidx, uint8_t *is_srq) 363321936Shselasky ALWAYS_INLINE; 364321936Shselaskystatic inline int get_resp_ctx_v1(struct mlx5_context *mctx, 365321936Shselasky struct mlx5_resource **cur_rsc, 366321936Shselasky struct mlx5_srq **cur_srq, 367321936Shselasky uint32_t uidx, uint8_t *is_srq) 368321936Shselasky{ 369321936Shselasky struct mlx5_qp *mqp; 370321936Shselasky 371321936Shselasky if (!*cur_rsc || (uidx != (*cur_rsc)->rsn)) { 372321936Shselasky *cur_rsc = mlx5_find_uidx(mctx, uidx); 373321936Shselasky if (unlikely(!*cur_rsc)) 374321936Shselasky return CQ_POLL_ERR; 375321936Shselasky } 376321936Shselasky 377321936Shselasky switch ((*cur_rsc)->type) { 378321936Shselasky case MLX5_RSC_TYPE_QP: 379321936Shselasky mqp = rsc_to_mqp(*cur_rsc); 380321936Shselasky if (mqp->verbs_qp.qp.srq) { 381321936Shselasky *cur_srq = to_msrq(mqp->verbs_qp.qp.srq); 382321936Shselasky *is_srq = 1; 383321936Shselasky } 384321936Shselasky break; 385321936Shselasky case MLX5_RSC_TYPE_XSRQ: 386321936Shselasky *cur_srq = rsc_to_msrq(*cur_rsc); 387321936Shselasky *is_srq = 1; 388321936Shselasky break; 389321936Shselasky case MLX5_RSC_TYPE_RWQ: 390321936Shselasky break; 391321936Shselasky default: 392321936Shselasky return CQ_POLL_ERR; 393321936Shselasky } 394321936Shselasky 395321936Shselasky return CQ_OK; 396321936Shselasky} 397321936Shselasky 398321936Shselaskystatic inline int get_qp_ctx(struct mlx5_context *mctx, 399321936Shselasky struct mlx5_resource **cur_rsc, 400321936Shselasky uint32_t qpn) 401321936Shselasky ALWAYS_INLINE; 402321936Shselaskystatic inline int get_qp_ctx(struct mlx5_context *mctx, 403321936Shselasky struct mlx5_resource **cur_rsc, 404321936Shselasky uint32_t qpn) 405321936Shselasky{ 406321936Shselasky if (!*cur_rsc || (qpn != (*cur_rsc)->rsn)) { 407321936Shselasky /* 408321936Shselasky * We do not have to take the QP table lock here, 409321936Shselasky * because CQs will be locked while QPs are removed 410321936Shselasky * from the table. 411321936Shselasky */ 412321936Shselasky *cur_rsc = (struct mlx5_resource *)mlx5_find_qp(mctx, qpn); 413321936Shselasky if (unlikely(!*cur_rsc)) 414321936Shselasky return CQ_POLL_ERR; 415321936Shselasky } 416321936Shselasky 417321936Shselasky return CQ_OK; 418321936Shselasky} 419321936Shselasky 420321936Shselaskystatic inline int get_srq_ctx(struct mlx5_context *mctx, 421321936Shselasky struct mlx5_srq **cur_srq, 422321936Shselasky uint32_t srqn_uidx) 423321936Shselasky ALWAYS_INLINE; 424321936Shselaskystatic inline int get_srq_ctx(struct mlx5_context *mctx, 425321936Shselasky struct mlx5_srq **cur_srq, 426321936Shselasky uint32_t srqn) 427321936Shselasky{ 428321936Shselasky if (!*cur_srq || (srqn != (*cur_srq)->srqn)) { 429321936Shselasky *cur_srq = mlx5_find_srq(mctx, srqn); 430321936Shselasky if (unlikely(!*cur_srq)) 431321936Shselasky return CQ_POLL_ERR; 432321936Shselasky } 433321936Shselasky 434321936Shselasky return CQ_OK; 435321936Shselasky} 436321936Shselasky 437321936Shselaskystatic inline int get_cur_rsc(struct mlx5_context *mctx, 438321936Shselasky int cqe_ver, 439321936Shselasky uint32_t qpn, 440321936Shselasky uint32_t srqn_uidx, 441321936Shselasky struct mlx5_resource **cur_rsc, 442321936Shselasky struct mlx5_srq **cur_srq, 443321936Shselasky uint8_t *is_srq) 444321936Shselasky{ 445321936Shselasky int err; 446321936Shselasky 447321936Shselasky if (cqe_ver) { 448321936Shselasky err = get_resp_ctx_v1(mctx, cur_rsc, cur_srq, srqn_uidx, 449321936Shselasky is_srq); 450321936Shselasky } else { 451321936Shselasky if (srqn_uidx) { 452321936Shselasky *is_srq = 1; 453321936Shselasky err = get_srq_ctx(mctx, cur_srq, srqn_uidx); 454321936Shselasky } else { 455321936Shselasky err = get_qp_ctx(mctx, cur_rsc, qpn); 456321936Shselasky } 457321936Shselasky } 458321936Shselasky 459321936Shselasky return err; 460321936Shselasky 461321936Shselasky} 462321936Shselasky 463321936Shselaskystatic inline int mlx5_get_next_cqe(struct mlx5_cq *cq, 464321936Shselasky struct mlx5_cqe64 **pcqe64, 465321936Shselasky void **pcqe) 466321936Shselasky ALWAYS_INLINE; 467321936Shselaskystatic inline int mlx5_get_next_cqe(struct mlx5_cq *cq, 468321936Shselasky struct mlx5_cqe64 **pcqe64, 469321936Shselasky void **pcqe) 470321936Shselasky{ 471321936Shselasky void *cqe; 472321936Shselasky struct mlx5_cqe64 *cqe64; 473321936Shselasky 474321936Shselasky cqe = next_cqe_sw(cq); 475321936Shselasky if (!cqe) 476321936Shselasky return CQ_EMPTY; 477321936Shselasky 478321936Shselasky cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; 479321936Shselasky 480321936Shselasky ++cq->cons_index; 481321936Shselasky 482321936Shselasky VALGRIND_MAKE_MEM_DEFINED(cqe64, sizeof *cqe64); 483321936Shselasky 484321936Shselasky /* 485321936Shselasky * Make sure we read CQ entry contents after we've checked the 486321936Shselasky * ownership bit. 487321936Shselasky */ 488321936Shselasky udma_from_device_barrier(); 489321936Shselasky 490321936Shselasky#ifdef MLX5_DEBUG 491321936Shselasky { 492321936Shselasky struct mlx5_context *mctx = to_mctx(cq->ibv_cq.context); 493321936Shselasky 494321936Shselasky if (mlx5_debug_mask & MLX5_DBG_CQ_CQE) { 495321936Shselasky FILE *fp = mctx->dbg_fp; 496321936Shselasky 497321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ_CQE, "dump cqe for cqn 0x%x:\n", cq->cqn); 498321936Shselasky dump_cqe(fp, cqe64); 499321936Shselasky } 500321936Shselasky } 501321936Shselasky#endif 502321936Shselasky *pcqe64 = cqe64; 503321936Shselasky *pcqe = cqe; 504321936Shselasky 505321936Shselasky return CQ_OK; 506321936Shselasky} 507321936Shselasky 508321936Shselaskystatic inline int mlx5_parse_cqe(struct mlx5_cq *cq, 509321936Shselasky struct mlx5_cqe64 *cqe64, 510321936Shselasky void *cqe, 511321936Shselasky struct mlx5_resource **cur_rsc, 512321936Shselasky struct mlx5_srq **cur_srq, 513321936Shselasky struct ibv_wc *wc, 514321936Shselasky int cqe_ver, int lazy) 515321936Shselasky ALWAYS_INLINE; 516321936Shselaskystatic inline int mlx5_parse_cqe(struct mlx5_cq *cq, 517321936Shselasky struct mlx5_cqe64 *cqe64, 518321936Shselasky void *cqe, 519321936Shselasky struct mlx5_resource **cur_rsc, 520321936Shselasky struct mlx5_srq **cur_srq, 521321936Shselasky struct ibv_wc *wc, 522321936Shselasky int cqe_ver, int lazy) 523321936Shselasky{ 524321936Shselasky struct mlx5_wq *wq; 525321936Shselasky uint16_t wqe_ctr; 526321936Shselasky uint32_t qpn; 527321936Shselasky uint32_t srqn_uidx; 528321936Shselasky int idx; 529321936Shselasky uint8_t opcode; 530321936Shselasky struct mlx5_err_cqe *ecqe; 531321936Shselasky int err = 0; 532321936Shselasky struct mlx5_qp *mqp; 533321936Shselasky struct mlx5_context *mctx; 534321936Shselasky uint8_t is_srq = 0; 535321936Shselasky 536321936Shselasky mctx = to_mctx(ibv_cq_ex_to_cq(&cq->ibv_cq)->context); 537321936Shselasky qpn = be32toh(cqe64->sop_drop_qpn) & 0xffffff; 538321936Shselasky if (lazy) { 539321936Shselasky cq->cqe64 = cqe64; 540321936Shselasky cq->flags &= (~MLX5_CQ_FLAGS_RX_CSUM_VALID); 541321936Shselasky } else { 542321936Shselasky wc->wc_flags = 0; 543321936Shselasky wc->qp_num = qpn; 544321936Shselasky } 545321936Shselasky 546321936Shselasky opcode = mlx5dv_get_cqe_opcode(cqe64); 547321936Shselasky switch (opcode) { 548321936Shselasky case MLX5_CQE_REQ: 549321936Shselasky { 550321936Shselasky mqp = get_req_context(mctx, cur_rsc, 551321936Shselasky (cqe_ver ? (be32toh(cqe64->srqn_uidx) & 0xffffff) : qpn), 552321936Shselasky cqe_ver); 553321936Shselasky if (unlikely(!mqp)) 554321936Shselasky return CQ_POLL_ERR; 555321936Shselasky wq = &mqp->sq; 556321936Shselasky wqe_ctr = be16toh(cqe64->wqe_counter); 557321936Shselasky idx = wqe_ctr & (wq->wqe_cnt - 1); 558321936Shselasky if (lazy) { 559321936Shselasky uint32_t wc_byte_len; 560321936Shselasky 561321936Shselasky switch (be32toh(cqe64->sop_drop_qpn) >> 24) { 562321936Shselasky case MLX5_OPCODE_UMR: 563321936Shselasky cq->umr_opcode = wq->wr_data[idx]; 564321936Shselasky break; 565321936Shselasky 566321936Shselasky case MLX5_OPCODE_RDMA_READ: 567321936Shselasky wc_byte_len = be32toh(cqe64->byte_cnt); 568321936Shselasky goto scatter_out; 569321936Shselasky case MLX5_OPCODE_ATOMIC_CS: 570321936Shselasky case MLX5_OPCODE_ATOMIC_FA: 571321936Shselasky wc_byte_len = 8; 572321936Shselasky 573321936Shselasky scatter_out: 574321936Shselasky if (cqe64->op_own & MLX5_INLINE_SCATTER_32) 575321936Shselasky err = mlx5_copy_to_send_wqe( 576321936Shselasky mqp, wqe_ctr, cqe, wc_byte_len); 577321936Shselasky else if (cqe64->op_own & MLX5_INLINE_SCATTER_64) 578321936Shselasky err = mlx5_copy_to_send_wqe( 579321936Shselasky mqp, wqe_ctr, cqe - 1, wc_byte_len); 580321936Shselasky break; 581321936Shselasky } 582321936Shselasky 583321936Shselasky cq->ibv_cq.wr_id = wq->wrid[idx]; 584321936Shselasky cq->ibv_cq.status = err; 585321936Shselasky } else { 586321936Shselasky handle_good_req(wc, cqe64, wq, idx); 587321936Shselasky 588321936Shselasky if (cqe64->op_own & MLX5_INLINE_SCATTER_32) 589321936Shselasky err = mlx5_copy_to_send_wqe(mqp, wqe_ctr, cqe, 590321936Shselasky wc->byte_len); 591321936Shselasky else if (cqe64->op_own & MLX5_INLINE_SCATTER_64) 592321936Shselasky err = mlx5_copy_to_send_wqe( 593321936Shselasky mqp, wqe_ctr, cqe - 1, wc->byte_len); 594321936Shselasky 595321936Shselasky wc->wr_id = wq->wrid[idx]; 596321936Shselasky wc->status = err; 597321936Shselasky } 598321936Shselasky 599321936Shselasky wq->tail = wq->wqe_head[idx] + 1; 600321936Shselasky break; 601321936Shselasky } 602321936Shselasky case MLX5_CQE_RESP_WR_IMM: 603321936Shselasky case MLX5_CQE_RESP_SEND: 604321936Shselasky case MLX5_CQE_RESP_SEND_IMM: 605321936Shselasky case MLX5_CQE_RESP_SEND_INV: 606321936Shselasky srqn_uidx = be32toh(cqe64->srqn_uidx) & 0xffffff; 607321936Shselasky err = get_cur_rsc(mctx, cqe_ver, qpn, srqn_uidx, cur_rsc, 608321936Shselasky cur_srq, &is_srq); 609321936Shselasky if (unlikely(err)) 610321936Shselasky return CQ_POLL_ERR; 611321936Shselasky 612321936Shselasky if (lazy) 613321936Shselasky cq->ibv_cq.status = handle_responder_lazy(cq, cqe64, 614321936Shselasky *cur_rsc, 615321936Shselasky is_srq ? *cur_srq : NULL); 616321936Shselasky else 617321936Shselasky wc->status = handle_responder(wc, cqe64, *cur_rsc, 618321936Shselasky is_srq ? *cur_srq : NULL); 619321936Shselasky break; 620321936Shselasky case MLX5_CQE_RESIZE_CQ: 621321936Shselasky break; 622321936Shselasky case MLX5_CQE_REQ_ERR: 623321936Shselasky case MLX5_CQE_RESP_ERR: 624321936Shselasky srqn_uidx = be32toh(cqe64->srqn_uidx) & 0xffffff; 625321936Shselasky ecqe = (struct mlx5_err_cqe *)cqe64; 626321936Shselasky { 627321936Shselasky enum ibv_wc_status *pstatus = lazy ? &cq->ibv_cq.status : &wc->status; 628321936Shselasky 629321936Shselasky *pstatus = mlx5_handle_error_cqe(ecqe); 630321936Shselasky } 631321936Shselasky 632321936Shselasky if (!lazy) 633321936Shselasky wc->vendor_err = ecqe->vendor_err_synd; 634321936Shselasky 635321936Shselasky if (unlikely(ecqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR && 636321936Shselasky ecqe->syndrome != MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR)) { 637321936Shselasky FILE *fp = mctx->dbg_fp; 638321936Shselasky fprintf(fp, PFX "%s: got completion with error:\n", 639321936Shselasky mctx->hostname); 640321936Shselasky dump_cqe(fp, ecqe); 641321936Shselasky if (mlx5_freeze_on_error_cqe) { 642321936Shselasky fprintf(fp, PFX "freezing at poll cq..."); 643321936Shselasky while (1) 644321936Shselasky sleep(10); 645321936Shselasky } 646321936Shselasky } 647321936Shselasky 648321936Shselasky if (opcode == MLX5_CQE_REQ_ERR) { 649321936Shselasky mqp = get_req_context(mctx, cur_rsc, 650321936Shselasky (cqe_ver ? srqn_uidx : qpn), cqe_ver); 651321936Shselasky if (unlikely(!mqp)) 652321936Shselasky return CQ_POLL_ERR; 653321936Shselasky wq = &mqp->sq; 654321936Shselasky wqe_ctr = be16toh(cqe64->wqe_counter); 655321936Shselasky idx = wqe_ctr & (wq->wqe_cnt - 1); 656321936Shselasky if (lazy) 657321936Shselasky cq->ibv_cq.wr_id = wq->wrid[idx]; 658321936Shselasky else 659321936Shselasky wc->wr_id = wq->wrid[idx]; 660321936Shselasky wq->tail = wq->wqe_head[idx] + 1; 661321936Shselasky } else { 662321936Shselasky err = get_cur_rsc(mctx, cqe_ver, qpn, srqn_uidx, 663321936Shselasky cur_rsc, cur_srq, &is_srq); 664321936Shselasky if (unlikely(err)) 665321936Shselasky return CQ_POLL_ERR; 666321936Shselasky 667321936Shselasky if (is_srq) { 668321936Shselasky wqe_ctr = be16toh(cqe64->wqe_counter); 669321936Shselasky if (lazy) 670321936Shselasky cq->ibv_cq.wr_id = (*cur_srq)->wrid[wqe_ctr]; 671321936Shselasky else 672321936Shselasky wc->wr_id = (*cur_srq)->wrid[wqe_ctr]; 673321936Shselasky mlx5_free_srq_wqe(*cur_srq, wqe_ctr); 674321936Shselasky } else { 675321936Shselasky switch ((*cur_rsc)->type) { 676321936Shselasky case MLX5_RSC_TYPE_RWQ: 677321936Shselasky wq = &(rsc_to_mrwq(*cur_rsc)->rq); 678321936Shselasky break; 679321936Shselasky default: 680321936Shselasky wq = &(rsc_to_mqp(*cur_rsc)->rq); 681321936Shselasky break; 682321936Shselasky } 683321936Shselasky 684321936Shselasky if (lazy) 685321936Shselasky cq->ibv_cq.wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 686321936Shselasky else 687321936Shselasky wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 688321936Shselasky ++wq->tail; 689321936Shselasky } 690321936Shselasky } 691321936Shselasky break; 692321936Shselasky } 693321936Shselasky 694321936Shselasky return CQ_OK; 695321936Shselasky} 696321936Shselasky 697321936Shselaskystatic inline int mlx5_parse_lazy_cqe(struct mlx5_cq *cq, 698321936Shselasky struct mlx5_cqe64 *cqe64, 699321936Shselasky void *cqe, int cqe_ver) 700321936Shselasky ALWAYS_INLINE; 701321936Shselaskystatic inline int mlx5_parse_lazy_cqe(struct mlx5_cq *cq, 702321936Shselasky struct mlx5_cqe64 *cqe64, 703321936Shselasky void *cqe, int cqe_ver) 704321936Shselasky{ 705321936Shselasky return mlx5_parse_cqe(cq, cqe64, cqe, &cq->cur_rsc, &cq->cur_srq, NULL, cqe_ver, 1); 706321936Shselasky} 707321936Shselasky 708321936Shselaskystatic inline int mlx5_poll_one(struct mlx5_cq *cq, 709321936Shselasky struct mlx5_resource **cur_rsc, 710321936Shselasky struct mlx5_srq **cur_srq, 711321936Shselasky struct ibv_wc *wc, int cqe_ver) 712321936Shselasky ALWAYS_INLINE; 713321936Shselaskystatic inline int mlx5_poll_one(struct mlx5_cq *cq, 714321936Shselasky struct mlx5_resource **cur_rsc, 715321936Shselasky struct mlx5_srq **cur_srq, 716321936Shselasky struct ibv_wc *wc, int cqe_ver) 717321936Shselasky{ 718321936Shselasky struct mlx5_cqe64 *cqe64; 719321936Shselasky void *cqe; 720321936Shselasky int err; 721321936Shselasky 722321936Shselasky err = mlx5_get_next_cqe(cq, &cqe64, &cqe); 723321936Shselasky if (err == CQ_EMPTY) 724321936Shselasky return err; 725321936Shselasky 726321936Shselasky return mlx5_parse_cqe(cq, cqe64, cqe, cur_rsc, cur_srq, wc, cqe_ver, 0); 727321936Shselasky} 728321936Shselasky 729321936Shselaskystatic inline int poll_cq(struct ibv_cq *ibcq, int ne, 730321936Shselasky struct ibv_wc *wc, int cqe_ver) 731321936Shselasky ALWAYS_INLINE; 732321936Shselaskystatic inline int poll_cq(struct ibv_cq *ibcq, int ne, 733321936Shselasky struct ibv_wc *wc, int cqe_ver) 734321936Shselasky{ 735321936Shselasky struct mlx5_cq *cq = to_mcq(ibcq); 736321936Shselasky struct mlx5_resource *rsc = NULL; 737321936Shselasky struct mlx5_srq *srq = NULL; 738321936Shselasky int npolled; 739321936Shselasky int err = CQ_OK; 740321936Shselasky 741321936Shselasky if (cq->stall_enable) { 742321936Shselasky if (cq->stall_adaptive_enable) { 743321936Shselasky if (cq->stall_last_count) 744321936Shselasky mlx5_stall_cycles_poll_cq(cq->stall_last_count + cq->stall_cycles); 745321936Shselasky } else if (cq->stall_next_poll) { 746321936Shselasky cq->stall_next_poll = 0; 747321936Shselasky mlx5_stall_poll_cq(); 748321936Shselasky } 749321936Shselasky } 750321936Shselasky 751321936Shselasky mlx5_spin_lock(&cq->lock); 752321936Shselasky 753321936Shselasky for (npolled = 0; npolled < ne; ++npolled) { 754321936Shselasky err = mlx5_poll_one(cq, &rsc, &srq, wc + npolled, cqe_ver); 755321936Shselasky if (err != CQ_OK) 756321936Shselasky break; 757321936Shselasky } 758321936Shselasky 759321936Shselasky update_cons_index(cq); 760321936Shselasky 761321936Shselasky mlx5_spin_unlock(&cq->lock); 762321936Shselasky 763321936Shselasky if (cq->stall_enable) { 764321936Shselasky if (cq->stall_adaptive_enable) { 765321936Shselasky if (npolled == 0) { 766321936Shselasky cq->stall_cycles = max(cq->stall_cycles-mlx5_stall_cq_dec_step, 767321936Shselasky mlx5_stall_cq_poll_min); 768321936Shselasky mlx5_get_cycles(&cq->stall_last_count); 769321936Shselasky } else if (npolled < ne) { 770321936Shselasky cq->stall_cycles = min(cq->stall_cycles+mlx5_stall_cq_inc_step, 771321936Shselasky mlx5_stall_cq_poll_max); 772321936Shselasky mlx5_get_cycles(&cq->stall_last_count); 773321936Shselasky } else { 774321936Shselasky cq->stall_cycles = max(cq->stall_cycles-mlx5_stall_cq_dec_step, 775321936Shselasky mlx5_stall_cq_poll_min); 776321936Shselasky cq->stall_last_count = 0; 777321936Shselasky } 778321936Shselasky } else if (err == CQ_EMPTY) { 779321936Shselasky cq->stall_next_poll = 1; 780321936Shselasky } 781321936Shselasky } 782321936Shselasky 783321936Shselasky return err == CQ_POLL_ERR ? err : npolled; 784321936Shselasky} 785321936Shselasky 786321936Shselaskyenum polling_mode { 787321936Shselasky POLLING_MODE_NO_STALL, 788321936Shselasky POLLING_MODE_STALL, 789321936Shselasky POLLING_MODE_STALL_ADAPTIVE 790321936Shselasky}; 791321936Shselasky 792321936Shselaskystatic inline void _mlx5_end_poll(struct ibv_cq_ex *ibcq, 793321936Shselasky int lock, enum polling_mode stall) 794321936Shselasky ALWAYS_INLINE; 795321936Shselaskystatic inline void _mlx5_end_poll(struct ibv_cq_ex *ibcq, 796321936Shselasky int lock, enum polling_mode stall) 797321936Shselasky{ 798321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 799321936Shselasky 800321936Shselasky update_cons_index(cq); 801321936Shselasky 802321936Shselasky if (lock) 803321936Shselasky mlx5_spin_unlock(&cq->lock); 804321936Shselasky 805321936Shselasky if (stall) { 806321936Shselasky if (stall == POLLING_MODE_STALL_ADAPTIVE) { 807321936Shselasky if (!(cq->flags & MLX5_CQ_FLAGS_FOUND_CQES)) { 808321936Shselasky cq->stall_cycles = max(cq->stall_cycles - mlx5_stall_cq_dec_step, 809321936Shselasky mlx5_stall_cq_poll_min); 810321936Shselasky mlx5_get_cycles(&cq->stall_last_count); 811321936Shselasky } else if (cq->flags & MLX5_CQ_FLAGS_EMPTY_DURING_POLL) { 812321936Shselasky cq->stall_cycles = min(cq->stall_cycles + mlx5_stall_cq_inc_step, 813321936Shselasky mlx5_stall_cq_poll_max); 814321936Shselasky mlx5_get_cycles(&cq->stall_last_count); 815321936Shselasky } else { 816321936Shselasky cq->stall_cycles = max(cq->stall_cycles - mlx5_stall_cq_dec_step, 817321936Shselasky mlx5_stall_cq_poll_min); 818321936Shselasky cq->stall_last_count = 0; 819321936Shselasky } 820321936Shselasky } else if (!(cq->flags & MLX5_CQ_FLAGS_FOUND_CQES)) { 821321936Shselasky cq->stall_next_poll = 1; 822321936Shselasky } 823321936Shselasky 824321936Shselasky cq->flags &= ~(MLX5_CQ_FLAGS_FOUND_CQES | MLX5_CQ_FLAGS_EMPTY_DURING_POLL); 825321936Shselasky } 826321936Shselasky} 827321936Shselasky 828321936Shselaskystatic inline int mlx5_start_poll(struct ibv_cq_ex *ibcq, struct ibv_poll_cq_attr *attr, 829321936Shselasky int lock, enum polling_mode stall, int cqe_version) 830321936Shselasky ALWAYS_INLINE; 831321936Shselaskystatic inline int mlx5_start_poll(struct ibv_cq_ex *ibcq, struct ibv_poll_cq_attr *attr, 832321936Shselasky int lock, enum polling_mode stall, int cqe_version) 833321936Shselasky{ 834321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 835321936Shselasky struct mlx5_cqe64 *cqe64; 836321936Shselasky void *cqe; 837321936Shselasky int err; 838321936Shselasky 839321936Shselasky if (unlikely(attr->comp_mask)) 840321936Shselasky return EINVAL; 841321936Shselasky 842321936Shselasky if (stall) { 843321936Shselasky if (stall == POLLING_MODE_STALL_ADAPTIVE) { 844321936Shselasky if (cq->stall_last_count) 845321936Shselasky mlx5_stall_cycles_poll_cq(cq->stall_last_count + cq->stall_cycles); 846321936Shselasky } else if (cq->stall_next_poll) { 847321936Shselasky cq->stall_next_poll = 0; 848321936Shselasky mlx5_stall_poll_cq(); 849321936Shselasky } 850321936Shselasky } 851321936Shselasky 852321936Shselasky if (lock) 853321936Shselasky mlx5_spin_lock(&cq->lock); 854321936Shselasky 855321936Shselasky cq->cur_rsc = NULL; 856321936Shselasky cq->cur_srq = NULL; 857321936Shselasky 858321936Shselasky err = mlx5_get_next_cqe(cq, &cqe64, &cqe); 859321936Shselasky if (err == CQ_EMPTY) { 860321936Shselasky if (lock) 861321936Shselasky mlx5_spin_unlock(&cq->lock); 862321936Shselasky 863321936Shselasky if (stall) { 864321936Shselasky if (stall == POLLING_MODE_STALL_ADAPTIVE) { 865321936Shselasky cq->stall_cycles = max(cq->stall_cycles - mlx5_stall_cq_dec_step, 866321936Shselasky mlx5_stall_cq_poll_min); 867321936Shselasky mlx5_get_cycles(&cq->stall_last_count); 868321936Shselasky } else { 869321936Shselasky cq->stall_next_poll = 1; 870321936Shselasky } 871321936Shselasky } 872321936Shselasky 873321936Shselasky return ENOENT; 874321936Shselasky } 875321936Shselasky 876321936Shselasky if (stall) 877321936Shselasky cq->flags |= MLX5_CQ_FLAGS_FOUND_CQES; 878321936Shselasky 879321936Shselasky err = mlx5_parse_lazy_cqe(cq, cqe64, cqe, cqe_version); 880321936Shselasky if (lock && err) 881321936Shselasky mlx5_spin_unlock(&cq->lock); 882321936Shselasky 883321936Shselasky if (stall && err) { 884321936Shselasky if (stall == POLLING_MODE_STALL_ADAPTIVE) { 885321936Shselasky cq->stall_cycles = max(cq->stall_cycles - mlx5_stall_cq_dec_step, 886321936Shselasky mlx5_stall_cq_poll_min); 887321936Shselasky cq->stall_last_count = 0; 888321936Shselasky } 889321936Shselasky 890321936Shselasky cq->flags &= ~(MLX5_CQ_FLAGS_FOUND_CQES); 891321936Shselasky } 892321936Shselasky 893321936Shselasky return err; 894321936Shselasky} 895321936Shselasky 896321936Shselaskystatic inline int mlx5_next_poll(struct ibv_cq_ex *ibcq, 897321936Shselasky enum polling_mode stall, int cqe_version) 898321936Shselasky ALWAYS_INLINE; 899321936Shselaskystatic inline int mlx5_next_poll(struct ibv_cq_ex *ibcq, 900321936Shselasky enum polling_mode stall, 901321936Shselasky int cqe_version) 902321936Shselasky{ 903321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 904321936Shselasky struct mlx5_cqe64 *cqe64; 905321936Shselasky void *cqe; 906321936Shselasky int err; 907321936Shselasky 908321936Shselasky err = mlx5_get_next_cqe(cq, &cqe64, &cqe); 909321936Shselasky if (err == CQ_EMPTY) { 910321936Shselasky if (stall == POLLING_MODE_STALL_ADAPTIVE) 911321936Shselasky cq->flags |= MLX5_CQ_FLAGS_EMPTY_DURING_POLL; 912321936Shselasky 913321936Shselasky return ENOENT; 914321936Shselasky } 915321936Shselasky 916321936Shselasky return mlx5_parse_lazy_cqe(cq, cqe64, cqe, cqe_version); 917321936Shselasky} 918321936Shselasky 919321936Shselaskystatic inline int mlx5_next_poll_adaptive_v0(struct ibv_cq_ex *ibcq) 920321936Shselasky{ 921321936Shselasky return mlx5_next_poll(ibcq, POLLING_MODE_STALL_ADAPTIVE, 0); 922321936Shselasky} 923321936Shselasky 924321936Shselaskystatic inline int mlx5_next_poll_adaptive_v1(struct ibv_cq_ex *ibcq) 925321936Shselasky{ 926321936Shselasky return mlx5_next_poll(ibcq, POLLING_MODE_STALL_ADAPTIVE, 1); 927321936Shselasky} 928321936Shselasky 929321936Shselaskystatic inline int mlx5_next_poll_v0(struct ibv_cq_ex *ibcq) 930321936Shselasky{ 931321936Shselasky return mlx5_next_poll(ibcq, 0, 0); 932321936Shselasky} 933321936Shselasky 934321936Shselaskystatic inline int mlx5_next_poll_v1(struct ibv_cq_ex *ibcq) 935321936Shselasky{ 936321936Shselasky return mlx5_next_poll(ibcq, 0, 1); 937321936Shselasky} 938321936Shselasky 939321936Shselaskystatic inline int mlx5_start_poll_v0(struct ibv_cq_ex *ibcq, 940321936Shselasky struct ibv_poll_cq_attr *attr) 941321936Shselasky{ 942321936Shselasky return mlx5_start_poll(ibcq, attr, 0, 0, 0); 943321936Shselasky} 944321936Shselasky 945321936Shselaskystatic inline int mlx5_start_poll_v1(struct ibv_cq_ex *ibcq, 946321936Shselasky struct ibv_poll_cq_attr *attr) 947321936Shselasky{ 948321936Shselasky return mlx5_start_poll(ibcq, attr, 0, 0, 1); 949321936Shselasky} 950321936Shselasky 951321936Shselaskystatic inline int mlx5_start_poll_v0_lock(struct ibv_cq_ex *ibcq, 952321936Shselasky struct ibv_poll_cq_attr *attr) 953321936Shselasky{ 954321936Shselasky return mlx5_start_poll(ibcq, attr, 1, 0, 0); 955321936Shselasky} 956321936Shselasky 957321936Shselaskystatic inline int mlx5_start_poll_v1_lock(struct ibv_cq_ex *ibcq, 958321936Shselasky struct ibv_poll_cq_attr *attr) 959321936Shselasky{ 960321936Shselasky return mlx5_start_poll(ibcq, attr, 1, 0, 1); 961321936Shselasky} 962321936Shselasky 963321936Shselaskystatic inline int mlx5_start_poll_adaptive_stall_v0_lock(struct ibv_cq_ex *ibcq, 964321936Shselasky struct ibv_poll_cq_attr *attr) 965321936Shselasky{ 966321936Shselasky return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL_ADAPTIVE, 0); 967321936Shselasky} 968321936Shselasky 969321936Shselaskystatic inline int mlx5_start_poll_stall_v0_lock(struct ibv_cq_ex *ibcq, 970321936Shselasky struct ibv_poll_cq_attr *attr) 971321936Shselasky{ 972321936Shselasky return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL, 0); 973321936Shselasky} 974321936Shselasky 975321936Shselaskystatic inline int mlx5_start_poll_adaptive_stall_v1_lock(struct ibv_cq_ex *ibcq, 976321936Shselasky struct ibv_poll_cq_attr *attr) 977321936Shselasky{ 978321936Shselasky return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL_ADAPTIVE, 1); 979321936Shselasky} 980321936Shselasky 981321936Shselaskystatic inline int mlx5_start_poll_stall_v1_lock(struct ibv_cq_ex *ibcq, 982321936Shselasky struct ibv_poll_cq_attr *attr) 983321936Shselasky{ 984321936Shselasky return mlx5_start_poll(ibcq, attr, 1, POLLING_MODE_STALL, 1); 985321936Shselasky} 986321936Shselasky 987321936Shselaskystatic inline int mlx5_start_poll_stall_v0(struct ibv_cq_ex *ibcq, 988321936Shselasky struct ibv_poll_cq_attr *attr) 989321936Shselasky{ 990321936Shselasky return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL, 0); 991321936Shselasky} 992321936Shselasky 993321936Shselaskystatic inline int mlx5_start_poll_adaptive_stall_v0(struct ibv_cq_ex *ibcq, 994321936Shselasky struct ibv_poll_cq_attr *attr) 995321936Shselasky{ 996321936Shselasky return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL_ADAPTIVE, 0); 997321936Shselasky} 998321936Shselasky 999321936Shselaskystatic inline int mlx5_start_poll_adaptive_stall_v1(struct ibv_cq_ex *ibcq, 1000321936Shselasky struct ibv_poll_cq_attr *attr) 1001321936Shselasky{ 1002321936Shselasky return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL_ADAPTIVE, 1); 1003321936Shselasky} 1004321936Shselasky 1005321936Shselaskystatic inline int mlx5_start_poll_stall_v1(struct ibv_cq_ex *ibcq, 1006321936Shselasky struct ibv_poll_cq_attr *attr) 1007321936Shselasky{ 1008321936Shselasky return mlx5_start_poll(ibcq, attr, 0, POLLING_MODE_STALL, 1); 1009321936Shselasky} 1010321936Shselasky 1011321936Shselaskystatic inline void mlx5_end_poll_adaptive_stall_lock(struct ibv_cq_ex *ibcq) 1012321936Shselasky{ 1013321936Shselasky _mlx5_end_poll(ibcq, 1, POLLING_MODE_STALL_ADAPTIVE); 1014321936Shselasky} 1015321936Shselasky 1016321936Shselaskystatic inline void mlx5_end_poll_stall_lock(struct ibv_cq_ex *ibcq) 1017321936Shselasky{ 1018321936Shselasky _mlx5_end_poll(ibcq, 1, POLLING_MODE_STALL); 1019321936Shselasky} 1020321936Shselasky 1021321936Shselaskystatic inline void mlx5_end_poll_adaptive_stall(struct ibv_cq_ex *ibcq) 1022321936Shselasky{ 1023321936Shselasky _mlx5_end_poll(ibcq, 0, POLLING_MODE_STALL_ADAPTIVE); 1024321936Shselasky} 1025321936Shselasky 1026321936Shselaskystatic inline void mlx5_end_poll_stall(struct ibv_cq_ex *ibcq) 1027321936Shselasky{ 1028321936Shselasky _mlx5_end_poll(ibcq, 0, POLLING_MODE_STALL); 1029321936Shselasky} 1030321936Shselasky 1031321936Shselaskystatic inline void mlx5_end_poll(struct ibv_cq_ex *ibcq) 1032321936Shselasky{ 1033321936Shselasky _mlx5_end_poll(ibcq, 0, 0); 1034321936Shselasky} 1035321936Shselasky 1036321936Shselaskystatic inline void mlx5_end_poll_lock(struct ibv_cq_ex *ibcq) 1037321936Shselasky{ 1038321936Shselasky _mlx5_end_poll(ibcq, 1, 0); 1039321936Shselasky} 1040321936Shselasky 1041321936Shselaskyint mlx5_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 1042321936Shselasky{ 1043321936Shselasky return poll_cq(ibcq, ne, wc, 0); 1044321936Shselasky} 1045321936Shselasky 1046321936Shselaskyint mlx5_poll_cq_v1(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 1047321936Shselasky{ 1048321936Shselasky return poll_cq(ibcq, ne, wc, 1); 1049321936Shselasky} 1050321936Shselasky 1051321936Shselaskystatic inline enum ibv_wc_opcode mlx5_cq_read_wc_opcode(struct ibv_cq_ex *ibcq) 1052321936Shselasky{ 1053321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1054321936Shselasky 1055321936Shselasky switch (mlx5dv_get_cqe_opcode(cq->cqe64)) { 1056321936Shselasky case MLX5_CQE_RESP_WR_IMM: 1057321936Shselasky return IBV_WC_RECV_RDMA_WITH_IMM; 1058321936Shselasky case MLX5_CQE_RESP_SEND: 1059321936Shselasky case MLX5_CQE_RESP_SEND_IMM: 1060321936Shselasky case MLX5_CQE_RESP_SEND_INV: 1061321936Shselasky return IBV_WC_RECV; 1062321936Shselasky case MLX5_CQE_REQ: 1063321936Shselasky switch (be32toh(cq->cqe64->sop_drop_qpn) >> 24) { 1064321936Shselasky case MLX5_OPCODE_RDMA_WRITE_IMM: 1065321936Shselasky case MLX5_OPCODE_RDMA_WRITE: 1066321936Shselasky return IBV_WC_RDMA_WRITE; 1067321936Shselasky case MLX5_OPCODE_SEND_IMM: 1068321936Shselasky case MLX5_OPCODE_SEND: 1069321936Shselasky case MLX5_OPCODE_SEND_INVAL: 1070321936Shselasky return IBV_WC_SEND; 1071321936Shselasky case MLX5_OPCODE_RDMA_READ: 1072321936Shselasky return IBV_WC_RDMA_READ; 1073321936Shselasky case MLX5_OPCODE_ATOMIC_CS: 1074321936Shselasky return IBV_WC_COMP_SWAP; 1075321936Shselasky case MLX5_OPCODE_ATOMIC_FA: 1076321936Shselasky return IBV_WC_FETCH_ADD; 1077321936Shselasky case MLX5_OPCODE_UMR: 1078321936Shselasky return cq->umr_opcode; 1079321936Shselasky case MLX5_OPCODE_TSO: 1080321936Shselasky return IBV_WC_TSO; 1081321936Shselasky } 1082321936Shselasky } 1083321936Shselasky 1084321936Shselasky#ifdef MLX5_DEBUG 1085321936Shselasky{ 1086321936Shselasky struct mlx5_context *ctx = to_mctx(ibcq->context); 1087321936Shselasky 1088321936Shselasky mlx5_dbg(ctx->dbg_fp, MLX5_DBG_CQ_CQE, "un-expected opcode in cqe\n"); 1089321936Shselasky} 1090321936Shselasky#endif 1091321936Shselasky return 0; 1092321936Shselasky} 1093321936Shselasky 1094321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_qp_num(struct ibv_cq_ex *ibcq) 1095321936Shselasky{ 1096321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1097321936Shselasky 1098321936Shselasky return be32toh(cq->cqe64->sop_drop_qpn) & 0xffffff; 1099321936Shselasky} 1100321936Shselasky 1101321936Shselaskystatic inline int mlx5_cq_read_wc_flags(struct ibv_cq_ex *ibcq) 1102321936Shselasky{ 1103321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1104321936Shselasky int wc_flags = 0; 1105321936Shselasky 1106321936Shselasky if (cq->flags & MLX5_CQ_FLAGS_RX_CSUM_VALID) 1107321936Shselasky wc_flags = (!!(cq->cqe64->hds_ip_ext & MLX5_CQE_L4_OK) & 1108321936Shselasky !!(cq->cqe64->hds_ip_ext & MLX5_CQE_L3_OK) & 1109321936Shselasky (get_cqe_l3_hdr_type(cq->cqe64) == 1110321936Shselasky MLX5_CQE_L3_HDR_TYPE_IPV4)) << 1111321936Shselasky IBV_WC_IP_CSUM_OK_SHIFT; 1112321936Shselasky 1113321936Shselasky switch (mlx5dv_get_cqe_opcode(cq->cqe64)) { 1114321936Shselasky case MLX5_CQE_RESP_WR_IMM: 1115321936Shselasky case MLX5_CQE_RESP_SEND_IMM: 1116321936Shselasky wc_flags |= IBV_WC_WITH_IMM; 1117321936Shselasky break; 1118321936Shselasky case MLX5_CQE_RESP_SEND_INV: 1119321936Shselasky wc_flags |= IBV_WC_WITH_INV; 1120321936Shselasky break; 1121321936Shselasky } 1122321936Shselasky 1123321936Shselasky wc_flags |= ((be32toh(cq->cqe64->flags_rqpn) >> 28) & 3) ? IBV_WC_GRH : 0; 1124321936Shselasky return wc_flags; 1125321936Shselasky} 1126321936Shselasky 1127321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_byte_len(struct ibv_cq_ex *ibcq) 1128321936Shselasky{ 1129321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1130321936Shselasky 1131321936Shselasky return be32toh(cq->cqe64->byte_cnt); 1132321936Shselasky} 1133321936Shselasky 1134321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_vendor_err(struct ibv_cq_ex *ibcq) 1135321936Shselasky{ 1136321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1137321936Shselasky struct mlx5_err_cqe *ecqe = (struct mlx5_err_cqe *)cq->cqe64; 1138321936Shselasky 1139321936Shselasky return ecqe->vendor_err_synd; 1140321936Shselasky} 1141321936Shselasky 1142321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq) 1143321936Shselasky{ 1144321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1145321936Shselasky 1146321936Shselasky switch (mlx5dv_get_cqe_opcode(cq->cqe64)) { 1147321936Shselasky case MLX5_CQE_RESP_SEND_INV: 1148321936Shselasky return be32toh(cq->cqe64->imm_inval_pkey); 1149321936Shselasky default: 1150321936Shselasky return cq->cqe64->imm_inval_pkey; 1151321936Shselasky } 1152321936Shselasky} 1153321936Shselasky 1154321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_slid(struct ibv_cq_ex *ibcq) 1155321936Shselasky{ 1156321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1157321936Shselasky 1158321936Shselasky return (uint32_t)be16toh(cq->cqe64->slid); 1159321936Shselasky} 1160321936Shselasky 1161321936Shselaskystatic inline uint8_t mlx5_cq_read_wc_sl(struct ibv_cq_ex *ibcq) 1162321936Shselasky{ 1163321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1164321936Shselasky 1165321936Shselasky return (be32toh(cq->cqe64->flags_rqpn) >> 24) & 0xf; 1166321936Shselasky} 1167321936Shselasky 1168321936Shselaskystatic inline uint32_t mlx5_cq_read_wc_src_qp(struct ibv_cq_ex *ibcq) 1169321936Shselasky{ 1170321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1171321936Shselasky 1172321936Shselasky return be32toh(cq->cqe64->flags_rqpn) & 0xffffff; 1173321936Shselasky} 1174321936Shselasky 1175321936Shselaskystatic inline uint8_t mlx5_cq_read_wc_dlid_path_bits(struct ibv_cq_ex *ibcq) 1176321936Shselasky{ 1177321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1178321936Shselasky 1179321936Shselasky return cq->cqe64->ml_path & 0x7f; 1180321936Shselasky} 1181321936Shselasky 1182321936Shselaskystatic inline uint64_t mlx5_cq_read_wc_completion_ts(struct ibv_cq_ex *ibcq) 1183321936Shselasky{ 1184321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1185321936Shselasky 1186321936Shselasky return be64toh(cq->cqe64->timestamp); 1187321936Shselasky} 1188321936Shselasky 1189321936Shselaskystatic inline uint16_t mlx5_cq_read_wc_cvlan(struct ibv_cq_ex *ibcq) 1190321936Shselasky{ 1191321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1192321936Shselasky 1193321936Shselasky return be16toh(cq->cqe64->vlan_info); 1194321936Shselasky} 1195321936Shselasky 1196321936Shselaskystatic inline uint32_t mlx5_cq_read_flow_tag(struct ibv_cq_ex *ibcq) 1197321936Shselasky{ 1198321936Shselasky struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); 1199321936Shselasky 1200321936Shselasky return be32toh(cq->cqe64->sop_drop_qpn) & MLX5_FLOW_TAG_MASK; 1201321936Shselasky} 1202321936Shselasky 1203321936Shselasky#define BIT(i) (1UL << (i)) 1204321936Shselasky 1205321936Shselasky#define SINGLE_THREADED BIT(0) 1206321936Shselasky#define STALL BIT(1) 1207321936Shselasky#define V1 BIT(2) 1208321936Shselasky#define ADAPTIVE BIT(3) 1209321936Shselasky 1210321936Shselasky#define mlx5_start_poll_name(cqe_ver, lock, stall, adaptive) \ 1211321936Shselasky mlx5_start_poll##adaptive##stall##cqe_ver##lock 1212321936Shselasky#define mlx5_next_poll_name(cqe_ver, adaptive) \ 1213321936Shselasky mlx5_next_poll##adaptive##cqe_ver 1214321936Shselasky#define mlx5_end_poll_name(lock, stall, adaptive) \ 1215321936Shselasky mlx5_end_poll##adaptive##stall##lock 1216321936Shselasky 1217321936Shselasky#define POLL_FN_ENTRY(cqe_ver, lock, stall, adaptive) { \ 1218321936Shselasky .start_poll = &mlx5_start_poll_name(cqe_ver, lock, stall, adaptive), \ 1219321936Shselasky .next_poll = &mlx5_next_poll_name(cqe_ver, adaptive), \ 1220321936Shselasky .end_poll = &mlx5_end_poll_name(lock, stall, adaptive), \ 1221321936Shselasky } 1222321936Shselasky 1223321936Shselaskystatic const struct op 1224321936Shselasky{ 1225321936Shselasky int (*start_poll)(struct ibv_cq_ex *ibcq, struct ibv_poll_cq_attr *attr); 1226321936Shselasky int (*next_poll)(struct ibv_cq_ex *ibcq); 1227321936Shselasky void (*end_poll)(struct ibv_cq_ex *ibcq); 1228321936Shselasky} ops[ADAPTIVE + V1 + STALL + SINGLE_THREADED + 1] = { 1229321936Shselasky [V1] = POLL_FN_ENTRY(_v1, _lock, , ), 1230321936Shselasky [0] = POLL_FN_ENTRY(_v0, _lock, , ), 1231321936Shselasky [V1 | SINGLE_THREADED] = POLL_FN_ENTRY(_v1, , , ), 1232321936Shselasky [SINGLE_THREADED] = POLL_FN_ENTRY(_v0, , , ), 1233321936Shselasky [V1 | STALL] = POLL_FN_ENTRY(_v1, _lock, _stall, ), 1234321936Shselasky [STALL] = POLL_FN_ENTRY(_v0, _lock, _stall, ), 1235321936Shselasky [V1 | SINGLE_THREADED | STALL] = POLL_FN_ENTRY(_v1, , _stall, ), 1236321936Shselasky [SINGLE_THREADED | STALL] = POLL_FN_ENTRY(_v0, , _stall, ), 1237321936Shselasky [V1 | STALL | ADAPTIVE] = POLL_FN_ENTRY(_v1, _lock, _stall, _adaptive), 1238321936Shselasky [STALL | ADAPTIVE] = POLL_FN_ENTRY(_v0, _lock, _stall, _adaptive), 1239321936Shselasky [V1 | SINGLE_THREADED | STALL | ADAPTIVE] = POLL_FN_ENTRY(_v1, , _stall, _adaptive), 1240321936Shselasky [SINGLE_THREADED | STALL | ADAPTIVE] = POLL_FN_ENTRY(_v0, , _stall, _adaptive), 1241321936Shselasky}; 1242321936Shselasky 1243321936Shselaskyvoid mlx5_cq_fill_pfns(struct mlx5_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr) 1244321936Shselasky{ 1245321936Shselasky struct mlx5_context *mctx = to_mctx(ibv_cq_ex_to_cq(&cq->ibv_cq)->context); 1246321936Shselasky const struct op *poll_ops = &ops[((cq->stall_enable && cq->stall_adaptive_enable) ? ADAPTIVE : 0) | 1247321936Shselasky (mctx->cqe_version ? V1 : 0) | 1248321936Shselasky (cq->flags & MLX5_CQ_FLAGS_SINGLE_THREADED ? 1249321936Shselasky SINGLE_THREADED : 0) | 1250321936Shselasky (cq->stall_enable ? STALL : 0)]; 1251321936Shselasky 1252321936Shselasky cq->ibv_cq.start_poll = poll_ops->start_poll; 1253321936Shselasky cq->ibv_cq.next_poll = poll_ops->next_poll; 1254321936Shselasky cq->ibv_cq.end_poll = poll_ops->end_poll; 1255321936Shselasky 1256321936Shselasky cq->ibv_cq.read_opcode = mlx5_cq_read_wc_opcode; 1257321936Shselasky cq->ibv_cq.read_vendor_err = mlx5_cq_read_wc_vendor_err; 1258321936Shselasky cq->ibv_cq.read_wc_flags = mlx5_cq_read_wc_flags; 1259321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_BYTE_LEN) 1260321936Shselasky cq->ibv_cq.read_byte_len = mlx5_cq_read_wc_byte_len; 1261321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_IMM) 1262321936Shselasky cq->ibv_cq.read_imm_data = mlx5_cq_read_wc_imm_data; 1263321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_QP_NUM) 1264321936Shselasky cq->ibv_cq.read_qp_num = mlx5_cq_read_wc_qp_num; 1265321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SRC_QP) 1266321936Shselasky cq->ibv_cq.read_src_qp = mlx5_cq_read_wc_src_qp; 1267321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SLID) 1268321936Shselasky cq->ibv_cq.read_slid = mlx5_cq_read_wc_slid; 1269321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SL) 1270321936Shselasky cq->ibv_cq.read_sl = mlx5_cq_read_wc_sl; 1271321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) 1272321936Shselasky cq->ibv_cq.read_dlid_path_bits = mlx5_cq_read_wc_dlid_path_bits; 1273321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) 1274321936Shselasky cq->ibv_cq.read_completion_ts = mlx5_cq_read_wc_completion_ts; 1275321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_CVLAN) 1276321936Shselasky cq->ibv_cq.read_cvlan = mlx5_cq_read_wc_cvlan; 1277321936Shselasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_FLOW_TAG) 1278321936Shselasky cq->ibv_cq.read_flow_tag = mlx5_cq_read_flow_tag; 1279321936Shselasky} 1280321936Shselasky 1281321936Shselaskyint mlx5_arm_cq(struct ibv_cq *ibvcq, int solicited) 1282321936Shselasky{ 1283321936Shselasky struct mlx5_cq *cq = to_mcq(ibvcq); 1284321936Shselasky struct mlx5_context *ctx = to_mctx(ibvcq->context); 1285321936Shselasky uint32_t doorbell[2]; 1286321936Shselasky uint32_t sn; 1287321936Shselasky uint32_t ci; 1288321936Shselasky uint32_t cmd; 1289321936Shselasky 1290321936Shselasky sn = cq->arm_sn & 3; 1291321936Shselasky ci = cq->cons_index & 0xffffff; 1292321936Shselasky cmd = solicited ? MLX5_CQ_DB_REQ_NOT_SOL : MLX5_CQ_DB_REQ_NOT; 1293321936Shselasky 1294321936Shselasky cq->dbrec[MLX5_CQ_ARM_DB] = htobe32(sn << 28 | cmd | ci); 1295321936Shselasky 1296321936Shselasky /* 1297321936Shselasky * Make sure that the doorbell record in host memory is 1298321936Shselasky * written before ringing the doorbell via PCI WC MMIO. 1299321936Shselasky */ 1300321936Shselasky mmio_wc_start(); 1301321936Shselasky 1302321936Shselasky doorbell[0] = htobe32(sn << 28 | cmd | ci); 1303321936Shselasky doorbell[1] = htobe32(cq->cqn); 1304321936Shselasky 1305321936Shselasky mlx5_write64(doorbell, ctx->uar[0] + MLX5_CQ_DOORBELL, &ctx->lock32); 1306321936Shselasky 1307321936Shselasky mmio_flush_writes(); 1308321936Shselasky 1309321936Shselasky return 0; 1310321936Shselasky} 1311321936Shselasky 1312321936Shselaskyvoid mlx5_cq_event(struct ibv_cq *cq) 1313321936Shselasky{ 1314321936Shselasky to_mcq(cq)->arm_sn++; 1315321936Shselasky} 1316321936Shselasky 1317321936Shselaskystatic int is_equal_rsn(struct mlx5_cqe64 *cqe64, uint32_t rsn) 1318321936Shselasky{ 1319321936Shselasky return rsn == (be32toh(cqe64->sop_drop_qpn) & 0xffffff); 1320321936Shselasky} 1321321936Shselasky 1322321936Shselaskystatic inline int is_equal_uidx(struct mlx5_cqe64 *cqe64, uint32_t uidx) 1323321936Shselasky{ 1324321936Shselasky return uidx == (be32toh(cqe64->srqn_uidx) & 0xffffff); 1325321936Shselasky} 1326321936Shselasky 1327321936Shselaskystatic inline int is_responder(uint8_t opcode) 1328321936Shselasky{ 1329321936Shselasky switch (opcode) { 1330321936Shselasky case MLX5_CQE_RESP_WR_IMM: 1331321936Shselasky case MLX5_CQE_RESP_SEND: 1332321936Shselasky case MLX5_CQE_RESP_SEND_IMM: 1333321936Shselasky case MLX5_CQE_RESP_SEND_INV: 1334321936Shselasky case MLX5_CQE_RESP_ERR: 1335321936Shselasky return 1; 1336321936Shselasky } 1337321936Shselasky 1338321936Shselasky return 0; 1339321936Shselasky} 1340321936Shselasky 1341321936Shselaskystatic inline int free_res_cqe(struct mlx5_cqe64 *cqe64, uint32_t rsn, 1342321936Shselasky struct mlx5_srq *srq, int cqe_version) 1343321936Shselasky{ 1344321936Shselasky if (cqe_version) { 1345321936Shselasky if (is_equal_uidx(cqe64, rsn)) { 1346321936Shselasky if (srq && is_responder(mlx5dv_get_cqe_opcode(cqe64))) 1347321936Shselasky mlx5_free_srq_wqe(srq, 1348321936Shselasky be16toh(cqe64->wqe_counter)); 1349321936Shselasky return 1; 1350321936Shselasky } 1351321936Shselasky } else { 1352321936Shselasky if (is_equal_rsn(cqe64, rsn)) { 1353321936Shselasky if (srq && (be32toh(cqe64->srqn_uidx) & 0xffffff)) 1354321936Shselasky mlx5_free_srq_wqe(srq, 1355321936Shselasky be16toh(cqe64->wqe_counter)); 1356321936Shselasky return 1; 1357321936Shselasky } 1358321936Shselasky } 1359321936Shselasky 1360321936Shselasky return 0; 1361321936Shselasky} 1362321936Shselasky 1363321936Shselaskyvoid __mlx5_cq_clean(struct mlx5_cq *cq, uint32_t rsn, struct mlx5_srq *srq) 1364321936Shselasky{ 1365321936Shselasky uint32_t prod_index; 1366321936Shselasky int nfreed = 0; 1367321936Shselasky struct mlx5_cqe64 *cqe64, *dest64; 1368321936Shselasky void *cqe, *dest; 1369321936Shselasky uint8_t owner_bit; 1370321936Shselasky int cqe_version; 1371321936Shselasky 1372321936Shselasky if (!cq || cq->flags & MLX5_CQ_FLAGS_DV_OWNED) 1373321936Shselasky return; 1374321936Shselasky 1375321936Shselasky /* 1376321936Shselasky * First we need to find the current producer index, so we 1377321936Shselasky * know where to start cleaning from. It doesn't matter if HW 1378321936Shselasky * adds new entries after this loop -- the QP we're worried 1379321936Shselasky * about is already in RESET, so the new entries won't come 1380321936Shselasky * from our QP and therefore don't need to be checked. 1381321936Shselasky */ 1382321936Shselasky for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index) 1383321936Shselasky if (prod_index == cq->cons_index + cq->ibv_cq.cqe) 1384321936Shselasky break; 1385321936Shselasky 1386321936Shselasky /* 1387321936Shselasky * Now sweep backwards through the CQ, removing CQ entries 1388321936Shselasky * that match our QP by copying older entries on top of them. 1389321936Shselasky */ 1390321936Shselasky cqe_version = (to_mctx(cq->ibv_cq.context))->cqe_version; 1391321936Shselasky while ((int) --prod_index - (int) cq->cons_index >= 0) { 1392321936Shselasky cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe); 1393321936Shselasky cqe64 = (cq->cqe_sz == 64) ? cqe : cqe + 64; 1394321936Shselasky if (free_res_cqe(cqe64, rsn, srq, cqe_version)) { 1395321936Shselasky ++nfreed; 1396321936Shselasky } else if (nfreed) { 1397321936Shselasky dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe); 1398321936Shselasky dest64 = (cq->cqe_sz == 64) ? dest : dest + 64; 1399321936Shselasky owner_bit = dest64->op_own & MLX5_CQE_OWNER_MASK; 1400321936Shselasky memcpy(dest, cqe, cq->cqe_sz); 1401321936Shselasky dest64->op_own = owner_bit | 1402321936Shselasky (dest64->op_own & ~MLX5_CQE_OWNER_MASK); 1403321936Shselasky } 1404321936Shselasky } 1405321936Shselasky 1406321936Shselasky if (nfreed) { 1407321936Shselasky cq->cons_index += nfreed; 1408321936Shselasky /* 1409321936Shselasky * Make sure update of buffer contents is done before 1410321936Shselasky * updating consumer index. 1411321936Shselasky */ 1412321936Shselasky udma_to_device_barrier(); 1413321936Shselasky update_cons_index(cq); 1414321936Shselasky } 1415321936Shselasky} 1416321936Shselasky 1417321936Shselaskyvoid mlx5_cq_clean(struct mlx5_cq *cq, uint32_t qpn, struct mlx5_srq *srq) 1418321936Shselasky{ 1419321936Shselasky mlx5_spin_lock(&cq->lock); 1420321936Shselasky __mlx5_cq_clean(cq, qpn, srq); 1421321936Shselasky mlx5_spin_unlock(&cq->lock); 1422321936Shselasky} 1423321936Shselasky 1424321936Shselaskystatic uint8_t sw_ownership_bit(int n, int nent) 1425321936Shselasky{ 1426321936Shselasky return (n & nent) ? 1 : 0; 1427321936Shselasky} 1428321936Shselasky 1429321936Shselaskystatic int is_hw(uint8_t own, int n, int mask) 1430321936Shselasky{ 1431321936Shselasky return (own & MLX5_CQE_OWNER_MASK) ^ !!(n & (mask + 1)); 1432321936Shselasky} 1433321936Shselasky 1434321936Shselaskyvoid mlx5_cq_resize_copy_cqes(struct mlx5_cq *cq) 1435321936Shselasky{ 1436321936Shselasky struct mlx5_cqe64 *scqe64; 1437321936Shselasky struct mlx5_cqe64 *dcqe64; 1438321936Shselasky void *start_cqe; 1439321936Shselasky void *scqe; 1440321936Shselasky void *dcqe; 1441321936Shselasky int ssize; 1442321936Shselasky int dsize; 1443321936Shselasky int i; 1444321936Shselasky uint8_t sw_own; 1445321936Shselasky 1446321936Shselasky ssize = cq->cqe_sz; 1447321936Shselasky dsize = cq->resize_cqe_sz; 1448321936Shselasky 1449321936Shselasky i = cq->cons_index; 1450321936Shselasky scqe = get_buf_cqe(cq->active_buf, i & cq->active_cqes, ssize); 1451321936Shselasky scqe64 = ssize == 64 ? scqe : scqe + 64; 1452321936Shselasky start_cqe = scqe; 1453321936Shselasky if (is_hw(scqe64->op_own, i, cq->active_cqes)) { 1454321936Shselasky fprintf(stderr, "expected cqe in sw ownership\n"); 1455321936Shselasky return; 1456321936Shselasky } 1457321936Shselasky 1458321936Shselasky while ((scqe64->op_own >> 4) != MLX5_CQE_RESIZE_CQ) { 1459321936Shselasky dcqe = get_buf_cqe(cq->resize_buf, (i + 1) & (cq->resize_cqes - 1), dsize); 1460321936Shselasky dcqe64 = dsize == 64 ? dcqe : dcqe + 64; 1461321936Shselasky sw_own = sw_ownership_bit(i + 1, cq->resize_cqes); 1462321936Shselasky memcpy(dcqe, scqe, ssize); 1463321936Shselasky dcqe64->op_own = (dcqe64->op_own & ~MLX5_CQE_OWNER_MASK) | sw_own; 1464321936Shselasky 1465321936Shselasky ++i; 1466321936Shselasky scqe = get_buf_cqe(cq->active_buf, i & cq->active_cqes, ssize); 1467321936Shselasky scqe64 = ssize == 64 ? scqe : scqe + 64; 1468321936Shselasky if (is_hw(scqe64->op_own, i, cq->active_cqes)) { 1469321936Shselasky fprintf(stderr, "expected cqe in sw ownership\n"); 1470321936Shselasky return; 1471321936Shselasky } 1472321936Shselasky 1473321936Shselasky if (scqe == start_cqe) { 1474321936Shselasky fprintf(stderr, "resize CQ failed to get resize CQE\n"); 1475321936Shselasky return; 1476321936Shselasky } 1477321936Shselasky } 1478321936Shselasky ++cq->cons_index; 1479321936Shselasky} 1480321936Shselasky 1481321936Shselaskyint mlx5_alloc_cq_buf(struct mlx5_context *mctx, struct mlx5_cq *cq, 1482321936Shselasky struct mlx5_buf *buf, int nent, int cqe_sz) 1483321936Shselasky{ 1484321936Shselasky struct mlx5_cqe64 *cqe; 1485321936Shselasky int i; 1486321936Shselasky struct mlx5_device *dev = to_mdev(mctx->ibv_ctx.device); 1487321936Shselasky int ret; 1488321936Shselasky enum mlx5_alloc_type type; 1489321936Shselasky enum mlx5_alloc_type default_type = MLX5_ALLOC_TYPE_ANON; 1490321936Shselasky 1491321936Shselasky if (mlx5_use_huge("HUGE_CQ")) 1492321936Shselasky default_type = MLX5_ALLOC_TYPE_HUGE; 1493321936Shselasky 1494321936Shselasky mlx5_get_alloc_type(MLX5_CQ_PREFIX, &type, default_type); 1495321936Shselasky 1496321936Shselasky ret = mlx5_alloc_prefered_buf(mctx, buf, 1497321936Shselasky align(nent * cqe_sz, dev->page_size), 1498321936Shselasky dev->page_size, 1499321936Shselasky type, 1500321936Shselasky MLX5_CQ_PREFIX); 1501321936Shselasky 1502321936Shselasky if (ret) 1503321936Shselasky return -1; 1504321936Shselasky 1505321936Shselasky memset(buf->buf, 0, nent * cqe_sz); 1506321936Shselasky 1507321936Shselasky for (i = 0; i < nent; ++i) { 1508321936Shselasky cqe = buf->buf + i * cqe_sz; 1509321936Shselasky cqe += cqe_sz == 128 ? 1 : 0; 1510321936Shselasky cqe->op_own = MLX5_CQE_INVALID << 4; 1511321936Shselasky } 1512321936Shselasky 1513321936Shselasky return 0; 1514321936Shselasky} 1515321936Shselasky 1516321936Shselaskyint mlx5_free_cq_buf(struct mlx5_context *ctx, struct mlx5_buf *buf) 1517321936Shselasky{ 1518321936Shselasky return mlx5_free_actual_buf(ctx, buf); 1519321936Shselasky} 1520