1321936Shselasky/* 2321936Shselasky * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3321936Shselasky * 4321936Shselasky * This software is available to you under a choice of one of two 5321936Shselasky * licenses. You may choose to be licensed under the terms of the GNU 6321936Shselasky * General Public License (GPL) Version 2, available from the file 7321936Shselasky * COPYING in the main directory of this source tree, or the 8321936Shselasky * OpenIB.org BSD license below: 9321936Shselasky * 10321936Shselasky * Redistribution and use in source and binary forms, with or 11321936Shselasky * without modification, are permitted provided that the following 12321936Shselasky * conditions are met: 13321936Shselasky * 14321936Shselasky * - Redistributions of source code must retain the above 15321936Shselasky * copyright notice, this list of conditions and the following 16321936Shselasky * disclaimer. 17321936Shselasky * 18321936Shselasky * - Redistributions in binary form must reproduce the above 19321936Shselasky * copyright notice, this list of conditions and the following 20321936Shselasky * disclaimer in the documentation and/or other materials 21321936Shselasky * provided with the distribution. 22321936Shselasky * 23321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30321936Shselasky * SOFTWARE. 31321936Shselasky */ 32321936Shselasky 33321936Shselasky#include <config.h> 34321936Shselasky 35321936Shselasky#include <stdlib.h> 36321936Shselasky#include <stdio.h> 37321936Shselasky#include <string.h> 38321936Shselasky#include <pthread.h> 39321936Shselasky#include <errno.h> 40321936Shselasky#include <limits.h> 41321936Shselasky#include <sys/types.h> 42321936Shselasky#include <sys/stat.h> 43321936Shselasky#include <fcntl.h> 44321936Shselasky#include <unistd.h> 45321936Shselasky#include <sys/mman.h> 46321936Shselasky 47321936Shselasky#include "mlx5.h" 48321936Shselasky#include "mlx5-abi.h" 49321936Shselasky#include "wqe.h" 50321936Shselasky 51321936Shselaskyint mlx5_single_threaded = 0; 52321936Shselasky 53321936Shselaskystatic inline int is_xrc_tgt(int type) 54321936Shselasky{ 55321936Shselasky return type == IBV_QPT_XRC_RECV; 56321936Shselasky} 57321936Shselasky 58321936Shselaskyint mlx5_query_device(struct ibv_context *context, struct ibv_device_attr *attr) 59321936Shselasky{ 60321936Shselasky struct ibv_query_device cmd; 61321936Shselasky uint64_t raw_fw_ver; 62321936Shselasky unsigned major, minor, sub_minor; 63321936Shselasky int ret; 64321936Shselasky 65321936Shselasky ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, sizeof cmd); 66321936Shselasky if (ret) 67321936Shselasky return ret; 68321936Shselasky 69321936Shselasky major = (raw_fw_ver >> 32) & 0xffff; 70321936Shselasky minor = (raw_fw_ver >> 16) & 0xffff; 71321936Shselasky sub_minor = raw_fw_ver & 0xffff; 72321936Shselasky 73321936Shselasky snprintf(attr->fw_ver, sizeof attr->fw_ver, 74321936Shselasky "%d.%d.%04d", major, minor, sub_minor); 75321936Shselasky 76321936Shselasky return 0; 77321936Shselasky} 78321936Shselasky 79321936Shselasky#define READL(ptr) (*((uint32_t *)(ptr))) 80321936Shselaskystatic int mlx5_read_clock(struct ibv_context *context, uint64_t *cycles) 81321936Shselasky{ 82321936Shselasky unsigned int clockhi, clocklo, clockhi1; 83321936Shselasky int i; 84321936Shselasky struct mlx5_context *ctx = to_mctx(context); 85321936Shselasky 86321936Shselasky if (!ctx->hca_core_clock) 87321936Shselasky return -EOPNOTSUPP; 88321936Shselasky 89321936Shselasky /* Handle wraparound */ 90321936Shselasky for (i = 0; i < 2; i++) { 91321936Shselasky clockhi = be32toh(READL(ctx->hca_core_clock)); 92321936Shselasky clocklo = be32toh(READL(ctx->hca_core_clock + 4)); 93321936Shselasky clockhi1 = be32toh(READL(ctx->hca_core_clock)); 94321936Shselasky if (clockhi == clockhi1) 95321936Shselasky break; 96321936Shselasky } 97321936Shselasky 98321936Shselasky *cycles = (uint64_t)clockhi << 32 | (uint64_t)clocklo; 99321936Shselasky 100321936Shselasky return 0; 101321936Shselasky} 102321936Shselasky 103321936Shselaskyint mlx5_query_rt_values(struct ibv_context *context, 104321936Shselasky struct ibv_values_ex *values) 105321936Shselasky{ 106321936Shselasky uint32_t comp_mask = 0; 107321936Shselasky int err = 0; 108321936Shselasky 109321936Shselasky if (values->comp_mask & IBV_VALUES_MASK_RAW_CLOCK) { 110321936Shselasky uint64_t cycles; 111321936Shselasky 112321936Shselasky err = mlx5_read_clock(context, &cycles); 113321936Shselasky if (!err) { 114321936Shselasky values->raw_clock.tv_sec = 0; 115321936Shselasky values->raw_clock.tv_nsec = cycles; 116321936Shselasky comp_mask |= IBV_VALUES_MASK_RAW_CLOCK; 117321936Shselasky } 118321936Shselasky } 119321936Shselasky 120321936Shselasky values->comp_mask = comp_mask; 121321936Shselasky 122321936Shselasky return err; 123321936Shselasky} 124321936Shselasky 125321936Shselaskyint mlx5_query_port(struct ibv_context *context, uint8_t port, 126321936Shselasky struct ibv_port_attr *attr) 127321936Shselasky{ 128321936Shselasky struct ibv_query_port cmd; 129321936Shselasky 130321936Shselasky return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); 131321936Shselasky} 132321936Shselasky 133321936Shselaskystruct ibv_pd *mlx5_alloc_pd(struct ibv_context *context) 134321936Shselasky{ 135321936Shselasky struct ibv_alloc_pd cmd; 136321936Shselasky struct mlx5_alloc_pd_resp resp; 137321936Shselasky struct mlx5_pd *pd; 138321936Shselasky 139321936Shselasky pd = calloc(1, sizeof *pd); 140321936Shselasky if (!pd) 141321936Shselasky return NULL; 142321936Shselasky 143321936Shselasky if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd, 144321936Shselasky &resp.ibv_resp, sizeof resp)) { 145321936Shselasky free(pd); 146321936Shselasky return NULL; 147321936Shselasky } 148321936Shselasky 149321936Shselasky pd->pdn = resp.pdn; 150321936Shselasky 151321936Shselasky return &pd->ibv_pd; 152321936Shselasky} 153321936Shselasky 154321936Shselaskyint mlx5_free_pd(struct ibv_pd *pd) 155321936Shselasky{ 156321936Shselasky int ret; 157321936Shselasky 158321936Shselasky ret = ibv_cmd_dealloc_pd(pd); 159321936Shselasky if (ret) 160321936Shselasky return ret; 161321936Shselasky 162321936Shselasky free(to_mpd(pd)); 163321936Shselasky return 0; 164321936Shselasky} 165321936Shselasky 166321936Shselaskystruct ibv_mr *mlx5_reg_mr(struct ibv_pd *pd, void *addr, size_t length, 167321936Shselasky int acc) 168321936Shselasky{ 169321936Shselasky struct mlx5_mr *mr; 170321936Shselasky struct ibv_reg_mr cmd; 171321936Shselasky int ret; 172321936Shselasky enum ibv_access_flags access = (enum ibv_access_flags)acc; 173321936Shselasky struct ibv_reg_mr_resp resp; 174321936Shselasky 175321936Shselasky mr = calloc(1, sizeof(*mr)); 176321936Shselasky if (!mr) 177321936Shselasky return NULL; 178321936Shselasky 179321936Shselasky ret = ibv_cmd_reg_mr(pd, addr, length, (uintptr_t)addr, access, 180321936Shselasky &(mr->ibv_mr), &cmd, sizeof(cmd), &resp, 181321936Shselasky sizeof resp); 182321936Shselasky if (ret) { 183321936Shselasky mlx5_free_buf(&(mr->buf)); 184321936Shselasky free(mr); 185321936Shselasky return NULL; 186321936Shselasky } 187321936Shselasky mr->alloc_flags = acc; 188321936Shselasky 189321936Shselasky return &mr->ibv_mr; 190321936Shselasky} 191321936Shselasky 192321936Shselaskyint mlx5_rereg_mr(struct ibv_mr *ibmr, int flags, struct ibv_pd *pd, void *addr, 193321936Shselasky size_t length, int access) 194321936Shselasky{ 195321936Shselasky struct ibv_rereg_mr cmd; 196321936Shselasky struct ibv_rereg_mr_resp resp; 197321936Shselasky 198321936Shselasky if (flags & IBV_REREG_MR_KEEP_VALID) 199321936Shselasky return ENOTSUP; 200321936Shselasky 201321936Shselasky return ibv_cmd_rereg_mr(ibmr, flags, addr, length, (uintptr_t)addr, 202321936Shselasky access, pd, &cmd, sizeof(cmd), &resp, 203321936Shselasky sizeof(resp)); 204321936Shselasky} 205321936Shselasky 206321936Shselaskyint mlx5_dereg_mr(struct ibv_mr *ibmr) 207321936Shselasky{ 208321936Shselasky int ret; 209321936Shselasky struct mlx5_mr *mr = to_mmr(ibmr); 210321936Shselasky 211321936Shselasky ret = ibv_cmd_dereg_mr(ibmr); 212321936Shselasky if (ret) 213321936Shselasky return ret; 214321936Shselasky 215321936Shselasky free(mr); 216321936Shselasky return 0; 217321936Shselasky} 218321936Shselasky 219321936Shselaskystruct ibv_mw *mlx5_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type) 220321936Shselasky{ 221321936Shselasky struct ibv_mw *mw; 222321936Shselasky struct ibv_alloc_mw cmd; 223321936Shselasky struct ibv_alloc_mw_resp resp; 224321936Shselasky int ret; 225321936Shselasky 226321936Shselasky mw = malloc(sizeof(*mw)); 227321936Shselasky if (!mw) 228321936Shselasky return NULL; 229321936Shselasky 230321936Shselasky memset(mw, 0, sizeof(*mw)); 231321936Shselasky 232321936Shselasky ret = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, 233321936Shselasky sizeof(resp)); 234321936Shselasky if (ret) { 235321936Shselasky free(mw); 236321936Shselasky return NULL; 237321936Shselasky } 238321936Shselasky 239321936Shselasky return mw; 240321936Shselasky} 241321936Shselasky 242321936Shselaskyint mlx5_dealloc_mw(struct ibv_mw *mw) 243321936Shselasky{ 244321936Shselasky int ret; 245321936Shselasky struct ibv_dealloc_mw cmd; 246321936Shselasky 247321936Shselasky ret = ibv_cmd_dealloc_mw(mw, &cmd, sizeof(cmd)); 248321936Shselasky if (ret) 249321936Shselasky return ret; 250321936Shselasky 251321936Shselasky free(mw); 252321936Shselasky return 0; 253321936Shselasky} 254321936Shselasky 255321936Shselaskyint mlx5_round_up_power_of_two(long long sz) 256321936Shselasky{ 257321936Shselasky long long ret; 258321936Shselasky 259321936Shselasky for (ret = 1; ret < sz; ret <<= 1) 260321936Shselasky ; /* nothing */ 261321936Shselasky 262321936Shselasky if (ret > INT_MAX) { 263321936Shselasky fprintf(stderr, "%s: roundup overflow\n", __func__); 264321936Shselasky return -ENOMEM; 265321936Shselasky } 266321936Shselasky 267321936Shselasky return (int)ret; 268321936Shselasky} 269321936Shselasky 270321936Shselaskystatic int align_queue_size(long long req) 271321936Shselasky{ 272321936Shselasky return mlx5_round_up_power_of_two(req); 273321936Shselasky} 274321936Shselasky 275321936Shselaskystatic int get_cqe_size(void) 276321936Shselasky{ 277321936Shselasky char *env; 278321936Shselasky int size = 64; 279321936Shselasky 280321936Shselasky env = getenv("MLX5_CQE_SIZE"); 281321936Shselasky if (env) 282321936Shselasky size = atoi(env); 283321936Shselasky 284321936Shselasky switch (size) { 285321936Shselasky case 64: 286321936Shselasky case 128: 287321936Shselasky return size; 288321936Shselasky 289321936Shselasky default: 290321936Shselasky return -EINVAL; 291321936Shselasky } 292321936Shselasky} 293321936Shselasky 294321936Shselaskystatic int use_scatter_to_cqe(void) 295321936Shselasky{ 296321936Shselasky char *env; 297321936Shselasky 298321936Shselasky env = getenv("MLX5_SCATTER_TO_CQE"); 299321936Shselasky if (env && !strcmp(env, "0")) 300321936Shselasky return 0; 301321936Shselasky 302321936Shselasky return 1; 303321936Shselasky} 304321936Shselasky 305321936Shselaskystatic int srq_sig_enabled(void) 306321936Shselasky{ 307321936Shselasky char *env; 308321936Shselasky 309321936Shselasky env = getenv("MLX5_SRQ_SIGNATURE"); 310321936Shselasky if (env) 311321936Shselasky return 1; 312321936Shselasky 313321936Shselasky return 0; 314321936Shselasky} 315321936Shselasky 316321936Shselaskystatic int qp_sig_enabled(void) 317321936Shselasky{ 318321936Shselasky char *env; 319321936Shselasky 320321936Shselasky env = getenv("MLX5_QP_SIGNATURE"); 321321936Shselasky if (env) 322321936Shselasky return 1; 323321936Shselasky 324321936Shselasky return 0; 325321936Shselasky} 326321936Shselasky 327321936Shselaskyenum { 328321936Shselasky CREATE_CQ_SUPPORTED_WC_FLAGS = IBV_WC_STANDARD_FLAGS | 329321936Shselasky IBV_WC_EX_WITH_COMPLETION_TIMESTAMP | 330321936Shselasky IBV_WC_EX_WITH_CVLAN | 331321936Shselasky IBV_WC_EX_WITH_FLOW_TAG 332321936Shselasky}; 333321936Shselasky 334321936Shselaskyenum { 335321936Shselasky CREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS 336321936Shselasky}; 337321936Shselasky 338321936Shselaskyenum { 339321936Shselasky CREATE_CQ_SUPPORTED_FLAGS = IBV_CREATE_CQ_ATTR_SINGLE_THREADED 340321936Shselasky}; 341321936Shselasky 342321936Shselaskystatic struct ibv_cq_ex *create_cq(struct ibv_context *context, 343321936Shselasky const struct ibv_cq_init_attr_ex *cq_attr, 344321936Shselasky int cq_alloc_flags, 345321936Shselasky struct mlx5dv_cq_init_attr *mlx5cq_attr) 346321936Shselasky{ 347321936Shselasky struct mlx5_create_cq cmd; 348321936Shselasky struct mlx5_create_cq_resp resp; 349321936Shselasky struct mlx5_cq *cq; 350321936Shselasky int cqe_sz; 351321936Shselasky int ret; 352321936Shselasky int ncqe; 353321936Shselasky struct mlx5_context *mctx = to_mctx(context); 354321936Shselasky FILE *fp = to_mctx(context)->dbg_fp; 355321936Shselasky 356321936Shselasky if (!cq_attr->cqe) { 357321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "CQE invalid\n"); 358321936Shselasky errno = EINVAL; 359321936Shselasky return NULL; 360321936Shselasky } 361321936Shselasky 362321936Shselasky if (cq_attr->comp_mask & ~CREATE_CQ_SUPPORTED_COMP_MASK) { 363321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, 364321936Shselasky "Unsupported comp_mask for create_cq\n"); 365321936Shselasky errno = EINVAL; 366321936Shselasky return NULL; 367321936Shselasky } 368321936Shselasky 369321936Shselasky if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && 370321936Shselasky cq_attr->flags & ~CREATE_CQ_SUPPORTED_FLAGS) { 371321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, 372321936Shselasky "Unsupported creation flags requested for create_cq\n"); 373321936Shselasky errno = EINVAL; 374321936Shselasky return NULL; 375321936Shselasky } 376321936Shselasky 377321936Shselasky if (cq_attr->wc_flags & ~CREATE_CQ_SUPPORTED_WC_FLAGS) { 378321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 379321936Shselasky errno = ENOTSUP; 380321936Shselasky return NULL; 381321936Shselasky } 382321936Shselasky 383321936Shselasky cq = calloc(1, sizeof *cq); 384321936Shselasky if (!cq) { 385321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 386321936Shselasky return NULL; 387321936Shselasky } 388321936Shselasky 389321936Shselasky memset(&cmd, 0, sizeof cmd); 390321936Shselasky cq->cons_index = 0; 391321936Shselasky 392321936Shselasky if (mlx5_spinlock_init(&cq->lock)) 393321936Shselasky goto err; 394321936Shselasky 395321936Shselasky ncqe = align_queue_size(cq_attr->cqe + 1); 396321936Shselasky if ((ncqe > (1 << 24)) || (ncqe < (cq_attr->cqe + 1))) { 397321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "ncqe %d\n", ncqe); 398321936Shselasky errno = EINVAL; 399321936Shselasky goto err_spl; 400321936Shselasky } 401321936Shselasky 402321936Shselasky cqe_sz = get_cqe_size(); 403321936Shselasky if (cqe_sz < 0) { 404321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 405321936Shselasky errno = -cqe_sz; 406321936Shselasky goto err_spl; 407321936Shselasky } 408321936Shselasky 409321936Shselasky if (mlx5_alloc_cq_buf(to_mctx(context), cq, &cq->buf_a, ncqe, cqe_sz)) { 410321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 411321936Shselasky goto err_spl; 412321936Shselasky } 413321936Shselasky 414321936Shselasky cq->dbrec = mlx5_alloc_dbrec(to_mctx(context)); 415321936Shselasky if (!cq->dbrec) { 416321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "\n"); 417321936Shselasky goto err_buf; 418321936Shselasky } 419321936Shselasky 420321936Shselasky cq->dbrec[MLX5_CQ_SET_CI] = 0; 421321936Shselasky cq->dbrec[MLX5_CQ_ARM_DB] = 0; 422321936Shselasky cq->arm_sn = 0; 423321936Shselasky cq->cqe_sz = cqe_sz; 424321936Shselasky cq->flags = cq_alloc_flags; 425321936Shselasky 426321936Shselasky if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && 427321936Shselasky cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) 428321936Shselasky cq->flags |= MLX5_CQ_FLAGS_SINGLE_THREADED; 429321936Shselasky cmd.buf_addr = (uintptr_t) cq->buf_a.buf; 430321936Shselasky cmd.db_addr = (uintptr_t) cq->dbrec; 431321936Shselasky cmd.cqe_size = cqe_sz; 432321936Shselasky 433321936Shselasky if (mlx5cq_attr) { 434321936Shselasky if (mlx5cq_attr->comp_mask & ~(MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { 435321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, 436321936Shselasky "Unsupported vendor comp_mask for create_cq\n"); 437321936Shselasky errno = EINVAL; 438321936Shselasky goto err_db; 439321936Shselasky } 440321936Shselasky 441321936Shselasky if (mlx5cq_attr->comp_mask & MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE) { 442321936Shselasky if (mctx->cqe_comp_caps.max_num && 443321936Shselasky (mlx5cq_attr->cqe_comp_res_format & 444321936Shselasky mctx->cqe_comp_caps.supported_format)) { 445321936Shselasky cmd.cqe_comp_en = 1; 446321936Shselasky cmd.cqe_comp_res_format = mlx5cq_attr->cqe_comp_res_format; 447321936Shselasky } else { 448321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "CQE Compression is not supported\n"); 449321936Shselasky errno = EINVAL; 450321936Shselasky goto err_db; 451321936Shselasky } 452321936Shselasky } 453321936Shselasky } 454321936Shselasky 455321936Shselasky ret = ibv_cmd_create_cq(context, ncqe - 1, cq_attr->channel, 456321936Shselasky cq_attr->comp_vector, 457321936Shselasky ibv_cq_ex_to_cq(&cq->ibv_cq), &cmd.ibv_cmd, 458321936Shselasky sizeof(cmd), &resp.ibv_resp, sizeof(resp)); 459321936Shselasky if (ret) { 460321936Shselasky mlx5_dbg(fp, MLX5_DBG_CQ, "ret %d\n", ret); 461321936Shselasky goto err_db; 462321936Shselasky } 463321936Shselasky 464321936Shselasky cq->active_buf = &cq->buf_a; 465321936Shselasky cq->resize_buf = NULL; 466321936Shselasky cq->cqn = resp.cqn; 467321936Shselasky cq->stall_enable = to_mctx(context)->stall_enable; 468321936Shselasky cq->stall_adaptive_enable = to_mctx(context)->stall_adaptive_enable; 469321936Shselasky cq->stall_cycles = to_mctx(context)->stall_cycles; 470321936Shselasky 471321936Shselasky if (cq_alloc_flags & MLX5_CQ_FLAGS_EXTENDED) 472321936Shselasky mlx5_cq_fill_pfns(cq, cq_attr); 473321936Shselasky 474321936Shselasky return &cq->ibv_cq; 475321936Shselasky 476321936Shselaskyerr_db: 477321936Shselasky mlx5_free_db(to_mctx(context), cq->dbrec); 478321936Shselasky 479321936Shselaskyerr_buf: 480321936Shselasky mlx5_free_cq_buf(to_mctx(context), &cq->buf_a); 481321936Shselasky 482321936Shselaskyerr_spl: 483321936Shselasky mlx5_spinlock_destroy(&cq->lock); 484321936Shselasky 485321936Shselaskyerr: 486321936Shselasky free(cq); 487321936Shselasky 488321936Shselasky return NULL; 489321936Shselasky} 490321936Shselasky 491321936Shselaskystruct ibv_cq *mlx5_create_cq(struct ibv_context *context, int cqe, 492321936Shselasky struct ibv_comp_channel *channel, 493321936Shselasky int comp_vector) 494321936Shselasky{ 495321936Shselasky struct ibv_cq_ex *cq; 496321936Shselasky struct ibv_cq_init_attr_ex cq_attr = {.cqe = cqe, .channel = channel, 497321936Shselasky .comp_vector = comp_vector, 498321936Shselasky .wc_flags = IBV_WC_STANDARD_FLAGS}; 499321936Shselasky 500321936Shselasky if (cqe <= 0) { 501321936Shselasky errno = EINVAL; 502321936Shselasky return NULL; 503321936Shselasky } 504321936Shselasky 505321936Shselasky cq = create_cq(context, &cq_attr, 0, NULL); 506321936Shselasky return cq ? ibv_cq_ex_to_cq(cq) : NULL; 507321936Shselasky} 508321936Shselasky 509321936Shselaskystruct ibv_cq_ex *mlx5_create_cq_ex(struct ibv_context *context, 510321936Shselasky struct ibv_cq_init_attr_ex *cq_attr) 511321936Shselasky{ 512321936Shselasky return create_cq(context, cq_attr, MLX5_CQ_FLAGS_EXTENDED, NULL); 513321936Shselasky} 514321936Shselasky 515321936Shselaskystruct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context, 516321936Shselasky struct ibv_cq_init_attr_ex *cq_attr, 517321936Shselasky struct mlx5dv_cq_init_attr *mlx5_cq_attr) 518321936Shselasky{ 519321936Shselasky struct ibv_cq_ex *cq; 520321936Shselasky 521321936Shselasky cq = create_cq(context, cq_attr, MLX5_CQ_FLAGS_EXTENDED, mlx5_cq_attr); 522321936Shselasky if (!cq) 523321936Shselasky return NULL; 524321936Shselasky 525321936Shselasky verbs_init_cq(ibv_cq_ex_to_cq(cq), context, 526321936Shselasky cq_attr->channel, cq_attr->cq_context); 527321936Shselasky return cq; 528321936Shselasky} 529321936Shselasky 530321936Shselaskyint mlx5_resize_cq(struct ibv_cq *ibcq, int cqe) 531321936Shselasky{ 532321936Shselasky struct mlx5_cq *cq = to_mcq(ibcq); 533321936Shselasky struct mlx5_resize_cq_resp resp; 534321936Shselasky struct mlx5_resize_cq cmd; 535321936Shselasky struct mlx5_context *mctx = to_mctx(ibcq->context); 536321936Shselasky int err; 537321936Shselasky 538321936Shselasky if (cqe < 0) { 539321936Shselasky errno = EINVAL; 540321936Shselasky return errno; 541321936Shselasky } 542321936Shselasky 543321936Shselasky memset(&cmd, 0, sizeof(cmd)); 544321936Shselasky memset(&resp, 0, sizeof(resp)); 545321936Shselasky 546321936Shselasky if (((long long)cqe * 64) > INT_MAX) 547321936Shselasky return EINVAL; 548321936Shselasky 549321936Shselasky mlx5_spin_lock(&cq->lock); 550321936Shselasky cq->active_cqes = cq->ibv_cq.cqe; 551321936Shselasky if (cq->active_buf == &cq->buf_a) 552321936Shselasky cq->resize_buf = &cq->buf_b; 553321936Shselasky else 554321936Shselasky cq->resize_buf = &cq->buf_a; 555321936Shselasky 556321936Shselasky cqe = align_queue_size(cqe + 1); 557321936Shselasky if (cqe == ibcq->cqe + 1) { 558321936Shselasky cq->resize_buf = NULL; 559321936Shselasky err = 0; 560321936Shselasky goto out; 561321936Shselasky } 562321936Shselasky 563321936Shselasky /* currently we don't change cqe size */ 564321936Shselasky cq->resize_cqe_sz = cq->cqe_sz; 565321936Shselasky cq->resize_cqes = cqe; 566321936Shselasky err = mlx5_alloc_cq_buf(mctx, cq, cq->resize_buf, cq->resize_cqes, cq->resize_cqe_sz); 567321936Shselasky if (err) { 568321936Shselasky cq->resize_buf = NULL; 569321936Shselasky errno = ENOMEM; 570321936Shselasky goto out; 571321936Shselasky } 572321936Shselasky 573321936Shselasky cmd.buf_addr = (uintptr_t)cq->resize_buf->buf; 574321936Shselasky cmd.cqe_size = cq->resize_cqe_sz; 575321936Shselasky 576321936Shselasky err = ibv_cmd_resize_cq(ibcq, cqe - 1, &cmd.ibv_cmd, sizeof(cmd), 577321936Shselasky &resp.ibv_resp, sizeof(resp)); 578321936Shselasky if (err) 579321936Shselasky goto out_buf; 580321936Shselasky 581321936Shselasky mlx5_cq_resize_copy_cqes(cq); 582321936Shselasky mlx5_free_cq_buf(mctx, cq->active_buf); 583321936Shselasky cq->active_buf = cq->resize_buf; 584321936Shselasky cq->ibv_cq.cqe = cqe - 1; 585321936Shselasky mlx5_spin_unlock(&cq->lock); 586321936Shselasky cq->resize_buf = NULL; 587321936Shselasky return 0; 588321936Shselasky 589321936Shselaskyout_buf: 590321936Shselasky mlx5_free_cq_buf(mctx, cq->resize_buf); 591321936Shselasky cq->resize_buf = NULL; 592321936Shselasky 593321936Shselaskyout: 594321936Shselasky mlx5_spin_unlock(&cq->lock); 595321936Shselasky return err; 596321936Shselasky} 597321936Shselasky 598321936Shselaskyint mlx5_destroy_cq(struct ibv_cq *cq) 599321936Shselasky{ 600321936Shselasky int ret; 601321936Shselasky 602321936Shselasky ret = ibv_cmd_destroy_cq(cq); 603321936Shselasky if (ret) 604321936Shselasky return ret; 605321936Shselasky 606321936Shselasky mlx5_free_db(to_mctx(cq->context), to_mcq(cq)->dbrec); 607321936Shselasky mlx5_free_cq_buf(to_mctx(cq->context), to_mcq(cq)->active_buf); 608321936Shselasky free(to_mcq(cq)); 609321936Shselasky 610321936Shselasky return 0; 611321936Shselasky} 612321936Shselasky 613321936Shselaskystruct ibv_srq *mlx5_create_srq(struct ibv_pd *pd, 614321936Shselasky struct ibv_srq_init_attr *attr) 615321936Shselasky{ 616321936Shselasky struct mlx5_create_srq cmd; 617321936Shselasky struct mlx5_create_srq_resp resp; 618321936Shselasky struct mlx5_srq *srq; 619321936Shselasky int ret; 620321936Shselasky struct mlx5_context *ctx; 621321936Shselasky int max_sge; 622321936Shselasky struct ibv_srq *ibsrq; 623321936Shselasky 624321936Shselasky ctx = to_mctx(pd->context); 625321936Shselasky srq = calloc(1, sizeof *srq); 626321936Shselasky if (!srq) { 627321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 628321936Shselasky return NULL; 629321936Shselasky } 630321936Shselasky ibsrq = &srq->vsrq.srq; 631321936Shselasky 632321936Shselasky memset(&cmd, 0, sizeof cmd); 633321936Shselasky if (mlx5_spinlock_init(&srq->lock)) { 634321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 635321936Shselasky goto err; 636321936Shselasky } 637321936Shselasky 638321936Shselasky if (attr->attr.max_wr > ctx->max_srq_recv_wr) { 639321936Shselasky fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", __func__, __LINE__, 640321936Shselasky attr->attr.max_wr, ctx->max_srq_recv_wr); 641321936Shselasky errno = EINVAL; 642321936Shselasky goto err; 643321936Shselasky } 644321936Shselasky 645321936Shselasky /* 646321936Shselasky * this calculation does not consider required control segments. The 647321936Shselasky * final calculation is done again later. This is done so to avoid 648321936Shselasky * overflows of variables 649321936Shselasky */ 650321936Shselasky max_sge = ctx->max_rq_desc_sz / sizeof(struct mlx5_wqe_data_seg); 651321936Shselasky if (attr->attr.max_sge > max_sge) { 652321936Shselasky fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", __func__, __LINE__, 653321936Shselasky attr->attr.max_wr, ctx->max_srq_recv_wr); 654321936Shselasky errno = EINVAL; 655321936Shselasky goto err; 656321936Shselasky } 657321936Shselasky 658321936Shselasky srq->max = align_queue_size(attr->attr.max_wr + 1); 659321936Shselasky srq->max_gs = attr->attr.max_sge; 660321936Shselasky srq->counter = 0; 661321936Shselasky 662321936Shselasky if (mlx5_alloc_srq_buf(pd->context, srq)) { 663321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 664321936Shselasky goto err; 665321936Shselasky } 666321936Shselasky 667321936Shselasky srq->db = mlx5_alloc_dbrec(to_mctx(pd->context)); 668321936Shselasky if (!srq->db) { 669321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 670321936Shselasky goto err_free; 671321936Shselasky } 672321936Shselasky 673321936Shselasky *srq->db = 0; 674321936Shselasky 675321936Shselasky cmd.buf_addr = (uintptr_t) srq->buf.buf; 676321936Shselasky cmd.db_addr = (uintptr_t) srq->db; 677321936Shselasky srq->wq_sig = srq_sig_enabled(); 678321936Shselasky if (srq->wq_sig) 679321936Shselasky cmd.flags = MLX5_SRQ_FLAG_SIGNATURE; 680321936Shselasky 681321936Shselasky attr->attr.max_sge = srq->max_gs; 682321936Shselasky pthread_mutex_lock(&ctx->srq_table_mutex); 683321936Shselasky ret = ibv_cmd_create_srq(pd, ibsrq, attr, &cmd.ibv_cmd, sizeof(cmd), 684321936Shselasky &resp.ibv_resp, sizeof(resp)); 685321936Shselasky if (ret) 686321936Shselasky goto err_db; 687321936Shselasky 688321936Shselasky ret = mlx5_store_srq(ctx, resp.srqn, srq); 689321936Shselasky if (ret) 690321936Shselasky goto err_destroy; 691321936Shselasky 692321936Shselasky pthread_mutex_unlock(&ctx->srq_table_mutex); 693321936Shselasky 694321936Shselasky srq->srqn = resp.srqn; 695321936Shselasky srq->rsc.rsn = resp.srqn; 696321936Shselasky srq->rsc.type = MLX5_RSC_TYPE_SRQ; 697321936Shselasky 698321936Shselasky return ibsrq; 699321936Shselasky 700321936Shselaskyerr_destroy: 701321936Shselasky ibv_cmd_destroy_srq(ibsrq); 702321936Shselasky 703321936Shselaskyerr_db: 704321936Shselasky pthread_mutex_unlock(&ctx->srq_table_mutex); 705321936Shselasky mlx5_free_db(to_mctx(pd->context), srq->db); 706321936Shselasky 707321936Shselaskyerr_free: 708321936Shselasky free(srq->wrid); 709321936Shselasky mlx5_free_buf(&srq->buf); 710321936Shselasky 711321936Shselaskyerr: 712321936Shselasky free(srq); 713321936Shselasky 714321936Shselasky return NULL; 715321936Shselasky} 716321936Shselasky 717321936Shselaskyint mlx5_modify_srq(struct ibv_srq *srq, 718321936Shselasky struct ibv_srq_attr *attr, 719321936Shselasky int attr_mask) 720321936Shselasky{ 721321936Shselasky struct ibv_modify_srq cmd; 722321936Shselasky 723321936Shselasky return ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof cmd); 724321936Shselasky} 725321936Shselasky 726321936Shselaskyint mlx5_query_srq(struct ibv_srq *srq, 727321936Shselasky struct ibv_srq_attr *attr) 728321936Shselasky{ 729321936Shselasky struct ibv_query_srq cmd; 730321936Shselasky 731321936Shselasky return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd); 732321936Shselasky} 733321936Shselasky 734321936Shselaskyint mlx5_destroy_srq(struct ibv_srq *srq) 735321936Shselasky{ 736321936Shselasky int ret; 737321936Shselasky struct mlx5_srq *msrq = to_msrq(srq); 738321936Shselasky struct mlx5_context *ctx = to_mctx(srq->context); 739321936Shselasky 740321936Shselasky ret = ibv_cmd_destroy_srq(srq); 741321936Shselasky if (ret) 742321936Shselasky return ret; 743321936Shselasky 744321936Shselasky if (ctx->cqe_version && msrq->rsc.type == MLX5_RSC_TYPE_XSRQ) 745321936Shselasky mlx5_clear_uidx(ctx, msrq->rsc.rsn); 746321936Shselasky else 747321936Shselasky mlx5_clear_srq(ctx, msrq->srqn); 748321936Shselasky 749321936Shselasky mlx5_free_db(ctx, msrq->db); 750321936Shselasky mlx5_free_buf(&msrq->buf); 751321936Shselasky free(msrq->wrid); 752321936Shselasky free(msrq); 753321936Shselasky 754321936Shselasky return 0; 755321936Shselasky} 756321936Shselasky 757321936Shselaskystatic int sq_overhead(enum ibv_qp_type qp_type) 758321936Shselasky{ 759321936Shselasky size_t size = 0; 760321936Shselasky size_t mw_bind_size = 761321936Shselasky sizeof(struct mlx5_wqe_umr_ctrl_seg) + 762321936Shselasky sizeof(struct mlx5_wqe_mkey_context_seg) + 763321936Shselasky max_t(size_t, sizeof(struct mlx5_wqe_umr_klm_seg), 64); 764321936Shselasky 765321936Shselasky switch (qp_type) { 766321936Shselasky case IBV_QPT_RC: 767321936Shselasky size += sizeof(struct mlx5_wqe_ctrl_seg) + 768321936Shselasky max(sizeof(struct mlx5_wqe_atomic_seg) + 769321936Shselasky sizeof(struct mlx5_wqe_raddr_seg), 770321936Shselasky mw_bind_size); 771321936Shselasky break; 772321936Shselasky 773321936Shselasky case IBV_QPT_UC: 774321936Shselasky size = sizeof(struct mlx5_wqe_ctrl_seg) + 775321936Shselasky max(sizeof(struct mlx5_wqe_raddr_seg), 776321936Shselasky mw_bind_size); 777321936Shselasky break; 778321936Shselasky 779321936Shselasky case IBV_QPT_UD: 780321936Shselasky size = sizeof(struct mlx5_wqe_ctrl_seg) + 781321936Shselasky sizeof(struct mlx5_wqe_datagram_seg); 782321936Shselasky break; 783321936Shselasky 784321936Shselasky case IBV_QPT_XRC_SEND: 785321936Shselasky size = sizeof(struct mlx5_wqe_ctrl_seg) + mw_bind_size; 786321936Shselasky SWITCH_FALLTHROUGH; 787321936Shselasky 788321936Shselasky case IBV_QPT_XRC_RECV: 789321936Shselasky size = max(size, sizeof(struct mlx5_wqe_ctrl_seg) + 790321936Shselasky sizeof(struct mlx5_wqe_xrc_seg) + 791321936Shselasky sizeof(struct mlx5_wqe_raddr_seg)); 792321936Shselasky break; 793321936Shselasky 794321936Shselasky case IBV_QPT_RAW_PACKET: 795321936Shselasky size = sizeof(struct mlx5_wqe_ctrl_seg) + 796321936Shselasky sizeof(struct mlx5_wqe_eth_seg); 797321936Shselasky break; 798321936Shselasky 799321936Shselasky default: 800321936Shselasky return -EINVAL; 801321936Shselasky } 802321936Shselasky 803321936Shselasky return size; 804321936Shselasky} 805321936Shselasky 806321936Shselaskystatic int mlx5_calc_send_wqe(struct mlx5_context *ctx, 807321936Shselasky struct ibv_qp_init_attr_ex *attr, 808321936Shselasky struct mlx5_qp *qp) 809321936Shselasky{ 810321936Shselasky int size; 811321936Shselasky int inl_size = 0; 812321936Shselasky int max_gather; 813321936Shselasky int tot_size; 814321936Shselasky 815321936Shselasky size = sq_overhead(attr->qp_type); 816321936Shselasky if (size < 0) 817321936Shselasky return size; 818321936Shselasky 819321936Shselasky if (attr->cap.max_inline_data) { 820321936Shselasky inl_size = size + align(sizeof(struct mlx5_wqe_inl_data_seg) + 821321936Shselasky attr->cap.max_inline_data, 16); 822321936Shselasky } 823321936Shselasky 824321936Shselasky if (attr->comp_mask & IBV_QP_INIT_ATTR_MAX_TSO_HEADER) { 825321936Shselasky size += align(attr->max_tso_header, 16); 826321936Shselasky qp->max_tso_header = attr->max_tso_header; 827321936Shselasky } 828321936Shselasky 829321936Shselasky max_gather = (ctx->max_sq_desc_sz - size) / 830321936Shselasky sizeof(struct mlx5_wqe_data_seg); 831321936Shselasky if (attr->cap.max_send_sge > max_gather) 832321936Shselasky return -EINVAL; 833321936Shselasky 834321936Shselasky size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); 835321936Shselasky tot_size = max_int(size, inl_size); 836321936Shselasky 837321936Shselasky if (tot_size > ctx->max_sq_desc_sz) 838321936Shselasky return -EINVAL; 839321936Shselasky 840321936Shselasky return align(tot_size, MLX5_SEND_WQE_BB); 841321936Shselasky} 842321936Shselasky 843321936Shselaskystatic int mlx5_calc_rcv_wqe(struct mlx5_context *ctx, 844321936Shselasky struct ibv_qp_init_attr_ex *attr, 845321936Shselasky struct mlx5_qp *qp) 846321936Shselasky{ 847321936Shselasky uint32_t size; 848321936Shselasky int num_scatter; 849321936Shselasky 850321936Shselasky if (attr->srq) 851321936Shselasky return 0; 852321936Shselasky 853321936Shselasky num_scatter = max_t(uint32_t, attr->cap.max_recv_sge, 1); 854321936Shselasky size = sizeof(struct mlx5_wqe_data_seg) * num_scatter; 855321936Shselasky if (qp->wq_sig) 856321936Shselasky size += sizeof(struct mlx5_rwqe_sig); 857321936Shselasky 858321936Shselasky if (size > ctx->max_rq_desc_sz) 859321936Shselasky return -EINVAL; 860321936Shselasky 861321936Shselasky size = mlx5_round_up_power_of_two(size); 862321936Shselasky 863321936Shselasky return size; 864321936Shselasky} 865321936Shselasky 866321936Shselaskystatic int mlx5_calc_sq_size(struct mlx5_context *ctx, 867321936Shselasky struct ibv_qp_init_attr_ex *attr, 868321936Shselasky struct mlx5_qp *qp) 869321936Shselasky{ 870321936Shselasky int wqe_size; 871321936Shselasky int wq_size; 872321936Shselasky FILE *fp = ctx->dbg_fp; 873321936Shselasky 874321936Shselasky if (!attr->cap.max_send_wr) 875321936Shselasky return 0; 876321936Shselasky 877321936Shselasky wqe_size = mlx5_calc_send_wqe(ctx, attr, qp); 878321936Shselasky if (wqe_size < 0) { 879321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 880321936Shselasky return wqe_size; 881321936Shselasky } 882321936Shselasky 883321936Shselasky if (wqe_size > ctx->max_sq_desc_sz) { 884321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 885321936Shselasky return -EINVAL; 886321936Shselasky } 887321936Shselasky 888321936Shselasky qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) - 889321936Shselasky sizeof(struct mlx5_wqe_inl_data_seg); 890321936Shselasky attr->cap.max_inline_data = qp->max_inline_data; 891321936Shselasky 892321936Shselasky /* 893321936Shselasky * to avoid overflow, we limit max_send_wr so 894321936Shselasky * that the multiplication will fit in int 895321936Shselasky */ 896321936Shselasky if (attr->cap.max_send_wr > 0x7fffffff / ctx->max_sq_desc_sz) { 897321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 898321936Shselasky return -EINVAL; 899321936Shselasky } 900321936Shselasky 901321936Shselasky wq_size = mlx5_round_up_power_of_two(attr->cap.max_send_wr * wqe_size); 902321936Shselasky qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; 903321936Shselasky if (qp->sq.wqe_cnt > ctx->max_send_wqebb) { 904321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 905321936Shselasky return -EINVAL; 906321936Shselasky } 907321936Shselasky 908321936Shselasky qp->sq.wqe_shift = mlx5_ilog2(MLX5_SEND_WQE_BB); 909321936Shselasky qp->sq.max_gs = attr->cap.max_send_sge; 910321936Shselasky qp->sq.max_post = wq_size / wqe_size; 911321936Shselasky 912321936Shselasky return wq_size; 913321936Shselasky} 914321936Shselasky 915321936Shselaskystatic int mlx5_calc_rwq_size(struct mlx5_context *ctx, 916321936Shselasky struct mlx5_rwq *rwq, 917321936Shselasky struct ibv_wq_init_attr *attr) 918321936Shselasky{ 919321936Shselasky size_t wqe_size; 920321936Shselasky int wq_size; 921321936Shselasky uint32_t num_scatter; 922321936Shselasky int scat_spc; 923321936Shselasky 924321936Shselasky if (!attr->max_wr) 925321936Shselasky return -EINVAL; 926321936Shselasky 927321936Shselasky /* TBD: check caps for RQ */ 928321936Shselasky num_scatter = max_t(uint32_t, attr->max_sge, 1); 929321936Shselasky wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter; 930321936Shselasky 931321936Shselasky if (rwq->wq_sig) 932321936Shselasky wqe_size += sizeof(struct mlx5_rwqe_sig); 933321936Shselasky 934321936Shselasky if (wqe_size <= 0 || wqe_size > ctx->max_rq_desc_sz) 935321936Shselasky return -EINVAL; 936321936Shselasky 937321936Shselasky wqe_size = mlx5_round_up_power_of_two(wqe_size); 938321936Shselasky wq_size = mlx5_round_up_power_of_two(attr->max_wr) * wqe_size; 939321936Shselasky wq_size = max(wq_size, MLX5_SEND_WQE_BB); 940321936Shselasky rwq->rq.wqe_cnt = wq_size / wqe_size; 941321936Shselasky rwq->rq.wqe_shift = mlx5_ilog2(wqe_size); 942321936Shselasky rwq->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); 943321936Shselasky scat_spc = wqe_size - 944321936Shselasky ((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0); 945321936Shselasky rwq->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); 946321936Shselasky return wq_size; 947321936Shselasky} 948321936Shselasky 949321936Shselaskystatic int mlx5_calc_rq_size(struct mlx5_context *ctx, 950321936Shselasky struct ibv_qp_init_attr_ex *attr, 951321936Shselasky struct mlx5_qp *qp) 952321936Shselasky{ 953321936Shselasky int wqe_size; 954321936Shselasky int wq_size; 955321936Shselasky int scat_spc; 956321936Shselasky FILE *fp = ctx->dbg_fp; 957321936Shselasky 958321936Shselasky if (!attr->cap.max_recv_wr) 959321936Shselasky return 0; 960321936Shselasky 961321936Shselasky if (attr->cap.max_recv_wr > ctx->max_recv_wr) { 962321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 963321936Shselasky return -EINVAL; 964321936Shselasky } 965321936Shselasky 966321936Shselasky wqe_size = mlx5_calc_rcv_wqe(ctx, attr, qp); 967321936Shselasky if (wqe_size < 0 || wqe_size > ctx->max_rq_desc_sz) { 968321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 969321936Shselasky return -EINVAL; 970321936Shselasky } 971321936Shselasky 972321936Shselasky wq_size = mlx5_round_up_power_of_two(attr->cap.max_recv_wr) * wqe_size; 973321936Shselasky if (wqe_size) { 974321936Shselasky wq_size = max(wq_size, MLX5_SEND_WQE_BB); 975321936Shselasky qp->rq.wqe_cnt = wq_size / wqe_size; 976321936Shselasky qp->rq.wqe_shift = mlx5_ilog2(wqe_size); 977321936Shselasky qp->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); 978321936Shselasky scat_spc = wqe_size - 979321936Shselasky (qp->wq_sig ? sizeof(struct mlx5_rwqe_sig) : 0); 980321936Shselasky qp->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); 981321936Shselasky } else { 982321936Shselasky qp->rq.wqe_cnt = 0; 983321936Shselasky qp->rq.wqe_shift = 0; 984321936Shselasky qp->rq.max_post = 0; 985321936Shselasky qp->rq.max_gs = 0; 986321936Shselasky } 987321936Shselasky return wq_size; 988321936Shselasky} 989321936Shselasky 990321936Shselaskystatic int mlx5_calc_wq_size(struct mlx5_context *ctx, 991321936Shselasky struct ibv_qp_init_attr_ex *attr, 992321936Shselasky struct mlx5_qp *qp) 993321936Shselasky{ 994321936Shselasky int ret; 995321936Shselasky int result; 996321936Shselasky 997321936Shselasky ret = mlx5_calc_sq_size(ctx, attr, qp); 998321936Shselasky if (ret < 0) 999321936Shselasky return ret; 1000321936Shselasky 1001321936Shselasky result = ret; 1002321936Shselasky ret = mlx5_calc_rq_size(ctx, attr, qp); 1003321936Shselasky if (ret < 0) 1004321936Shselasky return ret; 1005321936Shselasky 1006321936Shselasky result += ret; 1007321936Shselasky 1008321936Shselasky qp->sq.offset = ret; 1009321936Shselasky qp->rq.offset = 0; 1010321936Shselasky 1011321936Shselasky return result; 1012321936Shselasky} 1013321936Shselasky 1014321936Shselaskystatic void map_uuar(struct ibv_context *context, struct mlx5_qp *qp, 1015321936Shselasky int uuar_index) 1016321936Shselasky{ 1017321936Shselasky struct mlx5_context *ctx = to_mctx(context); 1018321936Shselasky 1019321936Shselasky qp->bf = &ctx->bfs[uuar_index]; 1020321936Shselasky} 1021321936Shselasky 1022321936Shselaskystatic const char *qptype2key(enum ibv_qp_type type) 1023321936Shselasky{ 1024321936Shselasky switch (type) { 1025321936Shselasky case IBV_QPT_RC: return "HUGE_RC"; 1026321936Shselasky case IBV_QPT_UC: return "HUGE_UC"; 1027321936Shselasky case IBV_QPT_UD: return "HUGE_UD"; 1028321936Shselasky case IBV_QPT_RAW_PACKET: return "HUGE_RAW_ETH"; 1029321936Shselasky default: return "HUGE_NA"; 1030321936Shselasky } 1031321936Shselasky} 1032321936Shselasky 1033321936Shselaskystatic int mlx5_alloc_qp_buf(struct ibv_context *context, 1034321936Shselasky struct ibv_qp_init_attr_ex *attr, 1035321936Shselasky struct mlx5_qp *qp, 1036321936Shselasky int size) 1037321936Shselasky{ 1038321936Shselasky int err; 1039321936Shselasky enum mlx5_alloc_type alloc_type; 1040321936Shselasky enum mlx5_alloc_type default_alloc_type = MLX5_ALLOC_TYPE_ANON; 1041321936Shselasky const char *qp_huge_key; 1042321936Shselasky 1043321936Shselasky if (qp->sq.wqe_cnt) { 1044321936Shselasky qp->sq.wrid = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid)); 1045321936Shselasky if (!qp->sq.wrid) { 1046321936Shselasky errno = ENOMEM; 1047321936Shselasky err = -1; 1048321936Shselasky return err; 1049321936Shselasky } 1050321936Shselasky 1051321936Shselasky qp->sq.wr_data = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data)); 1052321936Shselasky if (!qp->sq.wr_data) { 1053321936Shselasky errno = ENOMEM; 1054321936Shselasky err = -1; 1055321936Shselasky goto ex_wrid; 1056321936Shselasky } 1057321936Shselasky } 1058321936Shselasky 1059321936Shselasky qp->sq.wqe_head = malloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head)); 1060321936Shselasky if (!qp->sq.wqe_head) { 1061321936Shselasky errno = ENOMEM; 1062321936Shselasky err = -1; 1063321936Shselasky goto ex_wrid; 1064321936Shselasky } 1065321936Shselasky 1066321936Shselasky if (qp->rq.wqe_cnt) { 1067321936Shselasky qp->rq.wrid = malloc(qp->rq.wqe_cnt * sizeof(uint64_t)); 1068321936Shselasky if (!qp->rq.wrid) { 1069321936Shselasky errno = ENOMEM; 1070321936Shselasky err = -1; 1071321936Shselasky goto ex_wrid; 1072321936Shselasky } 1073321936Shselasky } 1074321936Shselasky 1075321936Shselasky /* compatibility support */ 1076321936Shselasky qp_huge_key = qptype2key(qp->ibv_qp->qp_type); 1077321936Shselasky if (mlx5_use_huge(qp_huge_key)) 1078321936Shselasky default_alloc_type = MLX5_ALLOC_TYPE_HUGE; 1079321936Shselasky 1080321936Shselasky mlx5_get_alloc_type(MLX5_QP_PREFIX, &alloc_type, 1081321936Shselasky default_alloc_type); 1082321936Shselasky 1083321936Shselasky err = mlx5_alloc_prefered_buf(to_mctx(context), &qp->buf, 1084321936Shselasky align(qp->buf_size, to_mdev 1085321936Shselasky (context->device)->page_size), 1086321936Shselasky to_mdev(context->device)->page_size, 1087321936Shselasky alloc_type, 1088321936Shselasky MLX5_QP_PREFIX); 1089321936Shselasky 1090321936Shselasky if (err) { 1091321936Shselasky err = -ENOMEM; 1092321936Shselasky goto ex_wrid; 1093321936Shselasky } 1094321936Shselasky 1095321936Shselasky memset(qp->buf.buf, 0, qp->buf_size); 1096321936Shselasky 1097321936Shselasky if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1098321936Shselasky size_t aligned_sq_buf_size = align(qp->sq_buf_size, 1099321936Shselasky to_mdev(context->device)->page_size); 1100321936Shselasky /* For Raw Packet QP, allocate a separate buffer for the SQ */ 1101321936Shselasky err = mlx5_alloc_prefered_buf(to_mctx(context), &qp->sq_buf, 1102321936Shselasky aligned_sq_buf_size, 1103321936Shselasky to_mdev(context->device)->page_size, 1104321936Shselasky alloc_type, 1105321936Shselasky MLX5_QP_PREFIX); 1106321936Shselasky if (err) { 1107321936Shselasky err = -ENOMEM; 1108321936Shselasky goto rq_buf; 1109321936Shselasky } 1110321936Shselasky 1111321936Shselasky memset(qp->sq_buf.buf, 0, aligned_sq_buf_size); 1112321936Shselasky } 1113321936Shselasky 1114321936Shselasky return 0; 1115321936Shselaskyrq_buf: 1116321936Shselasky mlx5_free_actual_buf(to_mctx(qp->verbs_qp.qp.context), &qp->buf); 1117321936Shselaskyex_wrid: 1118321936Shselasky if (qp->rq.wrid) 1119321936Shselasky free(qp->rq.wrid); 1120321936Shselasky 1121321936Shselasky if (qp->sq.wqe_head) 1122321936Shselasky free(qp->sq.wqe_head); 1123321936Shselasky 1124321936Shselasky if (qp->sq.wr_data) 1125321936Shselasky free(qp->sq.wr_data); 1126321936Shselasky if (qp->sq.wrid) 1127321936Shselasky free(qp->sq.wrid); 1128321936Shselasky 1129321936Shselasky return err; 1130321936Shselasky} 1131321936Shselasky 1132321936Shselaskystatic void mlx5_free_qp_buf(struct mlx5_qp *qp) 1133321936Shselasky{ 1134321936Shselasky struct mlx5_context *ctx = to_mctx(qp->ibv_qp->context); 1135321936Shselasky 1136321936Shselasky mlx5_free_actual_buf(ctx, &qp->buf); 1137321936Shselasky 1138321936Shselasky if (qp->sq_buf.buf) 1139321936Shselasky mlx5_free_actual_buf(ctx, &qp->sq_buf); 1140321936Shselasky 1141321936Shselasky if (qp->rq.wrid) 1142321936Shselasky free(qp->rq.wrid); 1143321936Shselasky 1144321936Shselasky if (qp->sq.wqe_head) 1145321936Shselasky free(qp->sq.wqe_head); 1146321936Shselasky 1147321936Shselasky if (qp->sq.wrid) 1148321936Shselasky free(qp->sq.wrid); 1149321936Shselasky 1150321936Shselasky if (qp->sq.wr_data) 1151321936Shselasky free(qp->sq.wr_data); 1152321936Shselasky} 1153321936Shselasky 1154321936Shselaskystatic int mlx5_cmd_create_rss_qp(struct ibv_context *context, 1155321936Shselasky struct ibv_qp_init_attr_ex *attr, 1156321936Shselasky struct mlx5_qp *qp) 1157321936Shselasky{ 1158321936Shselasky struct mlx5_create_qp_ex_rss cmd_ex_rss = {}; 1159321936Shselasky struct mlx5_create_qp_resp_ex resp = {}; 1160321936Shselasky int ret; 1161321936Shselasky 1162321936Shselasky if (attr->rx_hash_conf.rx_hash_key_len > sizeof(cmd_ex_rss.rx_hash_key)) { 1163321936Shselasky errno = EINVAL; 1164321936Shselasky return errno; 1165321936Shselasky } 1166321936Shselasky 1167321936Shselasky cmd_ex_rss.rx_hash_fields_mask = attr->rx_hash_conf.rx_hash_fields_mask; 1168321936Shselasky cmd_ex_rss.rx_hash_function = attr->rx_hash_conf.rx_hash_function; 1169321936Shselasky cmd_ex_rss.rx_key_len = attr->rx_hash_conf.rx_hash_key_len; 1170321936Shselasky memcpy(cmd_ex_rss.rx_hash_key, attr->rx_hash_conf.rx_hash_key, 1171321936Shselasky attr->rx_hash_conf.rx_hash_key_len); 1172321936Shselasky 1173321936Shselasky ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, 1174321936Shselasky sizeof(qp->verbs_qp), attr, 1175321936Shselasky &cmd_ex_rss.ibv_cmd, sizeof(cmd_ex_rss.ibv_cmd), 1176321936Shselasky sizeof(cmd_ex_rss), &resp.ibv_resp, 1177321936Shselasky sizeof(resp.ibv_resp), sizeof(resp)); 1178321936Shselasky if (ret) 1179321936Shselasky return ret; 1180321936Shselasky 1181321936Shselasky qp->rss_qp = 1; 1182321936Shselasky return 0; 1183321936Shselasky} 1184321936Shselasky 1185321936Shselaskystatic int mlx5_cmd_create_qp_ex(struct ibv_context *context, 1186321936Shselasky struct ibv_qp_init_attr_ex *attr, 1187321936Shselasky struct mlx5_create_qp *cmd, 1188321936Shselasky struct mlx5_qp *qp, 1189321936Shselasky struct mlx5_create_qp_resp_ex *resp) 1190321936Shselasky{ 1191321936Shselasky struct mlx5_create_qp_ex cmd_ex; 1192321936Shselasky int ret; 1193321936Shselasky 1194321936Shselasky memset(&cmd_ex, 0, sizeof(cmd_ex)); 1195321936Shselasky memcpy(&cmd_ex.ibv_cmd.base, &cmd->ibv_cmd.user_handle, 1196321936Shselasky offsetof(typeof(cmd->ibv_cmd), is_srq) + 1197321936Shselasky sizeof(cmd->ibv_cmd.is_srq) - 1198321936Shselasky offsetof(typeof(cmd->ibv_cmd), user_handle)); 1199321936Shselasky 1200321936Shselasky memcpy(&cmd_ex.drv_ex, &cmd->buf_addr, 1201321936Shselasky offsetof(typeof(*cmd), sq_buf_addr) + 1202321936Shselasky sizeof(cmd->sq_buf_addr) - sizeof(cmd->ibv_cmd)); 1203321936Shselasky 1204321936Shselasky ret = ibv_cmd_create_qp_ex2(context, &qp->verbs_qp, 1205321936Shselasky sizeof(qp->verbs_qp), attr, 1206321936Shselasky &cmd_ex.ibv_cmd, sizeof(cmd_ex.ibv_cmd), 1207321936Shselasky sizeof(cmd_ex), &resp->ibv_resp, 1208321936Shselasky sizeof(resp->ibv_resp), sizeof(*resp)); 1209321936Shselasky 1210321936Shselasky return ret; 1211321936Shselasky} 1212321936Shselasky 1213321936Shselaskyenum { 1214321936Shselasky MLX5_CREATE_QP_SUP_COMP_MASK = (IBV_QP_INIT_ATTR_PD | 1215321936Shselasky IBV_QP_INIT_ATTR_XRCD | 1216321936Shselasky IBV_QP_INIT_ATTR_CREATE_FLAGS | 1217321936Shselasky IBV_QP_INIT_ATTR_MAX_TSO_HEADER | 1218321936Shselasky IBV_QP_INIT_ATTR_IND_TABLE | 1219321936Shselasky IBV_QP_INIT_ATTR_RX_HASH), 1220321936Shselasky}; 1221321936Shselasky 1222321936Shselaskyenum { 1223321936Shselasky MLX5_CREATE_QP_EX2_COMP_MASK = (IBV_QP_INIT_ATTR_CREATE_FLAGS | 1224321936Shselasky IBV_QP_INIT_ATTR_MAX_TSO_HEADER | 1225321936Shselasky IBV_QP_INIT_ATTR_IND_TABLE | 1226321936Shselasky IBV_QP_INIT_ATTR_RX_HASH), 1227321936Shselasky}; 1228321936Shselasky 1229321936Shselaskystatic struct ibv_qp *create_qp(struct ibv_context *context, 1230321936Shselasky struct ibv_qp_init_attr_ex *attr) 1231321936Shselasky{ 1232321936Shselasky struct mlx5_create_qp cmd; 1233321936Shselasky struct mlx5_create_qp_resp resp; 1234321936Shselasky struct mlx5_create_qp_resp_ex resp_ex; 1235321936Shselasky struct mlx5_qp *qp; 1236321936Shselasky int ret; 1237321936Shselasky struct mlx5_context *ctx = to_mctx(context); 1238321936Shselasky struct ibv_qp *ibqp; 1239321936Shselasky int32_t usr_idx = 0; 1240321936Shselasky uint32_t uuar_index; 1241321936Shselasky FILE *fp = ctx->dbg_fp; 1242321936Shselasky 1243321936Shselasky if (attr->comp_mask & ~MLX5_CREATE_QP_SUP_COMP_MASK) 1244321936Shselasky return NULL; 1245321936Shselasky 1246321936Shselasky if ((attr->comp_mask & IBV_QP_INIT_ATTR_MAX_TSO_HEADER) && 1247321936Shselasky (attr->qp_type != IBV_QPT_RAW_PACKET)) 1248321936Shselasky return NULL; 1249321936Shselasky 1250321936Shselasky qp = calloc(1, sizeof(*qp)); 1251321936Shselasky if (!qp) { 1252321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1253321936Shselasky return NULL; 1254321936Shselasky } 1255321936Shselasky ibqp = (struct ibv_qp *)&qp->verbs_qp; 1256321936Shselasky qp->ibv_qp = ibqp; 1257321936Shselasky 1258321936Shselasky memset(&cmd, 0, sizeof(cmd)); 1259321936Shselasky memset(&resp, 0, sizeof(resp)); 1260321936Shselasky memset(&resp_ex, 0, sizeof(resp_ex)); 1261321936Shselasky 1262321936Shselasky if (attr->comp_mask & IBV_QP_INIT_ATTR_RX_HASH) { 1263321936Shselasky ret = mlx5_cmd_create_rss_qp(context, attr, qp); 1264321936Shselasky if (ret) 1265321936Shselasky goto err; 1266321936Shselasky 1267321936Shselasky return ibqp; 1268321936Shselasky } 1269321936Shselasky 1270321936Shselasky qp->wq_sig = qp_sig_enabled(); 1271321936Shselasky if (qp->wq_sig) 1272321936Shselasky cmd.flags |= MLX5_QP_FLAG_SIGNATURE; 1273321936Shselasky 1274321936Shselasky if (use_scatter_to_cqe()) 1275321936Shselasky cmd.flags |= MLX5_QP_FLAG_SCATTER_CQE; 1276321936Shselasky 1277321936Shselasky ret = mlx5_calc_wq_size(ctx, attr, qp); 1278321936Shselasky if (ret < 0) { 1279321936Shselasky errno = -ret; 1280321936Shselasky goto err; 1281321936Shselasky } 1282321936Shselasky 1283321936Shselasky if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1284321936Shselasky qp->buf_size = qp->sq.offset; 1285321936Shselasky qp->sq_buf_size = ret - qp->buf_size; 1286321936Shselasky qp->sq.offset = 0; 1287321936Shselasky } else { 1288321936Shselasky qp->buf_size = ret; 1289321936Shselasky qp->sq_buf_size = 0; 1290321936Shselasky } 1291321936Shselasky 1292321936Shselasky if (mlx5_alloc_qp_buf(context, attr, qp, ret)) { 1293321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1294321936Shselasky goto err; 1295321936Shselasky } 1296321936Shselasky 1297321936Shselasky if (attr->qp_type == IBV_QPT_RAW_PACKET) { 1298321936Shselasky qp->sq_start = qp->sq_buf.buf; 1299321936Shselasky qp->sq.qend = qp->sq_buf.buf + 1300321936Shselasky (qp->sq.wqe_cnt << qp->sq.wqe_shift); 1301321936Shselasky } else { 1302321936Shselasky qp->sq_start = qp->buf.buf + qp->sq.offset; 1303321936Shselasky qp->sq.qend = qp->buf.buf + qp->sq.offset + 1304321936Shselasky (qp->sq.wqe_cnt << qp->sq.wqe_shift); 1305321936Shselasky } 1306321936Shselasky 1307321936Shselasky mlx5_init_qp_indices(qp); 1308321936Shselasky 1309321936Shselasky if (mlx5_spinlock_init(&qp->sq.lock) || 1310321936Shselasky mlx5_spinlock_init(&qp->rq.lock)) 1311321936Shselasky goto err_free_qp_buf; 1312321936Shselasky 1313321936Shselasky qp->db = mlx5_alloc_dbrec(ctx); 1314321936Shselasky if (!qp->db) { 1315321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "\n"); 1316321936Shselasky goto err_free_qp_buf; 1317321936Shselasky } 1318321936Shselasky 1319321936Shselasky qp->db[MLX5_RCV_DBR] = 0; 1320321936Shselasky qp->db[MLX5_SND_DBR] = 0; 1321321936Shselasky 1322321936Shselasky cmd.buf_addr = (uintptr_t) qp->buf.buf; 1323321936Shselasky cmd.sq_buf_addr = (attr->qp_type == IBV_QPT_RAW_PACKET) ? 1324321936Shselasky (uintptr_t) qp->sq_buf.buf : 0; 1325321936Shselasky cmd.db_addr = (uintptr_t) qp->db; 1326321936Shselasky cmd.sq_wqe_count = qp->sq.wqe_cnt; 1327321936Shselasky cmd.rq_wqe_count = qp->rq.wqe_cnt; 1328321936Shselasky cmd.rq_wqe_shift = qp->rq.wqe_shift; 1329321936Shselasky 1330321936Shselasky if (ctx->atomic_cap == IBV_ATOMIC_HCA) 1331321936Shselasky qp->atomics_enabled = 1; 1332321936Shselasky 1333321936Shselasky if (!ctx->cqe_version) { 1334321936Shselasky cmd.uidx = 0xffffff; 1335321936Shselasky pthread_mutex_lock(&ctx->qp_table_mutex); 1336321936Shselasky } else if (!is_xrc_tgt(attr->qp_type)) { 1337321936Shselasky usr_idx = mlx5_store_uidx(ctx, qp); 1338321936Shselasky if (usr_idx < 0) { 1339321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 1340321936Shselasky goto err_rq_db; 1341321936Shselasky } 1342321936Shselasky 1343321936Shselasky cmd.uidx = usr_idx; 1344321936Shselasky } 1345321936Shselasky 1346321936Shselasky if (attr->comp_mask & MLX5_CREATE_QP_EX2_COMP_MASK) 1347321936Shselasky ret = mlx5_cmd_create_qp_ex(context, attr, &cmd, qp, &resp_ex); 1348321936Shselasky else 1349321936Shselasky ret = ibv_cmd_create_qp_ex(context, &qp->verbs_qp, sizeof(qp->verbs_qp), 1350321936Shselasky attr, &cmd.ibv_cmd, sizeof(cmd), 1351321936Shselasky &resp.ibv_resp, sizeof(resp)); 1352321936Shselasky if (ret) { 1353321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1354321936Shselasky goto err_free_uidx; 1355321936Shselasky } 1356321936Shselasky 1357321936Shselasky uuar_index = (attr->comp_mask & MLX5_CREATE_QP_EX2_COMP_MASK) ? 1358321936Shselasky resp_ex.uuar_index : resp.uuar_index; 1359321936Shselasky if (!ctx->cqe_version) { 1360321936Shselasky if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) { 1361321936Shselasky ret = mlx5_store_qp(ctx, ibqp->qp_num, qp); 1362321936Shselasky if (ret) { 1363321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "ret %d\n", ret); 1364321936Shselasky goto err_destroy; 1365321936Shselasky } 1366321936Shselasky } 1367321936Shselasky 1368321936Shselasky pthread_mutex_unlock(&ctx->qp_table_mutex); 1369321936Shselasky } 1370321936Shselasky 1371321936Shselasky map_uuar(context, qp, uuar_index); 1372321936Shselasky 1373321936Shselasky qp->rq.max_post = qp->rq.wqe_cnt; 1374321936Shselasky if (attr->sq_sig_all) 1375321936Shselasky qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; 1376321936Shselasky else 1377321936Shselasky qp->sq_signal_bits = 0; 1378321936Shselasky 1379321936Shselasky attr->cap.max_send_wr = qp->sq.max_post; 1380321936Shselasky attr->cap.max_recv_wr = qp->rq.max_post; 1381321936Shselasky attr->cap.max_recv_sge = qp->rq.max_gs; 1382321936Shselasky 1383321936Shselasky qp->rsc.type = MLX5_RSC_TYPE_QP; 1384321936Shselasky qp->rsc.rsn = (ctx->cqe_version && !is_xrc_tgt(attr->qp_type)) ? 1385321936Shselasky usr_idx : ibqp->qp_num; 1386321936Shselasky 1387321936Shselasky return ibqp; 1388321936Shselasky 1389321936Shselaskyerr_destroy: 1390321936Shselasky ibv_cmd_destroy_qp(ibqp); 1391321936Shselasky 1392321936Shselaskyerr_free_uidx: 1393321936Shselasky if (!ctx->cqe_version) 1394321936Shselasky pthread_mutex_unlock(&to_mctx(context)->qp_table_mutex); 1395321936Shselasky else if (!is_xrc_tgt(attr->qp_type)) 1396321936Shselasky mlx5_clear_uidx(ctx, usr_idx); 1397321936Shselasky 1398321936Shselaskyerr_rq_db: 1399321936Shselasky mlx5_free_db(to_mctx(context), qp->db); 1400321936Shselasky 1401321936Shselaskyerr_free_qp_buf: 1402321936Shselasky mlx5_free_qp_buf(qp); 1403321936Shselasky 1404321936Shselaskyerr: 1405321936Shselasky free(qp); 1406321936Shselasky 1407321936Shselasky return NULL; 1408321936Shselasky} 1409321936Shselasky 1410321936Shselaskystruct ibv_qp *mlx5_create_qp(struct ibv_pd *pd, 1411321936Shselasky struct ibv_qp_init_attr *attr) 1412321936Shselasky{ 1413321936Shselasky struct ibv_qp *qp; 1414321936Shselasky struct ibv_qp_init_attr_ex attrx; 1415321936Shselasky 1416321936Shselasky memset(&attrx, 0, sizeof(attrx)); 1417321936Shselasky memcpy(&attrx, attr, sizeof(*attr)); 1418321936Shselasky attrx.comp_mask = IBV_QP_INIT_ATTR_PD; 1419321936Shselasky attrx.pd = pd; 1420321936Shselasky qp = create_qp(pd->context, &attrx); 1421321936Shselasky if (qp) 1422321936Shselasky memcpy(attr, &attrx, sizeof(*attr)); 1423321936Shselasky 1424321936Shselasky return qp; 1425321936Shselasky} 1426321936Shselasky 1427321936Shselaskystatic void mlx5_lock_cqs(struct ibv_qp *qp) 1428321936Shselasky{ 1429321936Shselasky struct mlx5_cq *send_cq = to_mcq(qp->send_cq); 1430321936Shselasky struct mlx5_cq *recv_cq = to_mcq(qp->recv_cq); 1431321936Shselasky 1432321936Shselasky if (send_cq && recv_cq) { 1433321936Shselasky if (send_cq == recv_cq) { 1434321936Shselasky mlx5_spin_lock(&send_cq->lock); 1435321936Shselasky } else if (send_cq->cqn < recv_cq->cqn) { 1436321936Shselasky mlx5_spin_lock(&send_cq->lock); 1437321936Shselasky mlx5_spin_lock(&recv_cq->lock); 1438321936Shselasky } else { 1439321936Shselasky mlx5_spin_lock(&recv_cq->lock); 1440321936Shselasky mlx5_spin_lock(&send_cq->lock); 1441321936Shselasky } 1442321936Shselasky } else if (send_cq) { 1443321936Shselasky mlx5_spin_lock(&send_cq->lock); 1444321936Shselasky } else if (recv_cq) { 1445321936Shselasky mlx5_spin_lock(&recv_cq->lock); 1446321936Shselasky } 1447321936Shselasky} 1448321936Shselasky 1449321936Shselaskystatic void mlx5_unlock_cqs(struct ibv_qp *qp) 1450321936Shselasky{ 1451321936Shselasky struct mlx5_cq *send_cq = to_mcq(qp->send_cq); 1452321936Shselasky struct mlx5_cq *recv_cq = to_mcq(qp->recv_cq); 1453321936Shselasky 1454321936Shselasky if (send_cq && recv_cq) { 1455321936Shselasky if (send_cq == recv_cq) { 1456321936Shselasky mlx5_spin_unlock(&send_cq->lock); 1457321936Shselasky } else if (send_cq->cqn < recv_cq->cqn) { 1458321936Shselasky mlx5_spin_unlock(&recv_cq->lock); 1459321936Shselasky mlx5_spin_unlock(&send_cq->lock); 1460321936Shselasky } else { 1461321936Shselasky mlx5_spin_unlock(&send_cq->lock); 1462321936Shselasky mlx5_spin_unlock(&recv_cq->lock); 1463321936Shselasky } 1464321936Shselasky } else if (send_cq) { 1465321936Shselasky mlx5_spin_unlock(&send_cq->lock); 1466321936Shselasky } else if (recv_cq) { 1467321936Shselasky mlx5_spin_unlock(&recv_cq->lock); 1468321936Shselasky } 1469321936Shselasky} 1470321936Shselasky 1471321936Shselaskyint mlx5_destroy_qp(struct ibv_qp *ibqp) 1472321936Shselasky{ 1473321936Shselasky struct mlx5_qp *qp = to_mqp(ibqp); 1474321936Shselasky struct mlx5_context *ctx = to_mctx(ibqp->context); 1475321936Shselasky int ret; 1476321936Shselasky 1477321936Shselasky if (qp->rss_qp) { 1478321936Shselasky ret = ibv_cmd_destroy_qp(ibqp); 1479321936Shselasky if (ret) 1480321936Shselasky return ret; 1481321936Shselasky goto free; 1482321936Shselasky } 1483321936Shselasky 1484321936Shselasky if (!ctx->cqe_version) 1485321936Shselasky pthread_mutex_lock(&ctx->qp_table_mutex); 1486321936Shselasky 1487321936Shselasky ret = ibv_cmd_destroy_qp(ibqp); 1488321936Shselasky if (ret) { 1489321936Shselasky if (!ctx->cqe_version) 1490321936Shselasky pthread_mutex_unlock(&ctx->qp_table_mutex); 1491321936Shselasky return ret; 1492321936Shselasky } 1493321936Shselasky 1494321936Shselasky mlx5_lock_cqs(ibqp); 1495321936Shselasky 1496321936Shselasky __mlx5_cq_clean(to_mcq(ibqp->recv_cq), qp->rsc.rsn, 1497321936Shselasky ibqp->srq ? to_msrq(ibqp->srq) : NULL); 1498321936Shselasky if (ibqp->send_cq != ibqp->recv_cq) 1499321936Shselasky __mlx5_cq_clean(to_mcq(ibqp->send_cq), qp->rsc.rsn, NULL); 1500321936Shselasky 1501321936Shselasky if (!ctx->cqe_version) { 1502321936Shselasky if (qp->sq.wqe_cnt || qp->rq.wqe_cnt) 1503321936Shselasky mlx5_clear_qp(ctx, ibqp->qp_num); 1504321936Shselasky } 1505321936Shselasky 1506321936Shselasky mlx5_unlock_cqs(ibqp); 1507321936Shselasky if (!ctx->cqe_version) 1508321936Shselasky pthread_mutex_unlock(&ctx->qp_table_mutex); 1509321936Shselasky else if (!is_xrc_tgt(ibqp->qp_type)) 1510321936Shselasky mlx5_clear_uidx(ctx, qp->rsc.rsn); 1511321936Shselasky 1512321936Shselasky mlx5_free_db(ctx, qp->db); 1513321936Shselasky mlx5_free_qp_buf(qp); 1514321936Shselaskyfree: 1515321936Shselasky free(qp); 1516321936Shselasky 1517321936Shselasky return 0; 1518321936Shselasky} 1519321936Shselasky 1520321936Shselaskyint mlx5_query_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, 1521321936Shselasky int attr_mask, struct ibv_qp_init_attr *init_attr) 1522321936Shselasky{ 1523321936Shselasky struct ibv_query_qp cmd; 1524321936Shselasky struct mlx5_qp *qp = to_mqp(ibqp); 1525321936Shselasky int ret; 1526321936Shselasky 1527321936Shselasky if (qp->rss_qp) 1528321936Shselasky return ENOSYS; 1529321936Shselasky 1530321936Shselasky ret = ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); 1531321936Shselasky if (ret) 1532321936Shselasky return ret; 1533321936Shselasky 1534321936Shselasky init_attr->cap.max_send_wr = qp->sq.max_post; 1535321936Shselasky init_attr->cap.max_send_sge = qp->sq.max_gs; 1536321936Shselasky init_attr->cap.max_inline_data = qp->max_inline_data; 1537321936Shselasky 1538321936Shselasky attr->cap = init_attr->cap; 1539321936Shselasky 1540321936Shselasky return 0; 1541321936Shselasky} 1542321936Shselasky 1543321936Shselaskyenum { 1544321936Shselasky MLX5_MODIFY_QP_EX_ATTR_MASK = IBV_QP_RATE_LIMIT, 1545321936Shselasky}; 1546321936Shselasky 1547321936Shselaskyint mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1548321936Shselasky int attr_mask) 1549321936Shselasky{ 1550321936Shselasky struct ibv_modify_qp cmd = {}; 1551321936Shselasky struct ibv_modify_qp_ex cmd_ex = {}; 1552321936Shselasky struct ibv_modify_qp_resp_ex resp = {}; 1553321936Shselasky struct mlx5_qp *mqp = to_mqp(qp); 1554321936Shselasky struct mlx5_context *context = to_mctx(qp->context); 1555321936Shselasky int ret; 1556321936Shselasky uint32_t *db; 1557321936Shselasky 1558321936Shselasky if (mqp->rss_qp) 1559321936Shselasky return ENOSYS; 1560321936Shselasky 1561321936Shselasky if (attr_mask & IBV_QP_PORT) { 1562321936Shselasky switch (qp->qp_type) { 1563321936Shselasky case IBV_QPT_RAW_PACKET: 1564321936Shselasky if (context->cached_link_layer[attr->port_num - 1] == 1565321936Shselasky IBV_LINK_LAYER_ETHERNET) { 1566321936Shselasky if (context->cached_device_cap_flags & 1567321936Shselasky IBV_DEVICE_RAW_IP_CSUM) 1568321936Shselasky mqp->qp_cap_cache |= 1569321936Shselasky MLX5_CSUM_SUPPORT_RAW_OVER_ETH | 1570321936Shselasky MLX5_RX_CSUM_VALID; 1571321936Shselasky 1572321936Shselasky if (ibv_is_qpt_supported( 1573321936Shselasky context->cached_tso_caps.supported_qpts, 1574321936Shselasky IBV_QPT_RAW_PACKET)) 1575321936Shselasky mqp->max_tso = 1576321936Shselasky context->cached_tso_caps.max_tso; 1577321936Shselasky } 1578321936Shselasky break; 1579321936Shselasky default: 1580321936Shselasky break; 1581321936Shselasky } 1582321936Shselasky } 1583321936Shselasky 1584321936Shselasky if (attr_mask & MLX5_MODIFY_QP_EX_ATTR_MASK) 1585321936Shselasky ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, 1586321936Shselasky &cmd_ex, 1587321936Shselasky sizeof(cmd_ex), sizeof(cmd_ex), 1588321936Shselasky &resp, 1589321936Shselasky sizeof(resp), sizeof(resp)); 1590321936Shselasky else 1591321936Shselasky ret = ibv_cmd_modify_qp(qp, attr, attr_mask, 1592321936Shselasky &cmd, sizeof(cmd)); 1593321936Shselasky 1594321936Shselasky if (!ret && 1595321936Shselasky (attr_mask & IBV_QP_STATE) && 1596321936Shselasky attr->qp_state == IBV_QPS_RESET) { 1597321936Shselasky if (qp->recv_cq) { 1598321936Shselasky mlx5_cq_clean(to_mcq(qp->recv_cq), mqp->rsc.rsn, 1599321936Shselasky qp->srq ? to_msrq(qp->srq) : NULL); 1600321936Shselasky } 1601321936Shselasky if (qp->send_cq != qp->recv_cq && qp->send_cq) 1602321936Shselasky mlx5_cq_clean(to_mcq(qp->send_cq), 1603321936Shselasky to_mqp(qp)->rsc.rsn, NULL); 1604321936Shselasky 1605321936Shselasky mlx5_init_qp_indices(mqp); 1606321936Shselasky db = mqp->db; 1607321936Shselasky db[MLX5_RCV_DBR] = 0; 1608321936Shselasky db[MLX5_SND_DBR] = 0; 1609321936Shselasky } 1610321936Shselasky 1611321936Shselasky /* 1612321936Shselasky * When the Raw Packet QP is in INIT state, its RQ 1613321936Shselasky * underneath is already in RDY, which means it can 1614321936Shselasky * receive packets. According to the IB spec, a QP can't 1615321936Shselasky * receive packets until moved to RTR state. To achieve this, 1616321936Shselasky * for Raw Packet QPs, we update the doorbell record 1617321936Shselasky * once the QP is moved to RTR. 1618321936Shselasky */ 1619321936Shselasky if (!ret && 1620321936Shselasky (attr_mask & IBV_QP_STATE) && 1621321936Shselasky attr->qp_state == IBV_QPS_RTR && 1622321936Shselasky qp->qp_type == IBV_QPT_RAW_PACKET) { 1623321936Shselasky mlx5_spin_lock(&mqp->rq.lock); 1624321936Shselasky mqp->db[MLX5_RCV_DBR] = htobe32(mqp->rq.head & 0xffff); 1625321936Shselasky mlx5_spin_unlock(&mqp->rq.lock); 1626321936Shselasky } 1627321936Shselasky 1628321936Shselasky return ret; 1629321936Shselasky} 1630321936Shselasky 1631321936Shselasky#define RROCE_UDP_SPORT_MIN 0xC000 1632321936Shselasky#define RROCE_UDP_SPORT_MAX 0xFFFF 1633321936Shselaskystruct ibv_ah *mlx5_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 1634321936Shselasky{ 1635321936Shselasky struct mlx5_context *ctx = to_mctx(pd->context); 1636321936Shselasky struct ibv_port_attr port_attr; 1637321936Shselasky struct mlx5_ah *ah; 1638321936Shselasky uint32_t gid_type; 1639321936Shselasky uint32_t tmp; 1640321936Shselasky uint8_t grh; 1641321936Shselasky int is_eth; 1642321936Shselasky 1643321936Shselasky if (attr->port_num < 1 || attr->port_num > ctx->num_ports) 1644321936Shselasky return NULL; 1645321936Shselasky 1646321936Shselasky if (ctx->cached_link_layer[attr->port_num - 1]) { 1647321936Shselasky is_eth = ctx->cached_link_layer[attr->port_num - 1] == 1648321936Shselasky IBV_LINK_LAYER_ETHERNET; 1649321936Shselasky } else { 1650321936Shselasky if (ibv_query_port(pd->context, attr->port_num, &port_attr)) 1651321936Shselasky return NULL; 1652321936Shselasky 1653321936Shselasky is_eth = (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET); 1654321936Shselasky } 1655321936Shselasky 1656321936Shselasky if (unlikely((!attr->is_global) && is_eth)) { 1657321936Shselasky errno = EINVAL; 1658321936Shselasky return NULL; 1659321936Shselasky } 1660321936Shselasky 1661321936Shselasky ah = calloc(1, sizeof *ah); 1662321936Shselasky if (!ah) 1663321936Shselasky return NULL; 1664321936Shselasky 1665321936Shselasky if (is_eth) { 1666321936Shselasky if (ibv_query_gid_type(pd->context, attr->port_num, 1667321936Shselasky attr->grh.sgid_index, &gid_type)) 1668321936Shselasky goto err; 1669321936Shselasky 1670321936Shselasky if (gid_type == IBV_GID_TYPE_ROCE_V2) 1671321936Shselasky ah->av.rlid = htobe16(rand() % (RROCE_UDP_SPORT_MAX + 1 1672321936Shselasky - RROCE_UDP_SPORT_MIN) 1673321936Shselasky + RROCE_UDP_SPORT_MIN); 1674321936Shselasky /* Since RoCE packets must contain GRH, this bit is reserved 1675321936Shselasky * for RoCE and shouldn't be set. 1676321936Shselasky */ 1677321936Shselasky grh = 0; 1678321936Shselasky } else { 1679321936Shselasky ah->av.fl_mlid = attr->src_path_bits & 0x7f; 1680321936Shselasky ah->av.rlid = htobe16(attr->dlid); 1681321936Shselasky grh = 1; 1682321936Shselasky } 1683321936Shselasky ah->av.stat_rate_sl = (attr->static_rate << 4) | attr->sl; 1684321936Shselasky if (attr->is_global) { 1685321936Shselasky ah->av.tclass = attr->grh.traffic_class; 1686321936Shselasky ah->av.hop_limit = attr->grh.hop_limit; 1687321936Shselasky tmp = htobe32((grh << 30) | 1688321936Shselasky ((attr->grh.sgid_index & 0xff) << 20) | 1689321936Shselasky (attr->grh.flow_label & 0xfffff)); 1690321936Shselasky ah->av.grh_gid_fl = tmp; 1691321936Shselasky memcpy(ah->av.rgid, attr->grh.dgid.raw, 16); 1692321936Shselasky } 1693321936Shselasky 1694321936Shselasky if (is_eth) { 1695321936Shselasky if (ctx->cmds_supp_uhw & MLX5_USER_CMDS_SUPP_UHW_CREATE_AH) { 1696321936Shselasky struct mlx5_create_ah_resp resp = {}; 1697321936Shselasky 1698321936Shselasky if (ibv_cmd_create_ah(pd, &ah->ibv_ah, attr, &resp.ibv_resp, sizeof(resp))) 1699321936Shselasky goto err; 1700321936Shselasky 1701321936Shselasky ah->kern_ah = true; 1702321936Shselasky memcpy(ah->av.rmac, resp.dmac, ETHERNET_LL_SIZE); 1703321936Shselasky } else { 1704321936Shselasky uint16_t vid; 1705321936Shselasky 1706321936Shselasky if (ibv_resolve_eth_l2_from_gid(pd->context, attr, 1707321936Shselasky ah->av.rmac, &vid)) 1708321936Shselasky goto err; 1709321936Shselasky } 1710321936Shselasky } 1711321936Shselasky 1712321936Shselasky return &ah->ibv_ah; 1713321936Shselaskyerr: 1714321936Shselasky free(ah); 1715321936Shselasky return NULL; 1716321936Shselasky} 1717321936Shselasky 1718321936Shselaskyint mlx5_destroy_ah(struct ibv_ah *ah) 1719321936Shselasky{ 1720321936Shselasky struct mlx5_ah *mah = to_mah(ah); 1721321936Shselasky int err; 1722321936Shselasky 1723321936Shselasky if (mah->kern_ah) { 1724321936Shselasky err = ibv_cmd_destroy_ah(ah); 1725321936Shselasky if (err) 1726321936Shselasky return err; 1727321936Shselasky } 1728321936Shselasky 1729321936Shselasky free(mah); 1730321936Shselasky return 0; 1731321936Shselasky} 1732321936Shselasky 1733321936Shselaskyint mlx5_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1734321936Shselasky{ 1735321936Shselasky return ibv_cmd_attach_mcast(qp, gid, lid); 1736321936Shselasky} 1737321936Shselasky 1738321936Shselaskyint mlx5_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1739321936Shselasky{ 1740321936Shselasky return ibv_cmd_detach_mcast(qp, gid, lid); 1741321936Shselasky} 1742321936Shselasky 1743321936Shselaskystruct ibv_qp *mlx5_create_qp_ex(struct ibv_context *context, 1744321936Shselasky struct ibv_qp_init_attr_ex *attr) 1745321936Shselasky{ 1746321936Shselasky return create_qp(context, attr); 1747321936Shselasky} 1748321936Shselasky 1749321936Shselaskyint mlx5_get_srq_num(struct ibv_srq *srq, uint32_t *srq_num) 1750321936Shselasky{ 1751321936Shselasky struct mlx5_srq *msrq = to_msrq(srq); 1752321936Shselasky 1753321936Shselasky *srq_num = msrq->srqn; 1754321936Shselasky 1755321936Shselasky return 0; 1756321936Shselasky} 1757321936Shselasky 1758321936Shselaskystruct ibv_xrcd * 1759321936Shselaskymlx5_open_xrcd(struct ibv_context *context, 1760321936Shselasky struct ibv_xrcd_init_attr *xrcd_init_attr) 1761321936Shselasky{ 1762321936Shselasky int err; 1763321936Shselasky struct verbs_xrcd *xrcd; 1764321936Shselasky struct ibv_open_xrcd cmd = {}; 1765321936Shselasky struct ibv_open_xrcd_resp resp = {}; 1766321936Shselasky 1767321936Shselasky xrcd = calloc(1, sizeof(*xrcd)); 1768321936Shselasky if (!xrcd) 1769321936Shselasky return NULL; 1770321936Shselasky 1771321936Shselasky err = ibv_cmd_open_xrcd(context, xrcd, sizeof(*xrcd), xrcd_init_attr, 1772321936Shselasky &cmd, sizeof(cmd), &resp, sizeof(resp)); 1773321936Shselasky if (err) { 1774321936Shselasky free(xrcd); 1775321936Shselasky return NULL; 1776321936Shselasky } 1777321936Shselasky 1778321936Shselasky return &xrcd->xrcd; 1779321936Shselasky} 1780321936Shselasky 1781321936Shselaskyint mlx5_close_xrcd(struct ibv_xrcd *ib_xrcd) 1782321936Shselasky{ 1783321936Shselasky struct verbs_xrcd *xrcd = container_of(ib_xrcd, struct verbs_xrcd, xrcd); 1784321936Shselasky int ret; 1785321936Shselasky 1786321936Shselasky ret = ibv_cmd_close_xrcd(xrcd); 1787321936Shselasky if (!ret) 1788321936Shselasky free(xrcd); 1789321936Shselasky 1790321936Shselasky return ret; 1791321936Shselasky} 1792321936Shselasky 1793321936Shselaskystatic struct ibv_srq * 1794321936Shselaskymlx5_create_xrc_srq(struct ibv_context *context, 1795321936Shselasky struct ibv_srq_init_attr_ex *attr) 1796321936Shselasky{ 1797321936Shselasky int err; 1798321936Shselasky struct mlx5_create_srq_ex cmd; 1799321936Shselasky struct mlx5_create_srq_resp resp; 1800321936Shselasky struct mlx5_srq *msrq; 1801321936Shselasky struct mlx5_context *ctx = to_mctx(context); 1802321936Shselasky int max_sge; 1803321936Shselasky struct ibv_srq *ibsrq; 1804321936Shselasky int uidx; 1805321936Shselasky FILE *fp = ctx->dbg_fp; 1806321936Shselasky 1807321936Shselasky msrq = calloc(1, sizeof(*msrq)); 1808321936Shselasky if (!msrq) 1809321936Shselasky return NULL; 1810321936Shselasky 1811321936Shselasky ibsrq = (struct ibv_srq *)&msrq->vsrq; 1812321936Shselasky 1813321936Shselasky memset(&cmd, 0, sizeof(cmd)); 1814321936Shselasky memset(&resp, 0, sizeof(resp)); 1815321936Shselasky 1816321936Shselasky if (mlx5_spinlock_init(&msrq->lock)) { 1817321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 1818321936Shselasky goto err; 1819321936Shselasky } 1820321936Shselasky 1821321936Shselasky if (attr->attr.max_wr > ctx->max_srq_recv_wr) { 1822321936Shselasky fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", 1823321936Shselasky __func__, __LINE__, attr->attr.max_wr, 1824321936Shselasky ctx->max_srq_recv_wr); 1825321936Shselasky errno = EINVAL; 1826321936Shselasky goto err; 1827321936Shselasky } 1828321936Shselasky 1829321936Shselasky /* 1830321936Shselasky * this calculation does not consider required control segments. The 1831321936Shselasky * final calculation is done again later. This is done so to avoid 1832321936Shselasky * overflows of variables 1833321936Shselasky */ 1834321936Shselasky max_sge = ctx->max_recv_wr / sizeof(struct mlx5_wqe_data_seg); 1835321936Shselasky if (attr->attr.max_sge > max_sge) { 1836321936Shselasky fprintf(stderr, "%s-%d:max_wr %d, max_srq_recv_wr %d\n", 1837321936Shselasky __func__, __LINE__, attr->attr.max_wr, 1838321936Shselasky ctx->max_srq_recv_wr); 1839321936Shselasky errno = EINVAL; 1840321936Shselasky goto err; 1841321936Shselasky } 1842321936Shselasky 1843321936Shselasky msrq->max = align_queue_size(attr->attr.max_wr + 1); 1844321936Shselasky msrq->max_gs = attr->attr.max_sge; 1845321936Shselasky msrq->counter = 0; 1846321936Shselasky 1847321936Shselasky if (mlx5_alloc_srq_buf(context, msrq)) { 1848321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 1849321936Shselasky goto err; 1850321936Shselasky } 1851321936Shselasky 1852321936Shselasky msrq->db = mlx5_alloc_dbrec(ctx); 1853321936Shselasky if (!msrq->db) { 1854321936Shselasky fprintf(stderr, "%s-%d:\n", __func__, __LINE__); 1855321936Shselasky goto err_free; 1856321936Shselasky } 1857321936Shselasky 1858321936Shselasky *msrq->db = 0; 1859321936Shselasky 1860321936Shselasky cmd.buf_addr = (uintptr_t)msrq->buf.buf; 1861321936Shselasky cmd.db_addr = (uintptr_t)msrq->db; 1862321936Shselasky msrq->wq_sig = srq_sig_enabled(); 1863321936Shselasky if (msrq->wq_sig) 1864321936Shselasky cmd.flags = MLX5_SRQ_FLAG_SIGNATURE; 1865321936Shselasky 1866321936Shselasky attr->attr.max_sge = msrq->max_gs; 1867321936Shselasky if (ctx->cqe_version) { 1868321936Shselasky uidx = mlx5_store_uidx(ctx, msrq); 1869321936Shselasky if (uidx < 0) { 1870321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 1871321936Shselasky goto err_free_db; 1872321936Shselasky } 1873321936Shselasky cmd.uidx = uidx; 1874321936Shselasky } else { 1875321936Shselasky cmd.uidx = 0xffffff; 1876321936Shselasky pthread_mutex_lock(&ctx->srq_table_mutex); 1877321936Shselasky } 1878321936Shselasky 1879321936Shselasky err = ibv_cmd_create_srq_ex(context, &msrq->vsrq, sizeof(msrq->vsrq), 1880321936Shselasky attr, &cmd.ibv_cmd, sizeof(cmd), 1881321936Shselasky &resp.ibv_resp, sizeof(resp)); 1882321936Shselasky if (err) 1883321936Shselasky goto err_free_uidx; 1884321936Shselasky 1885321936Shselasky if (!ctx->cqe_version) { 1886321936Shselasky err = mlx5_store_srq(to_mctx(context), resp.srqn, msrq); 1887321936Shselasky if (err) 1888321936Shselasky goto err_destroy; 1889321936Shselasky 1890321936Shselasky pthread_mutex_unlock(&ctx->srq_table_mutex); 1891321936Shselasky } 1892321936Shselasky 1893321936Shselasky msrq->srqn = resp.srqn; 1894321936Shselasky msrq->rsc.type = MLX5_RSC_TYPE_XSRQ; 1895321936Shselasky msrq->rsc.rsn = ctx->cqe_version ? cmd.uidx : resp.srqn; 1896321936Shselasky 1897321936Shselasky return ibsrq; 1898321936Shselasky 1899321936Shselaskyerr_destroy: 1900321936Shselasky ibv_cmd_destroy_srq(ibsrq); 1901321936Shselasky 1902321936Shselaskyerr_free_uidx: 1903321936Shselasky if (ctx->cqe_version) 1904321936Shselasky mlx5_clear_uidx(ctx, cmd.uidx); 1905321936Shselasky else 1906321936Shselasky pthread_mutex_unlock(&ctx->srq_table_mutex); 1907321936Shselasky 1908321936Shselaskyerr_free_db: 1909321936Shselasky mlx5_free_db(ctx, msrq->db); 1910321936Shselasky 1911321936Shselaskyerr_free: 1912321936Shselasky free(msrq->wrid); 1913321936Shselasky mlx5_free_buf(&msrq->buf); 1914321936Shselasky 1915321936Shselaskyerr: 1916321936Shselasky free(msrq); 1917321936Shselasky 1918321936Shselasky return NULL; 1919321936Shselasky} 1920321936Shselasky 1921321936Shselaskystruct ibv_srq *mlx5_create_srq_ex(struct ibv_context *context, 1922321936Shselasky struct ibv_srq_init_attr_ex *attr) 1923321936Shselasky{ 1924321936Shselasky if (!(attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) || 1925321936Shselasky (attr->srq_type == IBV_SRQT_BASIC)) 1926321936Shselasky return mlx5_create_srq(attr->pd, 1927321936Shselasky (struct ibv_srq_init_attr *)attr); 1928321936Shselasky else if (attr->srq_type == IBV_SRQT_XRC) 1929321936Shselasky return mlx5_create_xrc_srq(context, attr); 1930321936Shselasky 1931321936Shselasky return NULL; 1932321936Shselasky} 1933321936Shselasky 1934321936Shselaskyint mlx5_query_device_ex(struct ibv_context *context, 1935321936Shselasky const struct ibv_query_device_ex_input *input, 1936321936Shselasky struct ibv_device_attr_ex *attr, 1937321936Shselasky size_t attr_size) 1938321936Shselasky{ 1939321936Shselasky struct mlx5_context *mctx = to_mctx(context); 1940321936Shselasky struct mlx5_query_device_ex_resp resp; 1941321936Shselasky struct mlx5_query_device_ex cmd; 1942321936Shselasky struct ibv_device_attr *a; 1943321936Shselasky uint64_t raw_fw_ver; 1944321936Shselasky unsigned sub_minor; 1945321936Shselasky unsigned major; 1946321936Shselasky unsigned minor; 1947321936Shselasky int err; 1948321936Shselasky int cmd_supp_uhw = mctx->cmds_supp_uhw & 1949321936Shselasky MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE; 1950321936Shselasky 1951321936Shselasky memset(&cmd, 0, sizeof(cmd)); 1952321936Shselasky memset(&resp, 0, sizeof(resp)); 1953321936Shselasky err = ibv_cmd_query_device_ex(context, input, attr, attr_size, 1954321936Shselasky &raw_fw_ver, 1955321936Shselasky &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), 1956321936Shselasky &resp.ibv_resp, sizeof(resp.ibv_resp), 1957321936Shselasky cmd_supp_uhw ? sizeof(resp) : sizeof(resp.ibv_resp)); 1958321936Shselasky if (err) 1959321936Shselasky return err; 1960321936Shselasky 1961321936Shselasky attr->tso_caps = resp.tso_caps; 1962321936Shselasky attr->rss_caps.rx_hash_fields_mask = resp.rss_caps.rx_hash_fields_mask; 1963321936Shselasky attr->rss_caps.rx_hash_function = resp.rss_caps.rx_hash_function; 1964321936Shselasky attr->packet_pacing_caps = resp.packet_pacing_caps.caps; 1965321936Shselasky 1966321936Shselasky if (resp.support_multi_pkt_send_wqe) 1967321936Shselasky mctx->vendor_cap_flags |= MLX5_VENDOR_CAP_FLAGS_MPW; 1968321936Shselasky 1969321936Shselasky mctx->cqe_comp_caps = resp.cqe_comp_caps; 1970321936Shselasky 1971321936Shselasky major = (raw_fw_ver >> 32) & 0xffff; 1972321936Shselasky minor = (raw_fw_ver >> 16) & 0xffff; 1973321936Shselasky sub_minor = raw_fw_ver & 0xffff; 1974321936Shselasky a = &attr->orig_attr; 1975321936Shselasky snprintf(a->fw_ver, sizeof(a->fw_ver), "%d.%d.%04d", 1976321936Shselasky major, minor, sub_minor); 1977321936Shselasky 1978321936Shselasky return 0; 1979321936Shselasky} 1980321936Shselasky 1981321936Shselaskystatic int rwq_sig_enabled(struct ibv_context *context) 1982321936Shselasky{ 1983321936Shselasky char *env; 1984321936Shselasky 1985321936Shselasky env = getenv("MLX5_RWQ_SIGNATURE"); 1986321936Shselasky if (env) 1987321936Shselasky return 1; 1988321936Shselasky 1989321936Shselasky return 0; 1990321936Shselasky} 1991321936Shselasky 1992321936Shselaskystatic void mlx5_free_rwq_buf(struct mlx5_rwq *rwq, struct ibv_context *context) 1993321936Shselasky{ 1994321936Shselasky struct mlx5_context *ctx = to_mctx(context); 1995321936Shselasky 1996321936Shselasky mlx5_free_actual_buf(ctx, &rwq->buf); 1997321936Shselasky free(rwq->rq.wrid); 1998321936Shselasky} 1999321936Shselasky 2000321936Shselaskystatic int mlx5_alloc_rwq_buf(struct ibv_context *context, 2001321936Shselasky struct mlx5_rwq *rwq, 2002321936Shselasky int size) 2003321936Shselasky{ 2004321936Shselasky int err; 2005321936Shselasky enum mlx5_alloc_type default_alloc_type = MLX5_ALLOC_TYPE_PREFER_CONTIG; 2006321936Shselasky 2007321936Shselasky rwq->rq.wrid = malloc(rwq->rq.wqe_cnt * sizeof(uint64_t)); 2008321936Shselasky if (!rwq->rq.wrid) { 2009321936Shselasky errno = ENOMEM; 2010321936Shselasky return -1; 2011321936Shselasky } 2012321936Shselasky 2013321936Shselasky err = mlx5_alloc_prefered_buf(to_mctx(context), &rwq->buf, 2014321936Shselasky align(rwq->buf_size, to_mdev 2015321936Shselasky (context->device)->page_size), 2016321936Shselasky to_mdev(context->device)->page_size, 2017321936Shselasky default_alloc_type, 2018321936Shselasky MLX5_RWQ_PREFIX); 2019321936Shselasky 2020321936Shselasky if (err) { 2021321936Shselasky free(rwq->rq.wrid); 2022321936Shselasky errno = ENOMEM; 2023321936Shselasky return -1; 2024321936Shselasky } 2025321936Shselasky 2026321936Shselasky return 0; 2027321936Shselasky} 2028321936Shselasky 2029321936Shselaskystruct ibv_wq *mlx5_create_wq(struct ibv_context *context, 2030321936Shselasky struct ibv_wq_init_attr *attr) 2031321936Shselasky{ 2032321936Shselasky struct mlx5_create_wq cmd; 2033321936Shselasky struct mlx5_create_wq_resp resp; 2034321936Shselasky int err; 2035321936Shselasky struct mlx5_rwq *rwq; 2036321936Shselasky struct mlx5_context *ctx = to_mctx(context); 2037321936Shselasky int ret; 2038321936Shselasky int32_t usr_idx = 0; 2039321936Shselasky FILE *fp = ctx->dbg_fp; 2040321936Shselasky 2041321936Shselasky if (attr->wq_type != IBV_WQT_RQ) 2042321936Shselasky return NULL; 2043321936Shselasky 2044321936Shselasky memset(&cmd, 0, sizeof(cmd)); 2045321936Shselasky memset(&resp, 0, sizeof(resp)); 2046321936Shselasky 2047321936Shselasky rwq = calloc(1, sizeof(*rwq)); 2048321936Shselasky if (!rwq) 2049321936Shselasky return NULL; 2050321936Shselasky 2051321936Shselasky rwq->wq_sig = rwq_sig_enabled(context); 2052321936Shselasky if (rwq->wq_sig) 2053321936Shselasky cmd.drv.flags = MLX5_RWQ_FLAG_SIGNATURE; 2054321936Shselasky 2055321936Shselasky ret = mlx5_calc_rwq_size(ctx, rwq, attr); 2056321936Shselasky if (ret < 0) { 2057321936Shselasky errno = -ret; 2058321936Shselasky goto err; 2059321936Shselasky } 2060321936Shselasky 2061321936Shselasky rwq->buf_size = ret; 2062321936Shselasky if (mlx5_alloc_rwq_buf(context, rwq, ret)) 2063321936Shselasky goto err; 2064321936Shselasky 2065321936Shselasky mlx5_init_rwq_indices(rwq); 2066321936Shselasky 2067321936Shselasky if (mlx5_spinlock_init(&rwq->rq.lock)) 2068321936Shselasky goto err_free_rwq_buf; 2069321936Shselasky 2070321936Shselasky rwq->db = mlx5_alloc_dbrec(ctx); 2071321936Shselasky if (!rwq->db) 2072321936Shselasky goto err_free_rwq_buf; 2073321936Shselasky 2074321936Shselasky rwq->db[MLX5_RCV_DBR] = 0; 2075321936Shselasky rwq->db[MLX5_SND_DBR] = 0; 2076321936Shselasky rwq->pbuff = rwq->buf.buf + rwq->rq.offset; 2077321936Shselasky rwq->recv_db = &rwq->db[MLX5_RCV_DBR]; 2078321936Shselasky cmd.drv.buf_addr = (uintptr_t)rwq->buf.buf; 2079321936Shselasky cmd.drv.db_addr = (uintptr_t)rwq->db; 2080321936Shselasky cmd.drv.rq_wqe_count = rwq->rq.wqe_cnt; 2081321936Shselasky cmd.drv.rq_wqe_shift = rwq->rq.wqe_shift; 2082321936Shselasky usr_idx = mlx5_store_uidx(ctx, rwq); 2083321936Shselasky if (usr_idx < 0) { 2084321936Shselasky mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); 2085321936Shselasky goto err_free_db_rec; 2086321936Shselasky } 2087321936Shselasky 2088321936Shselasky cmd.drv.user_index = usr_idx; 2089321936Shselasky err = ibv_cmd_create_wq(context, attr, &rwq->wq, &cmd.ibv_cmd, 2090321936Shselasky sizeof(cmd.ibv_cmd), 2091321936Shselasky sizeof(cmd), 2092321936Shselasky &resp.ibv_resp, sizeof(resp.ibv_resp), 2093321936Shselasky sizeof(resp)); 2094321936Shselasky if (err) 2095321936Shselasky goto err_create; 2096321936Shselasky 2097321936Shselasky rwq->rsc.type = MLX5_RSC_TYPE_RWQ; 2098321936Shselasky rwq->rsc.rsn = cmd.drv.user_index; 2099321936Shselasky 2100321936Shselasky rwq->wq.post_recv = mlx5_post_wq_recv; 2101321936Shselasky return &rwq->wq; 2102321936Shselasky 2103321936Shselaskyerr_create: 2104321936Shselasky mlx5_clear_uidx(ctx, cmd.drv.user_index); 2105321936Shselaskyerr_free_db_rec: 2106321936Shselasky mlx5_free_db(to_mctx(context), rwq->db); 2107321936Shselaskyerr_free_rwq_buf: 2108321936Shselasky mlx5_free_rwq_buf(rwq, context); 2109321936Shselaskyerr: 2110321936Shselasky free(rwq); 2111321936Shselasky return NULL; 2112321936Shselasky} 2113321936Shselasky 2114321936Shselaskyint mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr) 2115321936Shselasky{ 2116321936Shselasky struct mlx5_modify_wq cmd = {}; 2117321936Shselasky struct mlx5_rwq *rwq = to_mrwq(wq); 2118321936Shselasky 2119321936Shselasky if ((attr->attr_mask & IBV_WQ_ATTR_STATE) && 2120321936Shselasky attr->wq_state == IBV_WQS_RDY) { 2121321936Shselasky if ((attr->attr_mask & IBV_WQ_ATTR_CURR_STATE) && 2122321936Shselasky attr->curr_wq_state != wq->state) 2123321936Shselasky return -EINVAL; 2124321936Shselasky 2125321936Shselasky if (wq->state == IBV_WQS_RESET) { 2126321936Shselasky mlx5_spin_lock(&to_mcq(wq->cq)->lock); 2127321936Shselasky __mlx5_cq_clean(to_mcq(wq->cq), 2128321936Shselasky rwq->rsc.rsn, NULL); 2129321936Shselasky mlx5_spin_unlock(&to_mcq(wq->cq)->lock); 2130321936Shselasky mlx5_init_rwq_indices(rwq); 2131321936Shselasky rwq->db[MLX5_RCV_DBR] = 0; 2132321936Shselasky rwq->db[MLX5_SND_DBR] = 0; 2133321936Shselasky } 2134321936Shselasky } 2135321936Shselasky 2136321936Shselasky return ibv_cmd_modify_wq(wq, attr, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd)); 2137321936Shselasky} 2138321936Shselasky 2139321936Shselaskyint mlx5_destroy_wq(struct ibv_wq *wq) 2140321936Shselasky{ 2141321936Shselasky struct mlx5_rwq *rwq = to_mrwq(wq); 2142321936Shselasky int ret; 2143321936Shselasky 2144321936Shselasky ret = ibv_cmd_destroy_wq(wq); 2145321936Shselasky if (ret) 2146321936Shselasky return ret; 2147321936Shselasky 2148321936Shselasky mlx5_spin_lock(&to_mcq(wq->cq)->lock); 2149321936Shselasky __mlx5_cq_clean(to_mcq(wq->cq), rwq->rsc.rsn, NULL); 2150321936Shselasky mlx5_spin_unlock(&to_mcq(wq->cq)->lock); 2151321936Shselasky mlx5_clear_uidx(to_mctx(wq->context), rwq->rsc.rsn); 2152321936Shselasky mlx5_free_db(to_mctx(wq->context), rwq->db); 2153321936Shselasky mlx5_free_rwq_buf(rwq, wq->context); 2154321936Shselasky free(rwq); 2155321936Shselasky 2156321936Shselasky return 0; 2157321936Shselasky} 2158321936Shselasky 2159321936Shselaskystruct ibv_rwq_ind_table *mlx5_create_rwq_ind_table(struct ibv_context *context, 2160321936Shselasky struct ibv_rwq_ind_table_init_attr *init_attr) 2161321936Shselasky{ 2162321936Shselasky struct ibv_create_rwq_ind_table *cmd; 2163321936Shselasky struct mlx5_create_rwq_ind_table_resp resp; 2164321936Shselasky struct ibv_rwq_ind_table *ind_table; 2165321936Shselasky uint32_t required_tbl_size; 2166321936Shselasky int num_tbl_entries; 2167321936Shselasky int cmd_size; 2168321936Shselasky int err; 2169321936Shselasky 2170321936Shselasky num_tbl_entries = 1 << init_attr->log_ind_tbl_size; 2171321936Shselasky /* Data must be u64 aligned */ 2172321936Shselasky required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ? 2173321936Shselasky sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t)); 2174321936Shselasky 2175321936Shselasky cmd_size = required_tbl_size + sizeof(*cmd); 2176321936Shselasky cmd = calloc(1, cmd_size); 2177321936Shselasky if (!cmd) 2178321936Shselasky return NULL; 2179321936Shselasky 2180321936Shselasky memset(&resp, 0, sizeof(resp)); 2181321936Shselasky ind_table = calloc(1, sizeof(*ind_table)); 2182321936Shselasky if (!ind_table) 2183321936Shselasky goto free_cmd; 2184321936Shselasky 2185321936Shselasky err = ibv_cmd_create_rwq_ind_table(context, init_attr, ind_table, cmd, 2186321936Shselasky cmd_size, cmd_size, &resp.ibv_resp, sizeof(resp.ibv_resp), 2187321936Shselasky sizeof(resp)); 2188321936Shselasky if (err) 2189321936Shselasky goto err; 2190321936Shselasky 2191321936Shselasky free(cmd); 2192321936Shselasky return ind_table; 2193321936Shselasky 2194321936Shselaskyerr: 2195321936Shselasky free(ind_table); 2196321936Shselaskyfree_cmd: 2197321936Shselasky free(cmd); 2198321936Shselasky return NULL; 2199321936Shselasky} 2200321936Shselasky 2201321936Shselaskyint mlx5_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 2202321936Shselasky{ 2203321936Shselasky int ret; 2204321936Shselasky 2205321936Shselasky ret = ibv_cmd_destroy_rwq_ind_table(rwq_ind_table); 2206321936Shselasky 2207321936Shselasky if (ret) 2208321936Shselasky return ret; 2209321936Shselasky 2210321936Shselasky free(rwq_ind_table); 2211321936Shselasky return 0; 2212321936Shselasky} 2213