1290650Shselasky/*- 2337115Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_srq.c 337115 2018-08-02 08:56:27Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30290650Shselasky#include <dev/mlx5/driver.h> 31290650Shselasky#include <dev/mlx5/srq.h> 32290650Shselasky#include <rdma/ib_verbs.h> 33290650Shselasky#include "mlx5_core.h" 34290650Shselasky#include "transobj.h" 35290650Shselasky 36290650Shselaskyvoid mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type) 37290650Shselasky{ 38290650Shselasky struct mlx5_srq_table *table = &dev->priv.srq_table; 39290650Shselasky struct mlx5_core_srq *srq; 40290650Shselasky 41290650Shselasky spin_lock(&table->lock); 42290650Shselasky 43290650Shselasky srq = radix_tree_lookup(&table->tree, srqn); 44290650Shselasky if (srq) 45290650Shselasky atomic_inc(&srq->refcount); 46290650Shselasky 47290650Shselasky spin_unlock(&table->lock); 48290650Shselasky 49290650Shselasky if (!srq) { 50290650Shselasky mlx5_core_warn(dev, "Async event for bogus SRQ 0x%08x\n", srqn); 51290650Shselasky return; 52290650Shselasky } 53290650Shselasky 54290650Shselasky srq->event(srq, event_type); 55290650Shselasky 56290650Shselasky if (atomic_dec_and_test(&srq->refcount)) 57290650Shselasky complete(&srq->free); 58290650Shselasky} 59290650Shselasky 60331807Shselaskystatic void set_wq(void *wq, struct mlx5_srq_attr *in) 61290650Shselasky{ 62331807Shselasky MLX5_SET(wq, wq, wq_signature, !!(in->flags & MLX5_SRQ_FLAG_WQ_SIG)); 63331807Shselasky MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size); 64331807Shselasky MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4); 65331807Shselasky MLX5_SET(wq, wq, log_wq_sz, in->log_size); 66331807Shselasky MLX5_SET(wq, wq, page_offset, in->page_offset); 67331807Shselasky MLX5_SET(wq, wq, lwm, in->lwm); 68331807Shselasky MLX5_SET(wq, wq, pd, in->pd); 69331807Shselasky MLX5_SET64(wq, wq, dbr_addr, in->db_record); 70331807Shselasky} 71290650Shselasky 72331807Shselaskystatic void set_srqc(void *srqc, struct mlx5_srq_attr *in) 73331807Shselasky{ 74331807Shselasky MLX5_SET(srqc, srqc, wq_signature, !!(in->flags & MLX5_SRQ_FLAG_WQ_SIG)); 75331807Shselasky MLX5_SET(srqc, srqc, log_page_size, in->log_page_size); 76331807Shselasky MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift); 77331807Shselasky MLX5_SET(srqc, srqc, log_srq_size, in->log_size); 78331807Shselasky MLX5_SET(srqc, srqc, page_offset, in->page_offset); 79331807Shselasky MLX5_SET(srqc, srqc, lwm, in->lwm); 80331807Shselasky MLX5_SET(srqc, srqc, pd, in->pd); 81331807Shselasky MLX5_SET64(srqc, srqc, dbr_addr, in->db_record); 82331807Shselasky MLX5_SET(srqc, srqc, xrcd, in->xrcd); 83331807Shselasky MLX5_SET(srqc, srqc, cqn, in->cqn); 84331807Shselasky} 85290650Shselasky 86331807Shselaskystatic void get_wq(void *wq, struct mlx5_srq_attr *in) 87331807Shselasky{ 88331807Shselasky if (MLX5_GET(wq, wq, wq_signature)) 89331807Shselasky in->flags &= MLX5_SRQ_FLAG_WQ_SIG; 90331807Shselasky in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz); 91331807Shselasky in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4; 92331807Shselasky in->log_size = MLX5_GET(wq, wq, log_wq_sz); 93331807Shselasky in->page_offset = MLX5_GET(wq, wq, page_offset); 94331807Shselasky in->lwm = MLX5_GET(wq, wq, lwm); 95331807Shselasky in->pd = MLX5_GET(wq, wq, pd); 96331807Shselasky in->db_record = MLX5_GET64(wq, wq, dbr_addr); 97331807Shselasky} 98290650Shselasky 99331807Shselaskystatic void get_srqc(void *srqc, struct mlx5_srq_attr *in) 100331807Shselasky{ 101331807Shselasky if (MLX5_GET(srqc, srqc, wq_signature)) 102331807Shselasky in->flags &= MLX5_SRQ_FLAG_WQ_SIG; 103331807Shselasky in->log_page_size = MLX5_GET(srqc, srqc, log_page_size); 104331807Shselasky in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride); 105331807Shselasky in->log_size = MLX5_GET(srqc, srqc, log_srq_size); 106331807Shselasky in->page_offset = MLX5_GET(srqc, srqc, page_offset); 107331807Shselasky in->lwm = MLX5_GET(srqc, srqc, lwm); 108331807Shselasky in->pd = MLX5_GET(srqc, srqc, pd); 109331807Shselasky in->db_record = MLX5_GET64(srqc, srqc, dbr_addr); 110290650Shselasky} 111290650Shselasky 112290650Shselaskystruct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn) 113290650Shselasky{ 114290650Shselasky struct mlx5_srq_table *table = &dev->priv.srq_table; 115290650Shselasky struct mlx5_core_srq *srq; 116290650Shselasky 117290650Shselasky spin_lock(&table->lock); 118290650Shselasky 119290650Shselasky srq = radix_tree_lookup(&table->tree, srqn); 120290650Shselasky if (srq) 121290650Shselasky atomic_inc(&srq->refcount); 122290650Shselasky 123290650Shselasky spin_unlock(&table->lock); 124290650Shselasky 125290650Shselasky return srq; 126290650Shselasky} 127290650ShselaskyEXPORT_SYMBOL(mlx5_core_get_srq); 128290650Shselasky 129331807Shselaskystatic int get_pas_size(struct mlx5_srq_attr *in) 130290650Shselasky{ 131331807Shselasky u32 log_page_size = in->log_page_size + 12; 132331807Shselasky u32 log_srq_size = in->log_size; 133331807Shselasky u32 log_rq_stride = in->wqe_shift; 134331807Shselasky u32 page_offset = in->page_offset; 135290650Shselasky u32 po_quanta = 1 << (log_page_size - 6); 136290650Shselasky u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride); 137290650Shselasky u32 page_size = 1 << log_page_size; 138290650Shselasky u32 rq_sz_po = rq_sz + (page_offset * po_quanta); 139290650Shselasky u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size; 140290650Shselasky 141290650Shselasky return rq_num_pas * sizeof(u64); 142290650Shselasky 143290650Shselasky} 144290650Shselasky 145290650Shselaskystatic int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 146331807Shselasky struct mlx5_srq_attr *in) 147290650Shselasky{ 148290650Shselasky void *create_in; 149290650Shselasky void *rmpc; 150331807Shselasky void *wq; 151290650Shselasky int pas_size; 152290650Shselasky int inlen; 153290650Shselasky int err; 154290650Shselasky 155331807Shselasky pas_size = get_pas_size(in); 156290650Shselasky inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size; 157290650Shselasky create_in = mlx5_vzalloc(inlen); 158290650Shselasky if (!create_in) 159290650Shselasky return -ENOMEM; 160290650Shselasky 161290650Shselasky rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx); 162331807Shselasky wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 163290650Shselasky 164331807Shselasky MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY); 165331807Shselasky set_wq(wq, in); 166290650Shselasky memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size); 167290650Shselasky 168290650Shselasky err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn); 169290650Shselasky 170290650Shselasky kvfree(create_in); 171290650Shselasky return err; 172290650Shselasky} 173290650Shselasky 174290650Shselaskystatic int destroy_rmp_cmd(struct mlx5_core_dev *dev, 175290650Shselasky struct mlx5_core_srq *srq) 176290650Shselasky{ 177290650Shselasky return mlx5_core_destroy_rmp(dev, srq->srqn); 178290650Shselasky} 179290650Shselasky 180290650Shselaskystatic int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 181331807Shselasky struct mlx5_srq_attr *out) 182290650Shselasky{ 183290650Shselasky u32 *rmp_out; 184290650Shselasky void *rmpc; 185290650Shselasky int err; 186290650Shselasky 187290650Shselasky rmp_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_rmp_out)); 188290650Shselasky if (!rmp_out) 189290650Shselasky return -ENOMEM; 190290650Shselasky 191290650Shselasky err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out); 192290650Shselasky if (err) 193290650Shselasky goto out; 194290650Shselasky 195290650Shselasky rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context); 196331807Shselasky get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out); 197331807Shselasky if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY) 198331807Shselasky out->flags |= MLX5_SRQ_FLAG_ERR; 199290650Shselasky 200290650Shselaskyout: 201290650Shselasky kvfree(rmp_out); 202290650Shselasky return 0; 203290650Shselasky} 204290650Shselasky 205290650Shselaskystatic int arm_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, u16 lwm) 206290650Shselasky{ 207290650Shselasky return mlx5_core_arm_rmp(dev, srq->srqn, lwm); 208290650Shselasky} 209290650Shselasky 210290650Shselaskystatic int create_xrc_srq_cmd(struct mlx5_core_dev *dev, 211290650Shselasky struct mlx5_core_srq *srq, 212331807Shselasky struct mlx5_srq_attr *in) 213290650Shselasky{ 214290650Shselasky void *create_in; 215290650Shselasky void *xrc_srqc; 216290650Shselasky void *pas; 217290650Shselasky int pas_size; 218290650Shselasky int inlen; 219290650Shselasky int err; 220290650Shselasky 221331807Shselasky pas_size = get_pas_size(in); 222290650Shselasky inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size; 223290650Shselasky create_in = mlx5_vzalloc(inlen); 224290650Shselasky if (!create_in) 225290650Shselasky return -ENOMEM; 226290650Shselasky 227290650Shselasky xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in, xrc_srq_context_entry); 228290650Shselasky pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas); 229290650Shselasky 230331807Shselasky set_srqc(xrc_srqc, in); 231331807Shselasky MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index); 232290650Shselasky memcpy(pas, in->pas, pas_size); 233290650Shselasky 234290650Shselasky err = mlx5_core_create_xsrq(dev, create_in, inlen, &srq->srqn); 235290650Shselasky if (err) 236290650Shselasky goto out; 237290650Shselasky 238290650Shselaskyout: 239290650Shselasky kvfree(create_in); 240290650Shselasky return err; 241290650Shselasky} 242290650Shselasky 243290650Shselaskystatic int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev, 244290650Shselasky struct mlx5_core_srq *srq) 245290650Shselasky{ 246290650Shselasky return mlx5_core_destroy_xsrq(dev, srq->srqn); 247290650Shselasky} 248290650Shselasky 249290650Shselaskystatic int query_xrc_srq_cmd(struct mlx5_core_dev *dev, 250290650Shselasky struct mlx5_core_srq *srq, 251331807Shselasky struct mlx5_srq_attr *out) 252290650Shselasky{ 253290650Shselasky u32 *xrcsrq_out; 254331807Shselasky void *xrc_srqc; 255290650Shselasky int err; 256290650Shselasky 257290650Shselasky xrcsrq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out)); 258290650Shselasky if (!xrcsrq_out) 259290650Shselasky return -ENOMEM; 260290650Shselasky 261290650Shselasky err = mlx5_core_query_xsrq(dev, srq->srqn, xrcsrq_out); 262290650Shselasky if (err) 263290650Shselasky goto out; 264290650Shselasky 265331807Shselasky xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out, 266331807Shselasky xrc_srq_context_entry); 267331807Shselasky get_srqc(xrc_srqc, out); 268331807Shselasky if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD) 269331807Shselasky out->flags |= MLX5_SRQ_FLAG_ERR; 270331807Shselasky 271290650Shselaskyout: 272290650Shselasky kvfree(xrcsrq_out); 273290650Shselasky return err; 274290650Shselasky} 275290650Shselasky 276290650Shselaskystatic int arm_xrc_srq_cmd(struct mlx5_core_dev *dev, 277290650Shselasky struct mlx5_core_srq *srq, u16 lwm) 278290650Shselasky{ 279290650Shselasky return mlx5_core_arm_xsrq(dev, srq->srqn, lwm); 280290650Shselasky} 281290650Shselasky 282290650Shselaskystatic int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 283331807Shselasky struct mlx5_srq_attr *in) 284290650Shselasky{ 285331807Shselasky u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0}; 286331807Shselasky void *create_in; 287331807Shselasky void *srqc; 288331807Shselasky void *pas; 289331807Shselasky int pas_size; 290331807Shselasky int inlen; 291290650Shselasky int err; 292290650Shselasky 293331807Shselasky pas_size = get_pas_size(in); 294331807Shselasky inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size; 295331807Shselasky create_in = mlx5_vzalloc(inlen); 296331807Shselasky if (!create_in) 297331807Shselasky return -ENOMEM; 298290650Shselasky 299331807Shselasky srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry); 300331807Shselasky pas = MLX5_ADDR_OF(create_srq_in, create_in, pas); 301290650Shselasky 302331807Shselasky set_srqc(srqc, in); 303331807Shselasky memcpy(pas, in->pas, pas_size); 304290650Shselasky 305331807Shselasky MLX5_SET(create_srq_in, create_in, opcode, MLX5_CMD_OP_CREATE_SRQ); 306331807Shselasky err = mlx5_cmd_exec(dev, create_in, inlen, create_out, sizeof(create_out)); 307331807Shselasky kvfree(create_in); 308331807Shselasky if (!err) 309331807Shselasky srq->srqn = MLX5_GET(create_srq_out, create_out, srqn); 310290650Shselasky 311290650Shselasky return err; 312290650Shselasky} 313290650Shselasky 314290650Shselaskystatic int destroy_srq_cmd(struct mlx5_core_dev *dev, 315290650Shselasky struct mlx5_core_srq *srq) 316290650Shselasky{ 317331807Shselasky u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0}; 318331807Shselasky u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0}; 319290650Shselasky 320331807Shselasky MLX5_SET(destroy_srq_in, srq_in, opcode, MLX5_CMD_OP_DESTROY_SRQ); 321331807Shselasky MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn); 322290650Shselasky 323331807Shselasky return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, sizeof(srq_out)); 324290650Shselasky} 325290650Shselasky 326290650Shselaskystatic int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 327331807Shselasky struct mlx5_srq_attr *out) 328290650Shselasky{ 329331807Shselasky u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0}; 330331807Shselasky u32 *srq_out; 331331807Shselasky void *srqc; 332331807Shselasky int outlen = MLX5_ST_SZ_BYTES(query_srq_out); 333331807Shselasky int err; 334290650Shselasky 335331807Shselasky srq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_srq_out)); 336331807Shselasky if (!srq_out) 337331807Shselasky return -ENOMEM; 338290650Shselasky 339331807Shselasky MLX5_SET(query_srq_in, srq_in, opcode, MLX5_CMD_OP_QUERY_SRQ); 340331807Shselasky MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn); 341331807Shselasky err = mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, outlen); 342331807Shselasky if (err) 343331807Shselasky goto out; 344290650Shselasky 345331807Shselasky srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry); 346331807Shselasky get_srqc(srqc, out); 347331807Shselasky if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD) 348331807Shselasky out->flags |= MLX5_SRQ_FLAG_ERR; 349331807Shselaskyout: 350331807Shselasky kvfree(srq_out); 351331807Shselasky return err; 352290650Shselasky} 353290650Shselasky 354290650Shselaskystatic int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 355290650Shselasky u16 lwm, int is_srq) 356290650Shselasky{ 357331807Shselasky /* arm_srq structs missing using identical xrc ones */ 358331807Shselasky u32 srq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0}; 359331807Shselasky u32 srq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0}; 360290650Shselasky 361331807Shselasky MLX5_SET(arm_xrc_srq_in, srq_in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ); 362331807Shselasky MLX5_SET(arm_xrc_srq_in, srq_in, xrc_srqn, srq->srqn); 363331807Shselasky MLX5_SET(arm_xrc_srq_in, srq_in, lwm, lwm); 364290650Shselasky 365331807Shselasky return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in), srq_out, sizeof(srq_out)); 366290650Shselasky} 367290650Shselasky 368290650Shselaskystatic int create_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 369331807Shselasky struct mlx5_srq_attr *in) 370290650Shselasky{ 371290650Shselasky if (!dev->issi) 372331807Shselasky return create_srq_cmd(dev, srq, in); 373290650Shselasky else if (srq->common.res == MLX5_RES_XSRQ) 374331807Shselasky return create_xrc_srq_cmd(dev, srq, in); 375290650Shselasky else 376331807Shselasky return create_rmp_cmd(dev, srq, in); 377290650Shselasky} 378290650Shselasky 379290650Shselaskystatic int destroy_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq) 380290650Shselasky{ 381290650Shselasky if (!dev->issi) 382290650Shselasky return destroy_srq_cmd(dev, srq); 383290650Shselasky else if (srq->common.res == MLX5_RES_XSRQ) 384290650Shselasky return destroy_xrc_srq_cmd(dev, srq); 385290650Shselasky else 386290650Shselasky return destroy_rmp_cmd(dev, srq); 387290650Shselasky} 388290650Shselasky 389290650Shselaskyint mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 390331807Shselasky struct mlx5_srq_attr *in) 391290650Shselasky{ 392290650Shselasky int err; 393290650Shselasky struct mlx5_srq_table *table = &dev->priv.srq_table; 394290650Shselasky 395331807Shselasky if (in->type == IB_SRQT_XRC) 396331807Shselasky srq->common.res = MLX5_RES_XSRQ; 397331807Shselasky else 398331807Shselasky srq->common.res = MLX5_RES_SRQ; 399290650Shselasky 400331807Shselasky err = create_srq_split(dev, srq, in); 401290650Shselasky if (err) 402290650Shselasky return err; 403290650Shselasky 404290650Shselasky atomic_set(&srq->refcount, 1); 405290650Shselasky init_completion(&srq->free); 406290650Shselasky 407290650Shselasky spin_lock_irq(&table->lock); 408290650Shselasky err = radix_tree_insert(&table->tree, srq->srqn, srq); 409290650Shselasky spin_unlock_irq(&table->lock); 410290650Shselasky if (err) { 411290650Shselasky mlx5_core_warn(dev, "err %d, srqn 0x%x\n", err, srq->srqn); 412290650Shselasky goto err_destroy_srq_split; 413290650Shselasky } 414290650Shselasky 415290650Shselasky return 0; 416290650Shselasky 417290650Shselaskyerr_destroy_srq_split: 418290650Shselasky destroy_srq_split(dev, srq); 419290650Shselasky 420290650Shselasky return err; 421290650Shselasky} 422290650ShselaskyEXPORT_SYMBOL(mlx5_core_create_srq); 423290650Shselasky 424290650Shselaskyint mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq) 425290650Shselasky{ 426290650Shselasky struct mlx5_srq_table *table = &dev->priv.srq_table; 427290650Shselasky struct mlx5_core_srq *tmp; 428290650Shselasky int err; 429290650Shselasky 430290650Shselasky spin_lock_irq(&table->lock); 431290650Shselasky tmp = radix_tree_delete(&table->tree, srq->srqn); 432290650Shselasky spin_unlock_irq(&table->lock); 433290650Shselasky if (!tmp) { 434290650Shselasky mlx5_core_warn(dev, "srq 0x%x not found in tree\n", srq->srqn); 435290650Shselasky return -EINVAL; 436290650Shselasky } 437290650Shselasky if (tmp != srq) { 438290650Shselasky mlx5_core_warn(dev, "corruption on srqn 0x%x\n", srq->srqn); 439290650Shselasky return -EINVAL; 440290650Shselasky } 441290650Shselasky 442290650Shselasky err = destroy_srq_split(dev, srq); 443290650Shselasky if (err) 444290650Shselasky return err; 445290650Shselasky 446290650Shselasky if (atomic_dec_and_test(&srq->refcount)) 447290650Shselasky complete(&srq->free); 448290650Shselasky wait_for_completion(&srq->free); 449290650Shselasky 450290650Shselasky return 0; 451290650Shselasky} 452290650ShselaskyEXPORT_SYMBOL(mlx5_core_destroy_srq); 453290650Shselasky 454290650Shselaskyint mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 455331807Shselasky struct mlx5_srq_attr *out) 456290650Shselasky{ 457290650Shselasky if (!dev->issi) 458290650Shselasky return query_srq_cmd(dev, srq, out); 459290650Shselasky else if (srq->common.res == MLX5_RES_XSRQ) 460290650Shselasky return query_xrc_srq_cmd(dev, srq, out); 461290650Shselasky else 462290650Shselasky return query_rmp_cmd(dev, srq, out); 463290650Shselasky} 464290650ShselaskyEXPORT_SYMBOL(mlx5_core_query_srq); 465290650Shselasky 466290650Shselaskyint mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, 467290650Shselasky u16 lwm, int is_srq) 468290650Shselasky{ 469290650Shselasky if (!dev->issi) 470290650Shselasky return arm_srq_cmd(dev, srq, lwm, is_srq); 471290650Shselasky else if (srq->common.res == MLX5_RES_XSRQ) 472290650Shselasky return arm_xrc_srq_cmd(dev, srq, lwm); 473290650Shselasky else 474290650Shselasky return arm_rmp_cmd(dev, srq, lwm); 475290650Shselasky} 476290650ShselaskyEXPORT_SYMBOL(mlx5_core_arm_srq); 477290650Shselasky 478290650Shselaskyvoid mlx5_init_srq_table(struct mlx5_core_dev *dev) 479290650Shselasky{ 480290650Shselasky struct mlx5_srq_table *table = &dev->priv.srq_table; 481290650Shselasky 482331580Shselasky memset(table, 0, sizeof(*table)); 483290650Shselasky spin_lock_init(&table->lock); 484290650Shselasky INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 485290650Shselasky} 486290650Shselasky 487290650Shselaskyvoid mlx5_cleanup_srq_table(struct mlx5_core_dev *dev) 488290650Shselasky{ 489290650Shselasky /* nothing */ 490290650Shselasky} 491