mlx5_cq.c revision 331580
1290650Shselasky/*- 2290650Shselasky * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_cq.c 331580 2018-03-26 20:33:31Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30290650Shselasky#include <linux/hardirq.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include <rdma/ib_verbs.h> 33290650Shselasky#include <dev/mlx5/cq.h> 34290650Shselasky#include "mlx5_core.h" 35290650Shselasky 36290650Shselaskyvoid mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 37290650Shselasky{ 38290650Shselasky struct mlx5_core_cq *cq; 39290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 40290650Shselasky 41290650Shselasky if (cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 42290650Shselasky struct mlx5_cq_linear_array_entry *entry; 43290650Shselasky 44290650Shselasky entry = &table->linear_array[cqn]; 45290650Shselasky spin_lock(&entry->lock); 46290650Shselasky cq = entry->cq; 47290650Shselasky if (cq == NULL) { 48290650Shselasky mlx5_core_warn(dev, 49290650Shselasky "Completion event for bogus CQ 0x%x\n", cqn); 50290650Shselasky } else { 51290650Shselasky ++cq->arm_sn; 52290650Shselasky cq->comp(cq); 53290650Shselasky } 54290650Shselasky spin_unlock(&entry->lock); 55290650Shselasky return; 56290650Shselasky } 57290650Shselasky 58290650Shselasky spin_lock(&table->lock); 59290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 60290650Shselasky if (likely(cq)) 61290650Shselasky atomic_inc(&cq->refcount); 62290650Shselasky spin_unlock(&table->lock); 63290650Shselasky 64290650Shselasky if (!cq) { 65290650Shselasky mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn); 66290650Shselasky return; 67290650Shselasky } 68290650Shselasky 69290650Shselasky ++cq->arm_sn; 70290650Shselasky 71290650Shselasky cq->comp(cq); 72290650Shselasky 73290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 74290650Shselasky complete(&cq->free); 75290650Shselasky} 76290650Shselasky 77290650Shselaskyvoid mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 78290650Shselasky{ 79290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 80290650Shselasky struct mlx5_core_cq *cq; 81290650Shselasky 82290650Shselasky spin_lock(&table->lock); 83290650Shselasky 84290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 85290650Shselasky if (cq) 86290650Shselasky atomic_inc(&cq->refcount); 87290650Shselasky 88290650Shselasky spin_unlock(&table->lock); 89290650Shselasky 90290650Shselasky if (!cq) { 91290650Shselasky mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn); 92290650Shselasky return; 93290650Shselasky } 94290650Shselasky 95290650Shselasky cq->event(cq, event_type); 96290650Shselasky 97290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 98290650Shselasky complete(&cq->free); 99290650Shselasky} 100290650Shselasky 101290650Shselasky 102290650Shselaskyint mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 103290650Shselasky struct mlx5_create_cq_mbox_in *in, int inlen) 104290650Shselasky{ 105290650Shselasky int err; 106290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 107290650Shselasky struct mlx5_create_cq_mbox_out out; 108290650Shselasky struct mlx5_destroy_cq_mbox_in din; 109290650Shselasky struct mlx5_destroy_cq_mbox_out dout; 110290650Shselasky 111290650Shselasky in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ); 112290650Shselasky memset(&out, 0, sizeof(out)); 113290650Shselasky err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 114290650Shselasky if (err) 115290650Shselasky return err; 116290650Shselasky 117290650Shselasky if (out.hdr.status) 118290650Shselasky return mlx5_cmd_status_to_err(&out.hdr); 119290650Shselasky 120290650Shselasky cq->cqn = be32_to_cpu(out.cqn) & 0xffffff; 121290650Shselasky cq->cons_index = 0; 122290650Shselasky cq->arm_sn = 0; 123290650Shselasky atomic_set(&cq->refcount, 1); 124290650Shselasky init_completion(&cq->free); 125290650Shselasky 126290650Shselasky spin_lock_irq(&table->lock); 127290650Shselasky err = radix_tree_insert(&table->tree, cq->cqn, cq); 128290650Shselasky spin_unlock_irq(&table->lock); 129290650Shselasky if (err) 130290650Shselasky goto err_cmd; 131290650Shselasky 132290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 133290650Shselasky struct mlx5_cq_linear_array_entry *entry; 134290650Shselasky 135290650Shselasky entry = &table->linear_array[cq->cqn]; 136290650Shselasky spin_lock_irq(&entry->lock); 137290650Shselasky entry->cq = cq; 138290650Shselasky spin_unlock_irq(&entry->lock); 139290650Shselasky } 140290650Shselasky 141290650Shselasky cq->pid = curthread->td_proc->p_pid; 142290650Shselasky 143290650Shselasky return 0; 144290650Shselasky 145290650Shselaskyerr_cmd: 146290650Shselasky memset(&din, 0, sizeof(din)); 147290650Shselasky memset(&dout, 0, sizeof(dout)); 148290650Shselasky din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 149290650Shselasky din.cqn = cpu_to_be32(cq->cqn); 150290650Shselasky mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout)); 151290650Shselasky return err; 152290650Shselasky} 153290650ShselaskyEXPORT_SYMBOL(mlx5_core_create_cq); 154290650Shselasky 155290650Shselaskyint mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 156290650Shselasky{ 157290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 158290650Shselasky struct mlx5_destroy_cq_mbox_in in; 159290650Shselasky struct mlx5_destroy_cq_mbox_out out; 160290650Shselasky struct mlx5_core_cq *tmp; 161290650Shselasky int err; 162290650Shselasky 163290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 164290650Shselasky struct mlx5_cq_linear_array_entry *entry; 165290650Shselasky 166290650Shselasky entry = &table->linear_array[cq->cqn]; 167290650Shselasky spin_lock_irq(&entry->lock); 168290650Shselasky entry->cq = NULL; 169290650Shselasky spin_unlock_irq(&entry->lock); 170290650Shselasky } 171290650Shselasky 172290650Shselasky spin_lock_irq(&table->lock); 173290650Shselasky tmp = radix_tree_delete(&table->tree, cq->cqn); 174290650Shselasky spin_unlock_irq(&table->lock); 175290650Shselasky if (!tmp) { 176290650Shselasky mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 177290650Shselasky return -EINVAL; 178290650Shselasky } 179290650Shselasky if (tmp != cq) { 180290650Shselasky mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); 181290650Shselasky return -EINVAL; 182290650Shselasky } 183290650Shselasky 184290650Shselasky memset(&in, 0, sizeof(in)); 185290650Shselasky memset(&out, 0, sizeof(out)); 186290650Shselasky in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 187290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 188290650Shselasky err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 189290650Shselasky if (err) 190306233Shselasky goto out; 191290650Shselasky 192306233Shselasky if (out.hdr.status) { 193306233Shselasky err = mlx5_cmd_status_to_err(&out.hdr); 194306233Shselasky goto out; 195306233Shselasky } 196290650Shselasky 197290650Shselasky synchronize_irq(cq->irqn); 198290650Shselasky 199290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 200290650Shselasky complete(&cq->free); 201290650Shselasky wait_for_completion(&cq->free); 202290650Shselasky 203306233Shselaskyout: 204306233Shselasky 205306233Shselasky return err; 206290650Shselasky} 207290650ShselaskyEXPORT_SYMBOL(mlx5_core_destroy_cq); 208290650Shselasky 209290650Shselaskyint mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 210290650Shselasky struct mlx5_query_cq_mbox_out *out) 211290650Shselasky{ 212290650Shselasky struct mlx5_query_cq_mbox_in in; 213290650Shselasky int err; 214290650Shselasky 215290650Shselasky memset(&in, 0, sizeof(in)); 216290650Shselasky memset(out, 0, sizeof(*out)); 217290650Shselasky 218290650Shselasky in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ); 219290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 220290650Shselasky err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); 221290650Shselasky if (err) 222290650Shselasky return err; 223290650Shselasky 224290650Shselasky if (out->hdr.status) 225290650Shselasky return mlx5_cmd_status_to_err(&out->hdr); 226290650Shselasky 227290650Shselasky return err; 228290650Shselasky} 229290650ShselaskyEXPORT_SYMBOL(mlx5_core_query_cq); 230290650Shselasky 231290650Shselasky 232290650Shselaskyint mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 233290650Shselasky struct mlx5_modify_cq_mbox_in *in, int in_sz) 234290650Shselasky{ 235290650Shselasky struct mlx5_modify_cq_mbox_out out; 236290650Shselasky int err; 237290650Shselasky 238290650Shselasky memset(&out, 0, sizeof(out)); 239290650Shselasky in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ); 240290650Shselasky err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out)); 241290650Shselasky if (err) 242290650Shselasky return err; 243290650Shselasky 244290650Shselasky if (out.hdr.status) 245290650Shselasky return mlx5_cmd_status_to_err(&out.hdr); 246290650Shselasky 247290650Shselasky return 0; 248290650Shselasky} 249290650ShselaskyEXPORT_SYMBOL(mlx5_core_modify_cq); 250290650Shselasky 251290650Shselaskyint mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 252290650Shselasky struct mlx5_core_cq *cq, 253290650Shselasky u16 cq_period, 254290650Shselasky u16 cq_max_count) 255290650Shselasky{ 256290650Shselasky struct mlx5_modify_cq_mbox_in in; 257290650Shselasky 258290650Shselasky memset(&in, 0, sizeof(in)); 259290650Shselasky 260290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 261290650Shselasky in.ctx.cq_period = cpu_to_be16(cq_period); 262290650Shselasky in.ctx.cq_max_count = cpu_to_be16(cq_max_count); 263290650Shselasky in.field_select = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD | 264290650Shselasky MLX5_CQ_MODIFY_COUNT); 265290650Shselasky 266290650Shselasky return mlx5_core_modify_cq(dev, cq, &in, sizeof(in)); 267290650Shselasky} 268290650Shselasky 269321995Shselaskyint mlx5_core_modify_cq_moderation_mode(struct mlx5_core_dev *dev, 270321995Shselasky struct mlx5_core_cq *cq, 271321995Shselasky u16 cq_period, 272321995Shselasky u16 cq_max_count, 273321995Shselasky u8 cq_mode) 274321995Shselasky{ 275321995Shselasky struct mlx5_modify_cq_mbox_in in; 276321995Shselasky 277321995Shselasky memset(&in, 0, sizeof(in)); 278321995Shselasky 279321995Shselasky in.cqn = cpu_to_be32(cq->cqn); 280321995Shselasky in.ctx.cq_period = cpu_to_be16(cq_period); 281321995Shselasky in.ctx.cq_max_count = cpu_to_be16(cq_max_count); 282321995Shselasky in.ctx.cqe_sz_flags = (cq_mode & 2) >> 1; 283321995Shselasky in.ctx.st = (cq_mode & 1) << 7; 284321995Shselasky in.field_select = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD | 285321995Shselasky MLX5_CQ_MODIFY_COUNT | 286321995Shselasky MLX5_CQ_MODIFY_PERIOD_MODE); 287321995Shselasky 288321995Shselasky return mlx5_core_modify_cq(dev, cq, &in, sizeof(in)); 289321995Shselasky} 290321995Shselasky 291290650Shselaskyint mlx5_init_cq_table(struct mlx5_core_dev *dev) 292290650Shselasky{ 293290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 294290650Shselasky int err; 295290650Shselasky int x; 296290650Shselasky 297331580Shselasky memset(table, 0, sizeof(*table)); 298290650Shselasky spin_lock_init(&table->lock); 299290650Shselasky for (x = 0; x != MLX5_CQ_LINEAR_ARRAY_SIZE; x++) 300290650Shselasky spin_lock_init(&table->linear_array[x].lock); 301290650Shselasky INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 302290650Shselasky err = 0; 303290650Shselasky 304290650Shselasky return err; 305290650Shselasky} 306290650Shselasky 307290650Shselaskyvoid mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 308290650Shselasky{ 309290650Shselasky} 310