mlx5_cq.c revision 290650
1290650Shselasky/*- 2290650Shselasky * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: head/sys/dev/mlx5/mlx5_core/mlx5_cq.c 290650 2015-11-10 12:20:22Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30290650Shselasky#include <linux/hardirq.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include <rdma/ib_verbs.h> 33290650Shselasky#include <dev/mlx5/cq.h> 34290650Shselasky#include "mlx5_core.h" 35290650Shselasky 36290650Shselaskyvoid mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 37290650Shselasky{ 38290650Shselasky struct mlx5_core_cq *cq; 39290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 40290650Shselasky 41290650Shselasky if (cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 42290650Shselasky struct mlx5_cq_linear_array_entry *entry; 43290650Shselasky 44290650Shselasky entry = &table->linear_array[cqn]; 45290650Shselasky spin_lock(&entry->lock); 46290650Shselasky cq = entry->cq; 47290650Shselasky if (cq == NULL) { 48290650Shselasky mlx5_core_warn(dev, 49290650Shselasky "Completion event for bogus CQ 0x%x\n", cqn); 50290650Shselasky } else { 51290650Shselasky ++cq->arm_sn; 52290650Shselasky cq->comp(cq); 53290650Shselasky } 54290650Shselasky spin_unlock(&entry->lock); 55290650Shselasky return; 56290650Shselasky } 57290650Shselasky 58290650Shselasky spin_lock(&table->lock); 59290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 60290650Shselasky if (likely(cq)) 61290650Shselasky atomic_inc(&cq->refcount); 62290650Shselasky spin_unlock(&table->lock); 63290650Shselasky 64290650Shselasky if (!cq) { 65290650Shselasky mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn); 66290650Shselasky return; 67290650Shselasky } 68290650Shselasky 69290650Shselasky ++cq->arm_sn; 70290650Shselasky 71290650Shselasky cq->comp(cq); 72290650Shselasky 73290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 74290650Shselasky complete(&cq->free); 75290650Shselasky} 76290650Shselasky 77290650Shselaskyvoid mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 78290650Shselasky{ 79290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 80290650Shselasky struct mlx5_core_cq *cq; 81290650Shselasky 82290650Shselasky spin_lock(&table->lock); 83290650Shselasky 84290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 85290650Shselasky if (cq) 86290650Shselasky atomic_inc(&cq->refcount); 87290650Shselasky 88290650Shselasky spin_unlock(&table->lock); 89290650Shselasky 90290650Shselasky if (!cq) { 91290650Shselasky mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn); 92290650Shselasky return; 93290650Shselasky } 94290650Shselasky 95290650Shselasky cq->event(cq, event_type); 96290650Shselasky 97290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 98290650Shselasky complete(&cq->free); 99290650Shselasky} 100290650Shselasky 101290650Shselasky 102290650Shselaskyint mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 103290650Shselasky struct mlx5_create_cq_mbox_in *in, int inlen) 104290650Shselasky{ 105290650Shselasky int err; 106290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 107290650Shselasky struct mlx5_create_cq_mbox_out out; 108290650Shselasky struct mlx5_destroy_cq_mbox_in din; 109290650Shselasky struct mlx5_destroy_cq_mbox_out dout; 110290650Shselasky 111290650Shselasky in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ); 112290650Shselasky memset(&out, 0, sizeof(out)); 113290650Shselasky err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 114290650Shselasky if (err) 115290650Shselasky return err; 116290650Shselasky 117290650Shselasky if (out.hdr.status) 118290650Shselasky return mlx5_cmd_status_to_err(&out.hdr); 119290650Shselasky 120290650Shselasky cq->cqn = be32_to_cpu(out.cqn) & 0xffffff; 121290650Shselasky cq->cons_index = 0; 122290650Shselasky cq->arm_sn = 0; 123290650Shselasky atomic_set(&cq->refcount, 1); 124290650Shselasky init_completion(&cq->free); 125290650Shselasky 126290650Shselasky spin_lock_irq(&table->lock); 127290650Shselasky err = radix_tree_insert(&table->tree, cq->cqn, cq); 128290650Shselasky spin_unlock_irq(&table->lock); 129290650Shselasky if (err) 130290650Shselasky goto err_cmd; 131290650Shselasky 132290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 133290650Shselasky struct mlx5_cq_linear_array_entry *entry; 134290650Shselasky 135290650Shselasky entry = &table->linear_array[cq->cqn]; 136290650Shselasky spin_lock_irq(&entry->lock); 137290650Shselasky entry->cq = cq; 138290650Shselasky spin_unlock_irq(&entry->lock); 139290650Shselasky } 140290650Shselasky 141290650Shselasky cq->pid = curthread->td_proc->p_pid; 142290650Shselasky 143290650Shselasky return 0; 144290650Shselasky 145290650Shselaskyerr_cmd: 146290650Shselasky memset(&din, 0, sizeof(din)); 147290650Shselasky memset(&dout, 0, sizeof(dout)); 148290650Shselasky din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 149290650Shselasky din.cqn = cpu_to_be32(cq->cqn); 150290650Shselasky mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout)); 151290650Shselasky return err; 152290650Shselasky} 153290650ShselaskyEXPORT_SYMBOL(mlx5_core_create_cq); 154290650Shselasky 155290650Shselaskyint mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 156290650Shselasky{ 157290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 158290650Shselasky struct mlx5_destroy_cq_mbox_in in; 159290650Shselasky struct mlx5_destroy_cq_mbox_out out; 160290650Shselasky struct mlx5_core_cq *tmp; 161290650Shselasky int err; 162290650Shselasky 163290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 164290650Shselasky struct mlx5_cq_linear_array_entry *entry; 165290650Shselasky 166290650Shselasky entry = &table->linear_array[cq->cqn]; 167290650Shselasky spin_lock_irq(&entry->lock); 168290650Shselasky entry->cq = NULL; 169290650Shselasky spin_unlock_irq(&entry->lock); 170290650Shselasky } 171290650Shselasky 172290650Shselasky spin_lock_irq(&table->lock); 173290650Shselasky tmp = radix_tree_delete(&table->tree, cq->cqn); 174290650Shselasky spin_unlock_irq(&table->lock); 175290650Shselasky if (!tmp) { 176290650Shselasky mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 177290650Shselasky return -EINVAL; 178290650Shselasky } 179290650Shselasky if (tmp != cq) { 180290650Shselasky mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); 181290650Shselasky return -EINVAL; 182290650Shselasky } 183290650Shselasky 184290650Shselasky memset(&in, 0, sizeof(in)); 185290650Shselasky memset(&out, 0, sizeof(out)); 186290650Shselasky in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 187290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 188290650Shselasky err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 189290650Shselasky if (err) 190290650Shselasky return err; 191290650Shselasky 192290650Shselasky if (out.hdr.status) 193290650Shselasky return mlx5_cmd_status_to_err(&out.hdr); 194290650Shselasky 195290650Shselasky synchronize_irq(cq->irqn); 196290650Shselasky 197290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 198290650Shselasky complete(&cq->free); 199290650Shselasky wait_for_completion(&cq->free); 200290650Shselasky 201290650Shselasky return 0; 202290650Shselasky} 203290650ShselaskyEXPORT_SYMBOL(mlx5_core_destroy_cq); 204290650Shselasky 205290650Shselaskyint mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 206290650Shselasky struct mlx5_query_cq_mbox_out *out) 207290650Shselasky{ 208290650Shselasky struct mlx5_query_cq_mbox_in in; 209290650Shselasky int err; 210290650Shselasky 211290650Shselasky memset(&in, 0, sizeof(in)); 212290650Shselasky memset(out, 0, sizeof(*out)); 213290650Shselasky 214290650Shselasky in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ); 215290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 216290650Shselasky err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); 217290650Shselasky if (err) 218290650Shselasky return err; 219290650Shselasky 220290650Shselasky if (out->hdr.status) 221290650Shselasky return mlx5_cmd_status_to_err(&out->hdr); 222290650Shselasky 223290650Shselasky return err; 224290650Shselasky} 225290650ShselaskyEXPORT_SYMBOL(mlx5_core_query_cq); 226290650Shselasky 227290650Shselasky 228290650Shselaskyint mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 229290650Shselasky struct mlx5_modify_cq_mbox_in *in, int in_sz) 230290650Shselasky{ 231290650Shselasky struct mlx5_modify_cq_mbox_out out; 232290650Shselasky int err; 233290650Shselasky 234290650Shselasky memset(&out, 0, sizeof(out)); 235290650Shselasky in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ); 236290650Shselasky err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out)); 237290650Shselasky if (err) 238290650Shselasky return err; 239290650Shselasky 240290650Shselasky if (out.hdr.status) 241290650Shselasky return mlx5_cmd_status_to_err(&out.hdr); 242290650Shselasky 243290650Shselasky return 0; 244290650Shselasky} 245290650ShselaskyEXPORT_SYMBOL(mlx5_core_modify_cq); 246290650Shselasky 247290650Shselaskyint mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 248290650Shselasky struct mlx5_core_cq *cq, 249290650Shselasky u16 cq_period, 250290650Shselasky u16 cq_max_count) 251290650Shselasky{ 252290650Shselasky struct mlx5_modify_cq_mbox_in in; 253290650Shselasky 254290650Shselasky memset(&in, 0, sizeof(in)); 255290650Shselasky 256290650Shselasky in.cqn = cpu_to_be32(cq->cqn); 257290650Shselasky in.ctx.cq_period = cpu_to_be16(cq_period); 258290650Shselasky in.ctx.cq_max_count = cpu_to_be16(cq_max_count); 259290650Shselasky in.field_select = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD | 260290650Shselasky MLX5_CQ_MODIFY_COUNT); 261290650Shselasky 262290650Shselasky return mlx5_core_modify_cq(dev, cq, &in, sizeof(in)); 263290650Shselasky} 264290650Shselasky 265290650Shselaskyint mlx5_init_cq_table(struct mlx5_core_dev *dev) 266290650Shselasky{ 267290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 268290650Shselasky int err; 269290650Shselasky int x; 270290650Shselasky 271290650Shselasky spin_lock_init(&table->lock); 272290650Shselasky for (x = 0; x != MLX5_CQ_LINEAR_ARRAY_SIZE; x++) 273290650Shselasky spin_lock_init(&table->linear_array[x].lock); 274290650Shselasky INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 275290650Shselasky err = 0; 276290650Shselasky 277290650Shselasky return err; 278290650Shselasky} 279290650Shselasky 280290650Shselaskyvoid mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 281290650Shselasky{ 282290650Shselasky} 283