mlx5_cq.c revision 302408
155714Skris/*- 255714Skris * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 355714Skris * 455714Skris * Redistribution and use in source and binary forms, with or without 555714Skris * modification, are permitted provided that the following conditions 655714Skris * are met: 755714Skris * 1. Redistributions of source code must retain the above copyright 8280297Sjkim * notice, this list of conditions and the following disclaimer. 955714Skris * 2. Redistributions in binary form must reproduce the above copyright 1055714Skris * notice, this list of conditions and the following disclaimer in the 1155714Skris * documentation and/or other materials provided with the distribution. 1255714Skris * 1355714Skris * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 1455714Skris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15280297Sjkim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1655714Skris * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 1755714Skris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1855714Skris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1955714Skris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2055714Skris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2155714Skris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22280297Sjkim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2355714Skris * SUCH DAMAGE. 2455714Skris * 2555714Skris * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_cq.c 290650 2015-11-10 12:20:22Z hselasky $ 2655714Skris */ 2755714Skris 2855714Skris#include <linux/kernel.h> 2955714Skris#include <linux/module.h> 3055714Skris#include <linux/hardirq.h> 3155714Skris#include <dev/mlx5/driver.h> 3255714Skris#include <rdma/ib_verbs.h> 3355714Skris#include <dev/mlx5/cq.h> 3455714Skris#include "mlx5_core.h" 3555714Skris 3655714Skrisvoid mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 37280297Sjkim{ 3855714Skris struct mlx5_core_cq *cq; 3955714Skris struct mlx5_cq_table *table = &dev->priv.cq_table; 40280297Sjkim 4155714Skris if (cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 4255714Skris struct mlx5_cq_linear_array_entry *entry; 4355714Skris 4455714Skris entry = &table->linear_array[cqn]; 4555714Skris spin_lock(&entry->lock); 4655714Skris cq = entry->cq; 4755714Skris if (cq == NULL) { 4855714Skris mlx5_core_warn(dev, 4955714Skris "Completion event for bogus CQ 0x%x\n", cqn); 5055714Skris } else { 5155714Skris ++cq->arm_sn; 52280297Sjkim cq->comp(cq); 5355714Skris } 5455714Skris spin_unlock(&entry->lock); 5555714Skris return; 5655714Skris } 5755714Skris 5855714Skris spin_lock(&table->lock); 5955714Skris cq = radix_tree_lookup(&table->tree, cqn); 6055714Skris if (likely(cq)) 6155714Skris atomic_inc(&cq->refcount); 6255714Skris spin_unlock(&table->lock); 6355714Skris 6468651Skris if (!cq) { 6568651Skris mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn); 6668651Skris return; 6768651Skris } 6868651Skris 6955714Skris ++cq->arm_sn; 7055714Skris 71280297Sjkim cq->comp(cq); 72280297Sjkim 73280297Sjkim if (atomic_dec_and_test(&cq->refcount)) 74280297Sjkim complete(&cq->free); 75280297Sjkim} 76280297Sjkim 77280297Sjkimvoid mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 78280297Sjkim{ 79280297Sjkim struct mlx5_cq_table *table = &dev->priv.cq_table; 80280297Sjkim struct mlx5_core_cq *cq; 81280297Sjkim 82280297Sjkim spin_lock(&table->lock); 8355714Skris 84280297Sjkim cq = radix_tree_lookup(&table->tree, cqn); 85280297Sjkim if (cq) 86280297Sjkim atomic_inc(&cq->refcount); 87280297Sjkim 8855714Skris spin_unlock(&table->lock); 8955714Skris 90280297Sjkim if (!cq) { 91280297Sjkim mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn); 92280297Sjkim return; 9355714Skris } 94296279Sjkim 95296279Sjkim cq->event(cq, event_type); 9659191Skris 97280297Sjkim if (atomic_dec_and_test(&cq->refcount)) 98280297Sjkim complete(&cq->free); 99280297Sjkim} 100238405Sjkim 101280297Sjkim 102280297Sjkimint mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 103280297Sjkim struct mlx5_create_cq_mbox_in *in, int inlen) 104280297Sjkim{ 105280297Sjkim int err; 106280297Sjkim struct mlx5_cq_table *table = &dev->priv.cq_table; 107280297Sjkim struct mlx5_create_cq_mbox_out out; 108280297Sjkim struct mlx5_destroy_cq_mbox_in din; 109296279Sjkim struct mlx5_destroy_cq_mbox_out dout; 110296279Sjkim 111280297Sjkim in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ); 112280297Sjkim memset(&out, 0, sizeof(out)); 113280297Sjkim err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 114280297Sjkim if (err) 115280297Sjkim return err; 116280297Sjkim 11759191Skris if (out.hdr.status) 11859191Skris return mlx5_cmd_status_to_err(&out.hdr); 11955714Skris 120280297Sjkim cq->cqn = be32_to_cpu(out.cqn) & 0xffffff; 121280297Sjkim cq->cons_index = 0; 12255714Skris cq->arm_sn = 0; 123280297Sjkim atomic_set(&cq->refcount, 1); 124280297Sjkim init_completion(&cq->free); 125280297Sjkim 126280297Sjkim spin_lock_irq(&table->lock); 127280297Sjkim err = radix_tree_insert(&table->tree, cq->cqn, cq); 128280297Sjkim spin_unlock_irq(&table->lock); 129280297Sjkim if (err) 130280297Sjkim goto err_cmd; 13155714Skris 13255714Skris if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 133280297Sjkim struct mlx5_cq_linear_array_entry *entry; 134280297Sjkim 135280297Sjkim entry = &table->linear_array[cq->cqn]; 136280297Sjkim spin_lock_irq(&entry->lock); 137280297Sjkim entry->cq = cq; 138280297Sjkim spin_unlock_irq(&entry->lock); 139280297Sjkim } 140280297Sjkim 141280297Sjkim cq->pid = curthread->td_proc->p_pid; 142280297Sjkim 143280297Sjkim return 0; 144280297Sjkim 145280297Sjkimerr_cmd: 146280297Sjkim memset(&din, 0, sizeof(din)); 147280297Sjkim memset(&dout, 0, sizeof(dout)); 148280297Sjkim din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 14955714Skris din.cqn = cpu_to_be32(cq->cqn); 150280297Sjkim mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout)); 151280297Sjkim return err; 152280297Sjkim} 15355714SkrisEXPORT_SYMBOL(mlx5_core_create_cq); 154280297Sjkim 155280297Sjkimint mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 156280297Sjkim{ 157280297Sjkim struct mlx5_cq_table *table = &dev->priv.cq_table; 158280297Sjkim struct mlx5_destroy_cq_mbox_in in; 159280297Sjkim struct mlx5_destroy_cq_mbox_out out; 160280297Sjkim struct mlx5_core_cq *tmp; 161280297Sjkim int err; 162280297Sjkim 163280297Sjkim if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 164280297Sjkim struct mlx5_cq_linear_array_entry *entry; 165280297Sjkim 166280297Sjkim entry = &table->linear_array[cq->cqn]; 167280297Sjkim spin_lock_irq(&entry->lock); 168280297Sjkim entry->cq = NULL; 169280297Sjkim spin_unlock_irq(&entry->lock); 170280297Sjkim } 171280297Sjkim 17255714Skris spin_lock_irq(&table->lock); 17368651Skris tmp = radix_tree_delete(&table->tree, cq->cqn); 174280297Sjkim spin_unlock_irq(&table->lock); 175280297Sjkim if (!tmp) { 176280297Sjkim mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 177280297Sjkim return -EINVAL; 17855714Skris } 179280297Sjkim if (tmp != cq) { 180280297Sjkim mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); 181280297Sjkim return -EINVAL; 182280297Sjkim } 183280297Sjkim 18455714Skris memset(&in, 0, sizeof(in)); 185280297Sjkim memset(&out, 0, sizeof(out)); 186280297Sjkim in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ); 187280297Sjkim in.cqn = cpu_to_be32(cq->cqn); 188280297Sjkim err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 18959191Skris if (err) 190280297Sjkim return err; 191280297Sjkim 192280297Sjkim if (out.hdr.status) 193280297Sjkim return mlx5_cmd_status_to_err(&out.hdr); 194280297Sjkim 195280297Sjkim synchronize_irq(cq->irqn); 196280297Sjkim 197280297Sjkim if (atomic_dec_and_test(&cq->refcount)) 198280297Sjkim complete(&cq->free); 19955714Skris wait_for_completion(&cq->free); 20068651Skris 201280297Sjkim return 0; 202280297Sjkim} 203280297SjkimEXPORT_SYMBOL(mlx5_core_destroy_cq); 20455714Skris 205280297Sjkimint mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 20655714Skris struct mlx5_query_cq_mbox_out *out) 207280297Sjkim{ 208280297Sjkim struct mlx5_query_cq_mbox_in in; 209280297Sjkim int err; 210280297Sjkim 211280297Sjkim memset(&in, 0, sizeof(in)); 212280297Sjkim memset(out, 0, sizeof(*out)); 213280297Sjkim 214280297Sjkim in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ); 215280297Sjkim in.cqn = cpu_to_be32(cq->cqn); 216280297Sjkim err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out)); 217280297Sjkim if (err) 218280297Sjkim return err; 219280297Sjkim 220280297Sjkim if (out->hdr.status) 221280297Sjkim return mlx5_cmd_status_to_err(&out->hdr); 222280297Sjkim 223280297Sjkim return err; 224280297Sjkim} 225280297SjkimEXPORT_SYMBOL(mlx5_core_query_cq); 226280297Sjkim 227280297Sjkim 228280297Sjkimint mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 229280297Sjkim struct mlx5_modify_cq_mbox_in *in, int in_sz) 230280297Sjkim{ 231280297Sjkim struct mlx5_modify_cq_mbox_out out; 232280297Sjkim int err; 233280297Sjkim 234280297Sjkim memset(&out, 0, sizeof(out)); 235280297Sjkim in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ); 236280297Sjkim err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out)); 237280297Sjkim if (err) 238280297Sjkim return err; 239280297Sjkim 240280297Sjkim if (out.hdr.status) 241280297Sjkim return mlx5_cmd_status_to_err(&out.hdr); 242280297Sjkim 243280297Sjkim return 0; 244280297Sjkim} 245280297SjkimEXPORT_SYMBOL(mlx5_core_modify_cq); 246280297Sjkim 247280297Sjkimint mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 248280297Sjkim struct mlx5_core_cq *cq, 249280297Sjkim u16 cq_period, 25055714Skris u16 cq_max_count) 251280297Sjkim{ 252280297Sjkim struct mlx5_modify_cq_mbox_in in; 253280297Sjkim 254280297Sjkim memset(&in, 0, sizeof(in)); 255280297Sjkim 256280297Sjkim in.cqn = cpu_to_be32(cq->cqn); 257280297Sjkim in.ctx.cq_period = cpu_to_be16(cq_period); 258280297Sjkim in.ctx.cq_max_count = cpu_to_be16(cq_max_count); 259280297Sjkim in.field_select = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD | 260280297Sjkim MLX5_CQ_MODIFY_COUNT); 261280297Sjkim 262280297Sjkim return mlx5_core_modify_cq(dev, cq, &in, sizeof(in)); 263280297Sjkim} 264280297Sjkim 265280297Sjkimint mlx5_init_cq_table(struct mlx5_core_dev *dev) 266280297Sjkim{ 267280297Sjkim struct mlx5_cq_table *table = &dev->priv.cq_table; 268280297Sjkim int err; 26955714Skris int x; 27055714Skris 271280297Sjkim spin_lock_init(&table->lock); 272280297Sjkim for (x = 0; x != MLX5_CQ_LINEAR_ARRAY_SIZE; x++) 273280297Sjkim spin_lock_init(&table->linear_array[x].lock); 274280297Sjkim INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 275280297Sjkim err = 0; 27655714Skris 277280297Sjkim return err; 278280297Sjkim} 279280297Sjkim 280280297Sjkimvoid mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 281280297Sjkim{ 282280297Sjkim} 283280297Sjkim