1290650Shselasky/*- 2337115Shselasky * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3290650Shselasky * 4290650Shselasky * Redistribution and use in source and binary forms, with or without 5290650Shselasky * modification, are permitted provided that the following conditions 6290650Shselasky * are met: 7290650Shselasky * 1. Redistributions of source code must retain the above copyright 8290650Shselasky * notice, this list of conditions and the following disclaimer. 9290650Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10290650Shselasky * notice, this list of conditions and the following disclaimer in the 11290650Shselasky * documentation and/or other materials provided with the distribution. 12290650Shselasky * 13290650Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14290650Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15290650Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16290650Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17290650Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18290650Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19290650Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20290650Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21290650Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22290650Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23290650Shselasky * SUCH DAMAGE. 24290650Shselasky * 25290650Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_cq.c 368222 2020-12-01 12:52:15Z hselasky $ 26290650Shselasky */ 27290650Shselasky 28290650Shselasky#include <linux/kernel.h> 29290650Shselasky#include <linux/module.h> 30290650Shselasky#include <linux/hardirq.h> 31290650Shselasky#include <dev/mlx5/driver.h> 32290650Shselasky#include <rdma/ib_verbs.h> 33290650Shselasky#include <dev/mlx5/cq.h> 34290650Shselasky#include "mlx5_core.h" 35290650Shselasky 36290650Shselaskyvoid mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 37290650Shselasky{ 38290650Shselasky struct mlx5_core_cq *cq; 39290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 40290650Shselasky 41290650Shselasky if (cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 42290650Shselasky struct mlx5_cq_linear_array_entry *entry; 43290650Shselasky 44290650Shselasky entry = &table->linear_array[cqn]; 45290650Shselasky spin_lock(&entry->lock); 46290650Shselasky cq = entry->cq; 47290650Shselasky if (cq == NULL) { 48290650Shselasky mlx5_core_warn(dev, 49290650Shselasky "Completion event for bogus CQ 0x%x\n", cqn); 50290650Shselasky } else { 51290650Shselasky ++cq->arm_sn; 52290650Shselasky cq->comp(cq); 53290650Shselasky } 54290650Shselasky spin_unlock(&entry->lock); 55290650Shselasky return; 56290650Shselasky } 57290650Shselasky 58290650Shselasky spin_lock(&table->lock); 59290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 60290650Shselasky if (likely(cq)) 61290650Shselasky atomic_inc(&cq->refcount); 62290650Shselasky spin_unlock(&table->lock); 63290650Shselasky 64290650Shselasky if (!cq) { 65290650Shselasky mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn); 66290650Shselasky return; 67290650Shselasky } 68290650Shselasky 69290650Shselasky ++cq->arm_sn; 70290650Shselasky 71290650Shselasky cq->comp(cq); 72290650Shselasky 73290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 74290650Shselasky complete(&cq->free); 75290650Shselasky} 76290650Shselasky 77290650Shselaskyvoid mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 78290650Shselasky{ 79290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 80290650Shselasky struct mlx5_core_cq *cq; 81290650Shselasky 82290650Shselasky spin_lock(&table->lock); 83290650Shselasky 84290650Shselasky cq = radix_tree_lookup(&table->tree, cqn); 85290650Shselasky if (cq) 86290650Shselasky atomic_inc(&cq->refcount); 87290650Shselasky 88290650Shselasky spin_unlock(&table->lock); 89290650Shselasky 90290650Shselasky if (!cq) { 91290650Shselasky mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn); 92290650Shselasky return; 93290650Shselasky } 94290650Shselasky 95290650Shselasky cq->event(cq, event_type); 96290650Shselasky 97290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 98290650Shselasky complete(&cq->free); 99290650Shselasky} 100290650Shselasky 101290650Shselasky 102290650Shselaskyint mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 103368222Shselasky u32 *in, int inlen, u32 *out, int outlen) 104290650Shselasky{ 105331807Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 106331807Shselasky u32 din[MLX5_ST_SZ_DW(destroy_cq_in)] = {0}; 107331807Shselasky u32 dout[MLX5_ST_SZ_DW(destroy_cq_out)] = {0}; 108290650Shselasky int err; 109290650Shselasky 110368222Shselasky memset(out, 0, outlen); 111331807Shselasky MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ); 112368222Shselasky err = mlx5_cmd_exec(dev, in, inlen, out, outlen); 113290650Shselasky if (err) 114290650Shselasky return err; 115290650Shselasky 116331807Shselasky cq->cqn = MLX5_GET(create_cq_out, out, cqn); 117290650Shselasky cq->cons_index = 0; 118290650Shselasky cq->arm_sn = 0; 119290650Shselasky atomic_set(&cq->refcount, 1); 120290650Shselasky init_completion(&cq->free); 121290650Shselasky 122290650Shselasky spin_lock_irq(&table->lock); 123290650Shselasky err = radix_tree_insert(&table->tree, cq->cqn, cq); 124290650Shselasky spin_unlock_irq(&table->lock); 125290650Shselasky if (err) 126290650Shselasky goto err_cmd; 127290650Shselasky 128290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 129290650Shselasky struct mlx5_cq_linear_array_entry *entry; 130290650Shselasky 131290650Shselasky entry = &table->linear_array[cq->cqn]; 132290650Shselasky spin_lock_irq(&entry->lock); 133290650Shselasky entry->cq = cq; 134290650Shselasky spin_unlock_irq(&entry->lock); 135290650Shselasky } 136290650Shselasky 137290650Shselasky cq->pid = curthread->td_proc->p_pid; 138290650Shselasky 139290650Shselasky return 0; 140290650Shselasky 141290650Shselaskyerr_cmd: 142331807Shselasky MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ); 143331807Shselasky MLX5_SET(destroy_cq_in, din, cqn, cq->cqn); 144331807Shselasky mlx5_cmd_exec(dev, din, sizeof(din), dout, sizeof(dout)); 145290650Shselasky return err; 146290650Shselasky} 147290650ShselaskyEXPORT_SYMBOL(mlx5_core_create_cq); 148290650Shselasky 149290650Shselaskyint mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 150290650Shselasky{ 151290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 152331807Shselasky u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {0}; 153331807Shselasky u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {0}; 154290650Shselasky struct mlx5_core_cq *tmp; 155290650Shselasky int err; 156290650Shselasky 157290650Shselasky if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) { 158290650Shselasky struct mlx5_cq_linear_array_entry *entry; 159290650Shselasky 160290650Shselasky entry = &table->linear_array[cq->cqn]; 161290650Shselasky spin_lock_irq(&entry->lock); 162290650Shselasky entry->cq = NULL; 163290650Shselasky spin_unlock_irq(&entry->lock); 164290650Shselasky } 165290650Shselasky 166290650Shselasky spin_lock_irq(&table->lock); 167290650Shselasky tmp = radix_tree_delete(&table->tree, cq->cqn); 168290650Shselasky spin_unlock_irq(&table->lock); 169290650Shselasky if (!tmp) { 170290650Shselasky mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 171290650Shselasky return -EINVAL; 172290650Shselasky } 173290650Shselasky if (tmp != cq) { 174290650Shselasky mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn); 175290650Shselasky return -EINVAL; 176290650Shselasky } 177290650Shselasky 178331807Shselasky MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ); 179331807Shselasky MLX5_SET(destroy_cq_in, in, cqn, cq->cqn); 180331807Shselasky err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 181290650Shselasky if (err) 182306233Shselasky goto out; 183290650Shselasky 184290650Shselasky synchronize_irq(cq->irqn); 185290650Shselasky 186290650Shselasky if (atomic_dec_and_test(&cq->refcount)) 187290650Shselasky complete(&cq->free); 188290650Shselasky wait_for_completion(&cq->free); 189290650Shselasky 190306233Shselaskyout: 191306233Shselasky 192306233Shselasky return err; 193290650Shselasky} 194290650ShselaskyEXPORT_SYMBOL(mlx5_core_destroy_cq); 195290650Shselasky 196290650Shselaskyint mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 197331807Shselasky u32 *out, int outlen) 198290650Shselasky{ 199331807Shselasky u32 in[MLX5_ST_SZ_DW(query_cq_in)] = {0}; 200290650Shselasky 201331807Shselasky MLX5_SET(query_cq_in, in, opcode, MLX5_CMD_OP_QUERY_CQ); 202331807Shselasky MLX5_SET(query_cq_in, in, cqn, cq->cqn); 203290650Shselasky 204331807Shselasky return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); 205290650Shselasky} 206290650ShselaskyEXPORT_SYMBOL(mlx5_core_query_cq); 207290650Shselasky 208290650Shselasky 209290650Shselaskyint mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 210331807Shselasky u32 *in, int inlen) 211290650Shselasky{ 212331807Shselasky u32 out[MLX5_ST_SZ_DW(modify_cq_out)] = {0}; 213290650Shselasky 214331807Shselasky MLX5_SET(modify_cq_in, in, opcode, MLX5_CMD_OP_MODIFY_CQ); 215331807Shselasky return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 216290650Shselasky} 217290650ShselaskyEXPORT_SYMBOL(mlx5_core_modify_cq); 218290650Shselasky 219290650Shselaskyint mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 220290650Shselasky struct mlx5_core_cq *cq, 221290650Shselasky u16 cq_period, 222290650Shselasky u16 cq_max_count) 223290650Shselasky{ 224331807Shselasky u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {0}; 225331807Shselasky void *cqc; 226290650Shselasky 227331807Shselasky MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 228331807Shselasky cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 229331807Shselasky MLX5_SET(cqc, cqc, cq_period, cq_period); 230331807Shselasky MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 231331807Shselasky MLX5_SET(modify_cq_in, in, 232331807Shselasky modify_field_select_resize_field_select.modify_field_select.modify_field_select, 233331807Shselasky MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT); 234290650Shselasky 235331807Shselasky return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 236290650Shselasky} 237290650Shselasky 238321995Shselaskyint mlx5_core_modify_cq_moderation_mode(struct mlx5_core_dev *dev, 239321995Shselasky struct mlx5_core_cq *cq, 240321995Shselasky u16 cq_period, 241321995Shselasky u16 cq_max_count, 242321995Shselasky u8 cq_mode) 243321995Shselasky{ 244331807Shselasky u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {0}; 245331807Shselasky void *cqc; 246321995Shselasky 247331807Shselasky MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 248331807Shselasky cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 249331807Shselasky MLX5_SET(cqc, cqc, cq_period, cq_period); 250331807Shselasky MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 251331807Shselasky MLX5_SET(cqc, cqc, cq_period_mode, cq_mode); 252331807Shselasky MLX5_SET(modify_cq_in, in, 253331807Shselasky modify_field_select_resize_field_select.modify_field_select.modify_field_select, 254331807Shselasky MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE); 255321995Shselasky 256331807Shselasky return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 257321995Shselasky} 258321995Shselasky 259290650Shselaskyint mlx5_init_cq_table(struct mlx5_core_dev *dev) 260290650Shselasky{ 261290650Shselasky struct mlx5_cq_table *table = &dev->priv.cq_table; 262290650Shselasky int err; 263290650Shselasky int x; 264290650Shselasky 265331580Shselasky memset(table, 0, sizeof(*table)); 266290650Shselasky spin_lock_init(&table->lock); 267290650Shselasky for (x = 0; x != MLX5_CQ_LINEAR_ARRAY_SIZE; x++) 268290650Shselasky spin_lock_init(&table->linear_array[x].lock); 269290650Shselasky INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 270290650Shselasky err = 0; 271290650Shselasky 272290650Shselasky return err; 273290650Shselasky} 274290650Shselasky 275290650Shselaskyvoid mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 276290650Shselasky{ 277290650Shselasky} 278