1219820Sjeff/* 2219820Sjeff * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3219820Sjeff * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4219820Sjeff * 5219820Sjeff * This software is available to you under a choice of one of two 6219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 7219820Sjeff * General Public License (GPL) Version 2, available from the file 8219820Sjeff * COPYING in the main directory of this source tree, or the 9219820Sjeff * OpenIB.org BSD license below: 10219820Sjeff * 11219820Sjeff * Redistribution and use in source and binary forms, with or 12219820Sjeff * without modification, are permitted provided that the following 13219820Sjeff * conditions are met: 14219820Sjeff * 15219820Sjeff * - Redistributions of source code must retain the above 16219820Sjeff * copyright notice, this list of conditions and the following 17219820Sjeff * disclaimer. 18219820Sjeff * 19219820Sjeff * - Redistributions in binary form must reproduce the above 20219820Sjeff * copyright notice, this list of conditions and the following 21219820Sjeff * disclaimer in the documentation and/or other materials 22219820Sjeff * provided with the distribution. 23219820Sjeff * 24219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31219820Sjeff * SOFTWARE. 32219820Sjeff */ 33219820Sjeff 34219820Sjeff#include <linux/init.h> 35219820Sjeff 36219820Sjeff#include <linux/mlx4/cmd.h> 37255932Salfred#include <linux/gfp.h> 38219820Sjeff 39219820Sjeff#include "mlx4.h" 40219820Sjeff#include "icm.h" 41219820Sjeff 42219820Sjeffvoid mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type) 43219820Sjeff{ 44219820Sjeff struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 45219820Sjeff struct mlx4_srq *srq; 46219820Sjeff 47219820Sjeff spin_lock(&srq_table->lock); 48219820Sjeff 49255932Salfred srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1)); 50219820Sjeff if (srq) 51219820Sjeff atomic_inc(&srq->refcount); 52219820Sjeff 53219820Sjeff spin_unlock(&srq_table->lock); 54219820Sjeff 55219820Sjeff if (!srq) { 56219820Sjeff mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn); 57219820Sjeff return; 58219820Sjeff } 59219820Sjeff 60219820Sjeff srq->event(srq, event_type); 61219820Sjeff 62219820Sjeff if (atomic_dec_and_test(&srq->refcount)) 63219820Sjeff complete(&srq->free); 64219820Sjeff} 65219820Sjeff 66219820Sjeffstatic int mlx4_SW2HW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 67219820Sjeff int srq_num) 68219820Sjeff{ 69255932Salfred return mlx4_cmd(dev, mailbox->dma, srq_num, 0, 70255932Salfred MLX4_CMD_SW2HW_SRQ, MLX4_CMD_TIME_CLASS_A, 71255932Salfred MLX4_CMD_WRAPPED); 72219820Sjeff} 73219820Sjeff 74219820Sjeffstatic int mlx4_HW2SW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 75219820Sjeff int srq_num) 76219820Sjeff{ 77219820Sjeff return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, srq_num, 78219820Sjeff mailbox ? 0 : 1, MLX4_CMD_HW2SW_SRQ, 79255932Salfred MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 80219820Sjeff} 81219820Sjeff 82219820Sjeffstatic int mlx4_ARM_SRQ(struct mlx4_dev *dev, int srq_num, int limit_watermark) 83219820Sjeff{ 84219820Sjeff return mlx4_cmd(dev, limit_watermark, srq_num, 0, MLX4_CMD_ARM_SRQ, 85255932Salfred MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 86219820Sjeff} 87219820Sjeff 88219820Sjeffstatic int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 89219820Sjeff int srq_num) 90219820Sjeff{ 91219820Sjeff return mlx4_cmd_box(dev, 0, mailbox->dma, srq_num, 0, MLX4_CMD_QUERY_SRQ, 92255932Salfred MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 93219820Sjeff} 94219820Sjeff 95255932Salfredint __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) 96219820Sjeff{ 97219820Sjeff struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 98219820Sjeff int err; 99219820Sjeff 100255932Salfred 101255932Salfred *srqn = mlx4_bitmap_alloc(&srq_table->bitmap); 102255932Salfred if (*srqn == -1) 103219820Sjeff return -ENOMEM; 104219820Sjeff 105255932Salfred err = mlx4_table_get(dev, &srq_table->table, *srqn); 106219820Sjeff if (err) 107219820Sjeff goto err_out; 108219820Sjeff 109255932Salfred err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn); 110219820Sjeff if (err) 111219820Sjeff goto err_put; 112255932Salfred return 0; 113219820Sjeff 114255932Salfrederr_put: 115255932Salfred mlx4_table_put(dev, &srq_table->table, *srqn); 116255932Salfred 117255932Salfrederr_out: 118255932Salfred mlx4_bitmap_free(&srq_table->bitmap, *srqn); 119255932Salfred return err; 120255932Salfred} 121255932Salfred 122255932Salfredstatic int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn) 123255932Salfred{ 124255932Salfred u64 out_param; 125255932Salfred int err; 126255932Salfred 127255932Salfred if (mlx4_is_mfunc(dev)) { 128255932Salfred err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ, 129255932Salfred RES_OP_RESERVE_AND_MAP, 130255932Salfred MLX4_CMD_ALLOC_RES, 131255932Salfred MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 132255932Salfred if (!err) 133255932Salfred *srqn = get_param_l(&out_param); 134255932Salfred 135255932Salfred return err; 136255932Salfred } 137255932Salfred return __mlx4_srq_alloc_icm(dev, srqn); 138255932Salfred} 139255932Salfred 140255932Salfredvoid __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) 141255932Salfred{ 142255932Salfred struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 143255932Salfred 144255932Salfred mlx4_table_put(dev, &srq_table->cmpt_table, srqn); 145255932Salfred mlx4_table_put(dev, &srq_table->table, srqn); 146255932Salfred mlx4_bitmap_free(&srq_table->bitmap, srqn); 147255932Salfred} 148255932Salfred 149255932Salfredstatic void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn) 150255932Salfred{ 151255932Salfred u64 in_param = 0; 152255932Salfred 153255932Salfred if (mlx4_is_mfunc(dev)) { 154255932Salfred set_param_l(&in_param, srqn); 155255932Salfred if (mlx4_cmd(dev, in_param, RES_SRQ, RES_OP_RESERVE_AND_MAP, 156255932Salfred MLX4_CMD_FREE_RES, 157255932Salfred MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 158255932Salfred mlx4_warn(dev, "Failed freeing cq:%d\n", srqn); 159255932Salfred return; 160255932Salfred } 161255932Salfred __mlx4_srq_free_icm(dev, srqn); 162255932Salfred} 163255932Salfred 164255932Salfredint mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, 165255932Salfred struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq) 166255932Salfred{ 167255932Salfred struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 168255932Salfred struct mlx4_cmd_mailbox *mailbox; 169255932Salfred struct mlx4_srq_context *srq_context; 170255932Salfred u64 mtt_addr; 171255932Salfred int err; 172255932Salfred 173255932Salfred err = mlx4_srq_alloc_icm(dev, &srq->srqn); 174255932Salfred if (err) 175255932Salfred return err; 176255932Salfred 177219820Sjeff spin_lock_irq(&srq_table->lock); 178255932Salfred err = radix_tree_insert(&srq_table->tree, srq->srqn, srq); 179219820Sjeff spin_unlock_irq(&srq_table->lock); 180219820Sjeff if (err) 181255932Salfred goto err_icm; 182219820Sjeff 183219820Sjeff mailbox = mlx4_alloc_cmd_mailbox(dev); 184219820Sjeff if (IS_ERR(mailbox)) { 185219820Sjeff err = PTR_ERR(mailbox); 186219820Sjeff goto err_radix; 187219820Sjeff } 188219820Sjeff 189219820Sjeff srq_context = mailbox->buf; 190219820Sjeff memset(srq_context, 0, sizeof *srq_context); 191219820Sjeff 192219820Sjeff srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) | 193219820Sjeff srq->srqn); 194219820Sjeff srq_context->logstride = srq->wqe_shift - 4; 195255932Salfred srq_context->xrcd = cpu_to_be16(xrcd); 196219820Sjeff srq_context->pg_offset_cqn = cpu_to_be32(cqn & 0xffffff); 197219820Sjeff srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; 198219820Sjeff 199219820Sjeff mtt_addr = mlx4_mtt_addr(dev, mtt); 200219820Sjeff srq_context->mtt_base_addr_h = mtt_addr >> 32; 201219820Sjeff srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); 202219820Sjeff srq_context->pd = cpu_to_be32(pdn); 203219820Sjeff srq_context->db_rec_addr = cpu_to_be64(db_rec); 204219820Sjeff 205219820Sjeff err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn); 206219820Sjeff mlx4_free_cmd_mailbox(dev, mailbox); 207219820Sjeff if (err) 208219820Sjeff goto err_radix; 209219820Sjeff 210219820Sjeff atomic_set(&srq->refcount, 1); 211219820Sjeff init_completion(&srq->free); 212219820Sjeff 213219820Sjeff return 0; 214219820Sjeff 215219820Sjefferr_radix: 216219820Sjeff spin_lock_irq(&srq_table->lock); 217255932Salfred radix_tree_delete(&srq_table->tree, srq->srqn); 218219820Sjeff spin_unlock_irq(&srq_table->lock); 219219820Sjeff 220255932Salfrederr_icm: 221255932Salfred mlx4_srq_free_icm(dev, srq->srqn); 222219820Sjeff return err; 223219820Sjeff} 224219820SjeffEXPORT_SYMBOL_GPL(mlx4_srq_alloc); 225219820Sjeff 226255932Salfredvoid mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq) 227219820Sjeff{ 228255932Salfred struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 229219820Sjeff int err; 230219820Sjeff 231219820Sjeff err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn); 232219820Sjeff if (err) 233219820Sjeff mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn); 234219820Sjeff 235219820Sjeff spin_lock_irq(&srq_table->lock); 236255932Salfred radix_tree_delete(&srq_table->tree, srq->srqn); 237219820Sjeff spin_unlock_irq(&srq_table->lock); 238219820Sjeff 239219820Sjeff if (atomic_dec_and_test(&srq->refcount)) 240219820Sjeff complete(&srq->free); 241219820Sjeff wait_for_completion(&srq->free); 242219820Sjeff 243255932Salfred mlx4_srq_free_icm(dev, srq->srqn); 244219820Sjeff} 245219820SjeffEXPORT_SYMBOL_GPL(mlx4_srq_free); 246219820Sjeff 247219820Sjeffint mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark) 248219820Sjeff{ 249219820Sjeff return mlx4_ARM_SRQ(dev, srq->srqn, limit_watermark); 250219820Sjeff} 251219820SjeffEXPORT_SYMBOL_GPL(mlx4_srq_arm); 252219820Sjeff 253219820Sjeffint mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark) 254219820Sjeff{ 255219820Sjeff struct mlx4_cmd_mailbox *mailbox; 256219820Sjeff struct mlx4_srq_context *srq_context; 257219820Sjeff int err; 258219820Sjeff 259219820Sjeff mailbox = mlx4_alloc_cmd_mailbox(dev); 260219820Sjeff if (IS_ERR(mailbox)) 261219820Sjeff return PTR_ERR(mailbox); 262219820Sjeff 263219820Sjeff srq_context = mailbox->buf; 264219820Sjeff 265219820Sjeff err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn); 266219820Sjeff if (err) 267219820Sjeff goto err_out; 268219820Sjeff *limit_watermark = be16_to_cpu(srq_context->limit_watermark); 269219820Sjeff 270219820Sjefferr_out: 271219820Sjeff mlx4_free_cmd_mailbox(dev, mailbox); 272219820Sjeff return err; 273219820Sjeff} 274219820SjeffEXPORT_SYMBOL_GPL(mlx4_srq_query); 275219820Sjeff 276219820Sjeffint mlx4_init_srq_table(struct mlx4_dev *dev) 277219820Sjeff{ 278219820Sjeff struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; 279219820Sjeff int err; 280219820Sjeff 281219820Sjeff spin_lock_init(&srq_table->lock); 282255932Salfred INIT_RADIX_TREE(&srq_table->tree, GFP_ATOMIC); 283255932Salfred if (mlx4_is_slave(dev)) 284255932Salfred return 0; 285219820Sjeff 286219820Sjeff err = mlx4_bitmap_init(&srq_table->bitmap, dev->caps.num_srqs, 287219820Sjeff dev->caps.num_srqs - 1, dev->caps.reserved_srqs, 0); 288219820Sjeff if (err) 289219820Sjeff return err; 290219820Sjeff 291219820Sjeff return 0; 292219820Sjeff} 293219820Sjeff 294219820Sjeffvoid mlx4_cleanup_srq_table(struct mlx4_dev *dev) 295219820Sjeff{ 296255932Salfred if (mlx4_is_slave(dev)) 297255932Salfred return; 298219820Sjeff mlx4_bitmap_cleanup(&mlx4_priv(dev)->srq_table.bitmap); 299219820Sjeff} 300