1329222Shselasky/* 2329222Shselasky * Copyright (c) 2005 Cisco Systems. All rights reserved. 3329222Shselasky * 4329222Shselasky * This software is available to you under a choice of one of two 5329222Shselasky * licenses. You may choose to be licensed under the terms of the GNU 6329222Shselasky * General Public License (GPL) Version 2, available from the file 7329222Shselasky * COPYING in the main directory of this source tree, or the 8329222Shselasky * OpenIB.org BSD license below: 9329222Shselasky * 10329222Shselasky * Redistribution and use in source and binary forms, with or 11329222Shselasky * without modification, are permitted provided that the following 12329222Shselasky * conditions are met: 13329222Shselasky * 14329222Shselasky * - Redistributions of source code must retain the above 15329222Shselasky * copyright notice, this list of conditions and the following 16329222Shselasky * disclaimer. 17329222Shselasky * 18329222Shselasky * - Redistributions in binary form must reproduce the above 19329222Shselasky * copyright notice, this list of conditions and the following 20329222Shselasky * disclaimer in the documentation and/or other materials 21329222Shselasky * provided with the distribution. 22329222Shselasky * 23329222Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24329222Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25329222Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26329222Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27329222Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28329222Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29329222Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30329222Shselasky * SOFTWARE. 31329222Shselasky */ 32329222Shselasky 33329222Shselasky#include <linux/slab.h> 34329222Shselasky#include <linux/string.h> 35329222Shselasky#include <linux/sched.h> 36329222Shselasky 37329222Shselasky#include <asm/io.h> 38329222Shselasky 39329222Shselasky#include "mthca_dev.h" 40329222Shselasky#include "mthca_cmd.h" 41329222Shselasky#include "mthca_memfree.h" 42329222Shselasky#include "mthca_wqe.h" 43329222Shselasky 44329222Shselaskyenum { 45329222Shselasky MTHCA_MAX_DIRECT_SRQ_SIZE = 4 * PAGE_SIZE 46329222Shselasky}; 47329222Shselasky 48329222Shselaskystruct mthca_tavor_srq_context { 49329222Shselasky __be64 wqe_base_ds; /* low 6 bits is descriptor size */ 50329222Shselasky __be32 state_pd; 51329222Shselasky __be32 lkey; 52329222Shselasky __be32 uar; 53329222Shselasky __be16 limit_watermark; 54329222Shselasky __be16 wqe_cnt; 55329222Shselasky u32 reserved[2]; 56329222Shselasky}; 57329222Shselasky 58329222Shselaskystruct mthca_arbel_srq_context { 59329222Shselasky __be32 state_logsize_srqn; 60329222Shselasky __be32 lkey; 61329222Shselasky __be32 db_index; 62329222Shselasky __be32 logstride_usrpage; 63329222Shselasky __be64 wqe_base; 64329222Shselasky __be32 eq_pd; 65329222Shselasky __be16 limit_watermark; 66329222Shselasky __be16 wqe_cnt; 67329222Shselasky u16 reserved1; 68329222Shselasky __be16 wqe_counter; 69329222Shselasky u32 reserved2[3]; 70329222Shselasky}; 71329222Shselasky 72329222Shselaskystatic void *get_wqe(struct mthca_srq *srq, int n) 73329222Shselasky{ 74329222Shselasky if (srq->is_direct) 75329222Shselasky return srq->queue.direct.buf + (n << srq->wqe_shift); 76329222Shselasky else 77329222Shselasky return srq->queue.page_list[(n << srq->wqe_shift) >> PAGE_SHIFT].buf + 78329222Shselasky ((n << srq->wqe_shift) & (PAGE_SIZE - 1)); 79329222Shselasky} 80329222Shselasky 81329222Shselasky/* 82329222Shselasky * Return a pointer to the location within a WQE that we're using as a 83329222Shselasky * link when the WQE is in the free list. We use the imm field 84329222Shselasky * because in the Tavor case, posting a WQE may overwrite the next 85329222Shselasky * segment of the previous WQE, but a receive WQE will never touch the 86329222Shselasky * imm field. This avoids corrupting our free list if the previous 87329222Shselasky * WQE has already completed and been put on the free list when we 88329222Shselasky * post the next WQE. 89329222Shselasky */ 90329222Shselaskystatic inline int *wqe_to_link(void *wqe) 91329222Shselasky{ 92329222Shselasky return (int *) (wqe + offsetof(struct mthca_next_seg, imm)); 93329222Shselasky} 94329222Shselasky 95329222Shselaskystatic void mthca_tavor_init_srq_context(struct mthca_dev *dev, 96329222Shselasky struct mthca_pd *pd, 97329222Shselasky struct mthca_srq *srq, 98329222Shselasky struct mthca_tavor_srq_context *context) 99329222Shselasky{ 100329222Shselasky memset(context, 0, sizeof *context); 101329222Shselasky 102329222Shselasky context->wqe_base_ds = cpu_to_be64(1 << (srq->wqe_shift - 4)); 103329222Shselasky context->state_pd = cpu_to_be32(pd->pd_num); 104329222Shselasky context->lkey = cpu_to_be32(srq->mr.ibmr.lkey); 105329222Shselasky 106329222Shselasky if (pd->ibpd.uobject) 107329222Shselasky context->uar = 108329222Shselasky cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index); 109329222Shselasky else 110329222Shselasky context->uar = cpu_to_be32(dev->driver_uar.index); 111329222Shselasky} 112329222Shselasky 113329222Shselaskystatic void mthca_arbel_init_srq_context(struct mthca_dev *dev, 114329222Shselasky struct mthca_pd *pd, 115329222Shselasky struct mthca_srq *srq, 116329222Shselasky struct mthca_arbel_srq_context *context) 117329222Shselasky{ 118329222Shselasky int logsize, max; 119329222Shselasky 120329222Shselasky memset(context, 0, sizeof *context); 121329222Shselasky 122329222Shselasky /* 123329222Shselasky * Put max in a temporary variable to work around gcc bug 124329222Shselasky * triggered by ilog2() on sparc64. 125329222Shselasky */ 126329222Shselasky max = srq->max; 127329222Shselasky logsize = ilog2(max); 128329222Shselasky context->state_logsize_srqn = cpu_to_be32(logsize << 24 | srq->srqn); 129329222Shselasky context->lkey = cpu_to_be32(srq->mr.ibmr.lkey); 130329222Shselasky context->db_index = cpu_to_be32(srq->db_index); 131329222Shselasky context->logstride_usrpage = cpu_to_be32((srq->wqe_shift - 4) << 29); 132329222Shselasky if (pd->ibpd.uobject) 133329222Shselasky context->logstride_usrpage |= 134329222Shselasky cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index); 135329222Shselasky else 136329222Shselasky context->logstride_usrpage |= cpu_to_be32(dev->driver_uar.index); 137329222Shselasky context->eq_pd = cpu_to_be32(MTHCA_EQ_ASYNC << 24 | pd->pd_num); 138329222Shselasky} 139329222Shselasky 140329222Shselaskystatic void mthca_free_srq_buf(struct mthca_dev *dev, struct mthca_srq *srq) 141329222Shselasky{ 142329222Shselasky mthca_buf_free(dev, srq->max << srq->wqe_shift, &srq->queue, 143329222Shselasky srq->is_direct, &srq->mr); 144329222Shselasky kfree(srq->wrid); 145329222Shselasky} 146329222Shselasky 147329222Shselaskystatic int mthca_alloc_srq_buf(struct mthca_dev *dev, struct mthca_pd *pd, 148329222Shselasky struct mthca_srq *srq) 149329222Shselasky{ 150329222Shselasky struct mthca_data_seg *scatter; 151329222Shselasky void *wqe; 152329222Shselasky int err; 153329222Shselasky int i; 154329222Shselasky 155329222Shselasky if (pd->ibpd.uobject) 156329222Shselasky return 0; 157329222Shselasky 158329222Shselasky srq->wrid = kmalloc(srq->max * sizeof (u64), GFP_KERNEL); 159329222Shselasky if (!srq->wrid) 160329222Shselasky return -ENOMEM; 161329222Shselasky 162329222Shselasky err = mthca_buf_alloc(dev, srq->max << srq->wqe_shift, 163329222Shselasky MTHCA_MAX_DIRECT_SRQ_SIZE, 164329222Shselasky &srq->queue, &srq->is_direct, pd, 1, &srq->mr); 165329222Shselasky if (err) { 166329222Shselasky kfree(srq->wrid); 167329222Shselasky return err; 168329222Shselasky } 169329222Shselasky 170329222Shselasky /* 171329222Shselasky * Now initialize the SRQ buffer so that all of the WQEs are 172329222Shselasky * linked into the list of free WQEs. In addition, set the 173329222Shselasky * scatter list L_Keys to the sentry value of 0x100. 174329222Shselasky */ 175329222Shselasky for (i = 0; i < srq->max; ++i) { 176329222Shselasky struct mthca_next_seg *next; 177329222Shselasky 178329222Shselasky next = wqe = get_wqe(srq, i); 179329222Shselasky 180329222Shselasky if (i < srq->max - 1) { 181329222Shselasky *wqe_to_link(wqe) = i + 1; 182329222Shselasky next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); 183329222Shselasky } else { 184329222Shselasky *wqe_to_link(wqe) = -1; 185329222Shselasky next->nda_op = 0; 186329222Shselasky } 187329222Shselasky 188329222Shselasky for (scatter = wqe + sizeof (struct mthca_next_seg); 189329222Shselasky (void *) scatter < wqe + (1 << srq->wqe_shift); 190329222Shselasky ++scatter) 191329222Shselasky scatter->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); 192329222Shselasky } 193329222Shselasky 194329222Shselasky srq->last = get_wqe(srq, srq->max - 1); 195329222Shselasky 196329222Shselasky return 0; 197329222Shselasky} 198329222Shselasky 199329222Shselaskyint mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, 200329222Shselasky struct ib_srq_attr *attr, struct mthca_srq *srq) 201329222Shselasky{ 202329222Shselasky struct mthca_mailbox *mailbox; 203329222Shselasky int ds; 204329222Shselasky int err; 205329222Shselasky 206329222Shselasky /* Sanity check SRQ size before proceeding */ 207329222Shselasky if (attr->max_wr > dev->limits.max_srq_wqes || 208329222Shselasky attr->max_sge > dev->limits.max_srq_sge) 209329222Shselasky return -EINVAL; 210329222Shselasky 211329222Shselasky srq->max = attr->max_wr; 212329222Shselasky srq->max_gs = attr->max_sge; 213329222Shselasky srq->counter = 0; 214329222Shselasky 215329222Shselasky if (mthca_is_memfree(dev)) 216329222Shselasky srq->max = roundup_pow_of_two(srq->max + 1); 217329222Shselasky else 218329222Shselasky srq->max = srq->max + 1; 219329222Shselasky 220329222Shselasky ds = max(64UL, 221329222Shselasky roundup_pow_of_two(sizeof (struct mthca_next_seg) + 222329222Shselasky srq->max_gs * sizeof (struct mthca_data_seg))); 223329222Shselasky 224329222Shselasky if (!mthca_is_memfree(dev) && (ds > dev->limits.max_desc_sz)) 225329222Shselasky return -EINVAL; 226329222Shselasky 227329222Shselasky srq->wqe_shift = ilog2(ds); 228329222Shselasky 229329222Shselasky srq->srqn = mthca_alloc(&dev->srq_table.alloc); 230329222Shselasky if (srq->srqn == -1) 231329222Shselasky return -ENOMEM; 232329222Shselasky 233329222Shselasky if (mthca_is_memfree(dev)) { 234329222Shselasky err = mthca_table_get(dev, dev->srq_table.table, srq->srqn); 235329222Shselasky if (err) 236329222Shselasky goto err_out; 237329222Shselasky 238329222Shselasky if (!pd->ibpd.uobject) { 239329222Shselasky srq->db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SRQ, 240329222Shselasky srq->srqn, &srq->db); 241329222Shselasky if (srq->db_index < 0) { 242329222Shselasky err = -ENOMEM; 243329222Shselasky goto err_out_icm; 244329222Shselasky } 245329222Shselasky } 246329222Shselasky } 247329222Shselasky 248329222Shselasky mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 249329222Shselasky if (IS_ERR(mailbox)) { 250329222Shselasky err = PTR_ERR(mailbox); 251329222Shselasky goto err_out_db; 252329222Shselasky } 253329222Shselasky 254329222Shselasky err = mthca_alloc_srq_buf(dev, pd, srq); 255329222Shselasky if (err) 256329222Shselasky goto err_out_mailbox; 257329222Shselasky 258329222Shselasky spin_lock_init(&srq->lock); 259329222Shselasky srq->refcount = 1; 260329222Shselasky init_waitqueue_head(&srq->wait); 261329222Shselasky mutex_init(&srq->mutex); 262329222Shselasky 263329222Shselasky if (mthca_is_memfree(dev)) 264329222Shselasky mthca_arbel_init_srq_context(dev, pd, srq, mailbox->buf); 265329222Shselasky else 266329222Shselasky mthca_tavor_init_srq_context(dev, pd, srq, mailbox->buf); 267329222Shselasky 268329222Shselasky err = mthca_SW2HW_SRQ(dev, mailbox, srq->srqn); 269329222Shselasky 270329222Shselasky if (err) { 271329222Shselasky mthca_warn(dev, "SW2HW_SRQ failed (%d)\n", err); 272329222Shselasky goto err_out_free_buf; 273329222Shselasky } 274329222Shselasky 275329222Shselasky spin_lock_irq(&dev->srq_table.lock); 276329222Shselasky if (mthca_array_set(&dev->srq_table.srq, 277329222Shselasky srq->srqn & (dev->limits.num_srqs - 1), 278329222Shselasky srq)) { 279329222Shselasky spin_unlock_irq(&dev->srq_table.lock); 280329222Shselasky goto err_out_free_srq; 281329222Shselasky } 282329222Shselasky spin_unlock_irq(&dev->srq_table.lock); 283329222Shselasky 284329222Shselasky mthca_free_mailbox(dev, mailbox); 285329222Shselasky 286329222Shselasky srq->first_free = 0; 287329222Shselasky srq->last_free = srq->max - 1; 288329222Shselasky 289329222Shselasky attr->max_wr = srq->max - 1; 290329222Shselasky attr->max_sge = srq->max_gs; 291329222Shselasky 292329222Shselasky return 0; 293329222Shselasky 294329222Shselaskyerr_out_free_srq: 295329222Shselasky err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn); 296329222Shselasky if (err) 297329222Shselasky mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); 298329222Shselasky 299329222Shselaskyerr_out_free_buf: 300329222Shselasky if (!pd->ibpd.uobject) 301329222Shselasky mthca_free_srq_buf(dev, srq); 302329222Shselasky 303329222Shselaskyerr_out_mailbox: 304329222Shselasky mthca_free_mailbox(dev, mailbox); 305329222Shselasky 306329222Shselaskyerr_out_db: 307329222Shselasky if (!pd->ibpd.uobject && mthca_is_memfree(dev)) 308329222Shselasky mthca_free_db(dev, MTHCA_DB_TYPE_SRQ, srq->db_index); 309329222Shselasky 310329222Shselaskyerr_out_icm: 311329222Shselasky mthca_table_put(dev, dev->srq_table.table, srq->srqn); 312329222Shselasky 313329222Shselaskyerr_out: 314329222Shselasky mthca_free(&dev->srq_table.alloc, srq->srqn); 315329222Shselasky 316329222Shselasky return err; 317329222Shselasky} 318329222Shselasky 319329222Shselaskystatic inline int get_srq_refcount(struct mthca_dev *dev, struct mthca_srq *srq) 320329222Shselasky{ 321329222Shselasky int c; 322329222Shselasky 323329222Shselasky spin_lock_irq(&dev->srq_table.lock); 324329222Shselasky c = srq->refcount; 325329222Shselasky spin_unlock_irq(&dev->srq_table.lock); 326329222Shselasky 327329222Shselasky return c; 328329222Shselasky} 329329222Shselasky 330329222Shselaskyvoid mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq) 331329222Shselasky{ 332329222Shselasky struct mthca_mailbox *mailbox; 333329222Shselasky int err; 334329222Shselasky 335329222Shselasky mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 336329222Shselasky if (IS_ERR(mailbox)) { 337329222Shselasky mthca_warn(dev, "No memory for mailbox to free SRQ.\n"); 338329222Shselasky return; 339329222Shselasky } 340329222Shselasky 341329222Shselasky err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn); 342329222Shselasky if (err) 343329222Shselasky mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); 344329222Shselasky 345329222Shselasky spin_lock_irq(&dev->srq_table.lock); 346329222Shselasky mthca_array_clear(&dev->srq_table.srq, 347329222Shselasky srq->srqn & (dev->limits.num_srqs - 1)); 348329222Shselasky --srq->refcount; 349329222Shselasky spin_unlock_irq(&dev->srq_table.lock); 350329222Shselasky 351329222Shselasky wait_event(srq->wait, !get_srq_refcount(dev, srq)); 352329222Shselasky 353329222Shselasky if (!srq->ibsrq.uobject) { 354329222Shselasky mthca_free_srq_buf(dev, srq); 355329222Shselasky if (mthca_is_memfree(dev)) 356329222Shselasky mthca_free_db(dev, MTHCA_DB_TYPE_SRQ, srq->db_index); 357329222Shselasky } 358329222Shselasky 359329222Shselasky mthca_table_put(dev, dev->srq_table.table, srq->srqn); 360329222Shselasky mthca_free(&dev->srq_table.alloc, srq->srqn); 361329222Shselasky mthca_free_mailbox(dev, mailbox); 362329222Shselasky} 363329222Shselasky 364329222Shselaskyint mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 365329222Shselasky enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 366329222Shselasky{ 367329222Shselasky struct mthca_dev *dev = to_mdev(ibsrq->device); 368329222Shselasky struct mthca_srq *srq = to_msrq(ibsrq); 369329222Shselasky int ret = 0; 370329222Shselasky 371329222Shselasky /* We don't support resizing SRQs (yet?) */ 372329222Shselasky if (attr_mask & IB_SRQ_MAX_WR) 373329222Shselasky return -EINVAL; 374329222Shselasky 375329222Shselasky if (attr_mask & IB_SRQ_LIMIT) { 376329222Shselasky u32 max_wr = mthca_is_memfree(dev) ? srq->max - 1 : srq->max; 377329222Shselasky if (attr->srq_limit > max_wr) 378329222Shselasky return -EINVAL; 379329222Shselasky 380329222Shselasky mutex_lock(&srq->mutex); 381329222Shselasky ret = mthca_ARM_SRQ(dev, srq->srqn, attr->srq_limit); 382329222Shselasky mutex_unlock(&srq->mutex); 383329222Shselasky } 384329222Shselasky 385329222Shselasky return ret; 386329222Shselasky} 387329222Shselasky 388329222Shselaskyint mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 389329222Shselasky{ 390329222Shselasky struct mthca_dev *dev = to_mdev(ibsrq->device); 391329222Shselasky struct mthca_srq *srq = to_msrq(ibsrq); 392329222Shselasky struct mthca_mailbox *mailbox; 393329222Shselasky struct mthca_arbel_srq_context *arbel_ctx; 394329222Shselasky struct mthca_tavor_srq_context *tavor_ctx; 395329222Shselasky int err; 396329222Shselasky 397329222Shselasky mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 398329222Shselasky if (IS_ERR(mailbox)) 399329222Shselasky return PTR_ERR(mailbox); 400329222Shselasky 401329222Shselasky err = mthca_QUERY_SRQ(dev, srq->srqn, mailbox); 402329222Shselasky if (err) 403329222Shselasky goto out; 404329222Shselasky 405329222Shselasky if (mthca_is_memfree(dev)) { 406329222Shselasky arbel_ctx = mailbox->buf; 407329222Shselasky srq_attr->srq_limit = be16_to_cpu(arbel_ctx->limit_watermark); 408329222Shselasky } else { 409329222Shselasky tavor_ctx = mailbox->buf; 410329222Shselasky srq_attr->srq_limit = be16_to_cpu(tavor_ctx->limit_watermark); 411329222Shselasky } 412329222Shselasky 413329222Shselasky srq_attr->max_wr = srq->max - 1; 414329222Shselasky srq_attr->max_sge = srq->max_gs; 415329222Shselasky 416329222Shselaskyout: 417329222Shselasky mthca_free_mailbox(dev, mailbox); 418329222Shselasky 419329222Shselasky return err; 420329222Shselasky} 421329222Shselasky 422329222Shselaskyvoid mthca_srq_event(struct mthca_dev *dev, u32 srqn, 423329222Shselasky enum ib_event_type event_type) 424329222Shselasky{ 425329222Shselasky struct mthca_srq *srq; 426329222Shselasky struct ib_event event; 427329222Shselasky 428329222Shselasky spin_lock(&dev->srq_table.lock); 429329222Shselasky srq = mthca_array_get(&dev->srq_table.srq, srqn & (dev->limits.num_srqs - 1)); 430329222Shselasky if (srq) 431329222Shselasky ++srq->refcount; 432329222Shselasky spin_unlock(&dev->srq_table.lock); 433329222Shselasky 434329222Shselasky if (!srq) { 435329222Shselasky mthca_warn(dev, "Async event for bogus SRQ %08x\n", srqn); 436329222Shselasky return; 437329222Shselasky } 438329222Shselasky 439329222Shselasky if (!srq->ibsrq.event_handler) 440329222Shselasky goto out; 441329222Shselasky 442329222Shselasky event.device = &dev->ib_dev; 443329222Shselasky event.event = event_type; 444329222Shselasky event.element.srq = &srq->ibsrq; 445329222Shselasky srq->ibsrq.event_handler(&event, srq->ibsrq.srq_context); 446329222Shselasky 447329222Shselaskyout: 448329222Shselasky spin_lock(&dev->srq_table.lock); 449329222Shselasky if (!--srq->refcount) 450329222Shselasky wake_up(&srq->wait); 451329222Shselasky spin_unlock(&dev->srq_table.lock); 452329222Shselasky} 453329222Shselasky 454329222Shselasky/* 455329222Shselasky * This function must be called with IRQs disabled. 456329222Shselasky */ 457329222Shselaskyvoid mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr) 458329222Shselasky{ 459329222Shselasky int ind; 460329222Shselasky struct mthca_next_seg *last_free; 461329222Shselasky 462329222Shselasky ind = wqe_addr >> srq->wqe_shift; 463329222Shselasky 464329222Shselasky spin_lock(&srq->lock); 465329222Shselasky 466329222Shselasky last_free = get_wqe(srq, srq->last_free); 467329222Shselasky *wqe_to_link(last_free) = ind; 468329222Shselasky last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); 469329222Shselasky *wqe_to_link(get_wqe(srq, ind)) = -1; 470329222Shselasky srq->last_free = ind; 471329222Shselasky 472329222Shselasky spin_unlock(&srq->lock); 473329222Shselasky} 474329222Shselasky 475329222Shselaskyint mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 476329222Shselasky struct ib_recv_wr **bad_wr) 477329222Shselasky{ 478329222Shselasky struct mthca_dev *dev = to_mdev(ibsrq->device); 479329222Shselasky struct mthca_srq *srq = to_msrq(ibsrq); 480329222Shselasky unsigned long flags; 481329222Shselasky int err = 0; 482329222Shselasky int first_ind; 483329222Shselasky int ind; 484329222Shselasky int next_ind; 485329222Shselasky int nreq; 486329222Shselasky int i; 487329222Shselasky void *wqe; 488329222Shselasky void *prev_wqe; 489329222Shselasky 490329222Shselasky spin_lock_irqsave(&srq->lock, flags); 491329222Shselasky 492329222Shselasky first_ind = srq->first_free; 493329222Shselasky 494329222Shselasky for (nreq = 0; wr; wr = wr->next) { 495329222Shselasky ind = srq->first_free; 496329222Shselasky wqe = get_wqe(srq, ind); 497329222Shselasky next_ind = *wqe_to_link(wqe); 498329222Shselasky 499329222Shselasky if (unlikely(next_ind < 0)) { 500329222Shselasky mthca_err(dev, "SRQ %06x full\n", srq->srqn); 501329222Shselasky err = -ENOMEM; 502329222Shselasky *bad_wr = wr; 503329222Shselasky break; 504329222Shselasky } 505329222Shselasky 506329222Shselasky prev_wqe = srq->last; 507329222Shselasky srq->last = wqe; 508329222Shselasky 509329222Shselasky ((struct mthca_next_seg *) wqe)->ee_nds = 0; 510329222Shselasky /* flags field will always remain 0 */ 511329222Shselasky 512329222Shselasky wqe += sizeof (struct mthca_next_seg); 513329222Shselasky 514329222Shselasky if (unlikely(wr->num_sge > srq->max_gs)) { 515329222Shselasky err = -EINVAL; 516329222Shselasky *bad_wr = wr; 517329222Shselasky srq->last = prev_wqe; 518329222Shselasky break; 519329222Shselasky } 520329222Shselasky 521329222Shselasky for (i = 0; i < wr->num_sge; ++i) { 522329222Shselasky mthca_set_data_seg(wqe, wr->sg_list + i); 523329222Shselasky wqe += sizeof (struct mthca_data_seg); 524329222Shselasky } 525329222Shselasky 526329222Shselasky if (i < srq->max_gs) 527329222Shselasky mthca_set_data_seg_inval(wqe); 528329222Shselasky 529329222Shselasky ((struct mthca_next_seg *) prev_wqe)->ee_nds = 530329222Shselasky cpu_to_be32(MTHCA_NEXT_DBD); 531329222Shselasky 532329222Shselasky srq->wrid[ind] = wr->wr_id; 533329222Shselasky srq->first_free = next_ind; 534329222Shselasky 535329222Shselasky ++nreq; 536329222Shselasky if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) { 537329222Shselasky nreq = 0; 538329222Shselasky 539329222Shselasky /* 540329222Shselasky * Make sure that descriptors are written 541329222Shselasky * before doorbell is rung. 542329222Shselasky */ 543329222Shselasky wmb(); 544329222Shselasky 545329222Shselasky mthca_write64(first_ind << srq->wqe_shift, srq->srqn << 8, 546329222Shselasky dev->kar + MTHCA_RECEIVE_DOORBELL, 547329222Shselasky MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); 548329222Shselasky 549329222Shselasky first_ind = srq->first_free; 550329222Shselasky } 551329222Shselasky } 552329222Shselasky 553329222Shselasky if (likely(nreq)) { 554329222Shselasky /* 555329222Shselasky * Make sure that descriptors are written before 556329222Shselasky * doorbell is rung. 557329222Shselasky */ 558329222Shselasky wmb(); 559329222Shselasky 560329222Shselasky mthca_write64(first_ind << srq->wqe_shift, (srq->srqn << 8) | nreq, 561329222Shselasky dev->kar + MTHCA_RECEIVE_DOORBELL, 562329222Shselasky MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); 563329222Shselasky } 564329222Shselasky 565329222Shselasky /* 566329222Shselasky * Make sure doorbells don't leak out of SRQ spinlock and 567329222Shselasky * reach the HCA out of order: 568329222Shselasky */ 569329222Shselasky mmiowb(); 570329222Shselasky 571329222Shselasky spin_unlock_irqrestore(&srq->lock, flags); 572329222Shselasky return err; 573329222Shselasky} 574329222Shselasky 575329222Shselaskyint mthca_arbel_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, 576329222Shselasky struct ib_recv_wr **bad_wr) 577329222Shselasky{ 578329222Shselasky struct mthca_dev *dev = to_mdev(ibsrq->device); 579329222Shselasky struct mthca_srq *srq = to_msrq(ibsrq); 580329222Shselasky unsigned long flags; 581329222Shselasky int err = 0; 582329222Shselasky int ind; 583329222Shselasky int next_ind; 584329222Shselasky int nreq; 585329222Shselasky int i; 586329222Shselasky void *wqe; 587329222Shselasky 588329222Shselasky spin_lock_irqsave(&srq->lock, flags); 589329222Shselasky 590329222Shselasky for (nreq = 0; wr; ++nreq, wr = wr->next) { 591329222Shselasky ind = srq->first_free; 592329222Shselasky wqe = get_wqe(srq, ind); 593329222Shselasky next_ind = *wqe_to_link(wqe); 594329222Shselasky 595329222Shselasky if (unlikely(next_ind < 0)) { 596329222Shselasky mthca_err(dev, "SRQ %06x full\n", srq->srqn); 597329222Shselasky err = -ENOMEM; 598329222Shselasky *bad_wr = wr; 599329222Shselasky break; 600329222Shselasky } 601329222Shselasky 602329222Shselasky ((struct mthca_next_seg *) wqe)->ee_nds = 0; 603329222Shselasky /* flags field will always remain 0 */ 604329222Shselasky 605329222Shselasky wqe += sizeof (struct mthca_next_seg); 606329222Shselasky 607329222Shselasky if (unlikely(wr->num_sge > srq->max_gs)) { 608329222Shselasky err = -EINVAL; 609329222Shselasky *bad_wr = wr; 610329222Shselasky break; 611329222Shselasky } 612329222Shselasky 613329222Shselasky for (i = 0; i < wr->num_sge; ++i) { 614329222Shselasky mthca_set_data_seg(wqe, wr->sg_list + i); 615329222Shselasky wqe += sizeof (struct mthca_data_seg); 616329222Shselasky } 617329222Shselasky 618329222Shselasky if (i < srq->max_gs) 619329222Shselasky mthca_set_data_seg_inval(wqe); 620329222Shselasky 621329222Shselasky srq->wrid[ind] = wr->wr_id; 622329222Shselasky srq->first_free = next_ind; 623329222Shselasky } 624329222Shselasky 625329222Shselasky if (likely(nreq)) { 626329222Shselasky srq->counter += nreq; 627329222Shselasky 628329222Shselasky /* 629329222Shselasky * Make sure that descriptors are written before 630329222Shselasky * we write doorbell record. 631329222Shselasky */ 632329222Shselasky wmb(); 633329222Shselasky *srq->db = cpu_to_be32(srq->counter); 634329222Shselasky } 635329222Shselasky 636329222Shselasky spin_unlock_irqrestore(&srq->lock, flags); 637329222Shselasky return err; 638329222Shselasky} 639329222Shselasky 640329222Shselaskyint mthca_max_srq_sge(struct mthca_dev *dev) 641329222Shselasky{ 642329222Shselasky if (mthca_is_memfree(dev)) 643329222Shselasky return dev->limits.max_sg; 644329222Shselasky 645329222Shselasky /* 646329222Shselasky * SRQ allocations are based on powers of 2 for Tavor, 647329222Shselasky * (although they only need to be multiples of 16 bytes). 648329222Shselasky * 649329222Shselasky * Therefore, we need to base the max number of sg entries on 650329222Shselasky * the largest power of 2 descriptor size that is <= to the 651329222Shselasky * actual max WQE descriptor size, rather than return the 652329222Shselasky * max_sg value given by the firmware (which is based on WQE 653329222Shselasky * sizes as multiples of 16, not powers of 2). 654329222Shselasky * 655329222Shselasky * If SRQ implementation is changed for Tavor to be based on 656329222Shselasky * multiples of 16, the calculation below can be deleted and 657329222Shselasky * the FW max_sg value returned. 658329222Shselasky */ 659329222Shselasky return min_t(int, dev->limits.max_sg, 660329222Shselasky ((1 << (fls(dev->limits.max_desc_sz) - 1)) - 661329222Shselasky sizeof (struct mthca_next_seg)) / 662329222Shselasky sizeof (struct mthca_data_seg)); 663329222Shselasky} 664329222Shselasky 665329222Shselaskyint mthca_init_srq_table(struct mthca_dev *dev) 666329222Shselasky{ 667329222Shselasky int err; 668329222Shselasky 669329222Shselasky if (!(dev->mthca_flags & MTHCA_FLAG_SRQ)) 670329222Shselasky return 0; 671329222Shselasky 672329222Shselasky spin_lock_init(&dev->srq_table.lock); 673329222Shselasky 674329222Shselasky err = mthca_alloc_init(&dev->srq_table.alloc, 675329222Shselasky dev->limits.num_srqs, 676329222Shselasky dev->limits.num_srqs - 1, 677329222Shselasky dev->limits.reserved_srqs); 678329222Shselasky if (err) 679329222Shselasky return err; 680329222Shselasky 681329222Shselasky err = mthca_array_init(&dev->srq_table.srq, 682329222Shselasky dev->limits.num_srqs); 683329222Shselasky if (err) 684329222Shselasky mthca_alloc_cleanup(&dev->srq_table.alloc); 685329222Shselasky 686329222Shselasky return err; 687329222Shselasky} 688329222Shselasky 689329222Shselaskyvoid mthca_cleanup_srq_table(struct mthca_dev *dev) 690329222Shselasky{ 691329222Shselasky if (!(dev->mthca_flags & MTHCA_FLAG_SRQ)) 692329222Shselasky return; 693329222Shselasky 694329222Shselasky mthca_array_cleanup(&dev->srq_table.srq, dev->limits.num_srqs); 695329222Shselasky mthca_alloc_cleanup(&dev->srq_table.alloc); 696329222Shselasky} 697