1219820Sjeff/* 2219820Sjeff * Copyright (c) 2005 Cisco Systems. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#if HAVE_CONFIG_H 34219820Sjeff# include <config.h> 35219820Sjeff#endif /* HAVE_CONFIG_H */ 36219820Sjeff 37219820Sjeff#include <stdlib.h> 38219820Sjeff#include <netinet/in.h> 39219820Sjeff#include <pthread.h> 40219820Sjeff#include <string.h> 41219820Sjeff 42219820Sjeff#include "mthca.h" 43219820Sjeff#include "doorbell.h" 44219820Sjeff#include "wqe.h" 45219820Sjeff 46219820Sjeffstatic void *get_wqe(struct mthca_srq *srq, int n) 47219820Sjeff{ 48219820Sjeff return srq->buf.buf + (n << srq->wqe_shift); 49219820Sjeff} 50219820Sjeff 51219820Sjeff/* 52219820Sjeff * Return a pointer to the location within a WQE that we're using as a 53219820Sjeff * link when the WQE is in the free list. We use the imm field at an 54219820Sjeff * offset of 12 bytes because in the Tavor case, posting a WQE may 55219820Sjeff * overwrite the next segment of the previous WQE, but a receive WQE 56219820Sjeff * will never touch the imm field. This avoids corrupting our free 57219820Sjeff * list if the previous WQE has already completed and been put on the 58219820Sjeff * free list when we post the next WQE. 59219820Sjeff */ 60219820Sjeffstatic inline int *wqe_to_link(void *wqe) 61219820Sjeff{ 62219820Sjeff return (int *) (wqe + 12); 63219820Sjeff} 64219820Sjeff 65219820Sjeffvoid mthca_free_srq_wqe(struct mthca_srq *srq, int ind) 66219820Sjeff{ 67219820Sjeff struct mthca_next_seg *last_free; 68219820Sjeff 69219820Sjeff pthread_spin_lock(&srq->lock); 70219820Sjeff 71219820Sjeff last_free = get_wqe(srq, srq->last_free); 72219820Sjeff *wqe_to_link(last_free) = ind; 73219820Sjeff last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); 74219820Sjeff *wqe_to_link(get_wqe(srq, ind)) = -1; 75219820Sjeff srq->last_free = ind; 76219820Sjeff 77219820Sjeff pthread_spin_unlock(&srq->lock); 78219820Sjeff} 79219820Sjeff 80219820Sjeffint mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, 81219820Sjeff struct ibv_recv_wr *wr, 82219820Sjeff struct ibv_recv_wr **bad_wr) 83219820Sjeff{ 84219820Sjeff struct mthca_srq *srq = to_msrq(ibsrq); 85219820Sjeff uint32_t doorbell[2]; 86219820Sjeff int err = 0; 87219820Sjeff int first_ind; 88219820Sjeff int ind; 89219820Sjeff int next_ind; 90219820Sjeff int nreq; 91219820Sjeff int i; 92219820Sjeff void *wqe; 93219820Sjeff void *prev_wqe; 94219820Sjeff 95219820Sjeff pthread_spin_lock(&srq->lock); 96219820Sjeff 97219820Sjeff first_ind = srq->first_free; 98219820Sjeff 99219820Sjeff for (nreq = 0; wr; wr = wr->next) { 100219820Sjeff ind = srq->first_free; 101219820Sjeff wqe = get_wqe(srq, ind); 102219820Sjeff next_ind = *wqe_to_link(wqe); 103219820Sjeff 104219820Sjeff if (next_ind < 0) { 105219820Sjeff err = -1; 106219820Sjeff *bad_wr = wr; 107219820Sjeff break; 108219820Sjeff } 109219820Sjeff 110219820Sjeff prev_wqe = srq->last; 111219820Sjeff srq->last = wqe; 112219820Sjeff 113219820Sjeff ((struct mthca_next_seg *) wqe)->ee_nds = 0; 114219820Sjeff /* flags field will always remain 0 */ 115219820Sjeff 116219820Sjeff wqe += sizeof (struct mthca_next_seg); 117219820Sjeff 118219820Sjeff if (wr->num_sge > srq->max_gs) { 119219820Sjeff err = -1; 120219820Sjeff *bad_wr = wr; 121219820Sjeff srq->last = prev_wqe; 122219820Sjeff break; 123219820Sjeff } 124219820Sjeff 125219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 126219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 127219820Sjeff htonl(wr->sg_list[i].length); 128219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = 129219820Sjeff htonl(wr->sg_list[i].lkey); 130219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 131219820Sjeff htonll(wr->sg_list[i].addr); 132219820Sjeff wqe += sizeof (struct mthca_data_seg); 133219820Sjeff } 134219820Sjeff 135219820Sjeff if (i < srq->max_gs) { 136219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 0; 137219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); 138219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 0; 139219820Sjeff } 140219820Sjeff 141219820Sjeff ((struct mthca_next_seg *) prev_wqe)->ee_nds = 142219820Sjeff htonl(MTHCA_NEXT_DBD); 143219820Sjeff 144219820Sjeff srq->wrid[ind] = wr->wr_id; 145219820Sjeff srq->first_free = next_ind; 146219820Sjeff 147219820Sjeff if (++nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) { 148219820Sjeff nreq = 0; 149219820Sjeff 150219820Sjeff doorbell[0] = htonl(first_ind << srq->wqe_shift); 151219820Sjeff doorbell[1] = htonl(srq->srqn << 8); 152219820Sjeff 153219820Sjeff /* 154219820Sjeff * Make sure that descriptors are written 155219820Sjeff * before doorbell is rung. 156219820Sjeff */ 157219820Sjeff wmb(); 158219820Sjeff 159219820Sjeff mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); 160219820Sjeff 161219820Sjeff first_ind = srq->first_free; 162219820Sjeff } 163219820Sjeff } 164219820Sjeff 165219820Sjeff if (nreq) { 166219820Sjeff doorbell[0] = htonl(first_ind << srq->wqe_shift); 167219820Sjeff doorbell[1] = htonl((srq->srqn << 8) | nreq); 168219820Sjeff 169219820Sjeff /* 170219820Sjeff * Make sure that descriptors are written before 171219820Sjeff * doorbell is rung. 172219820Sjeff */ 173219820Sjeff wmb(); 174219820Sjeff 175219820Sjeff mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); 176219820Sjeff } 177219820Sjeff 178219820Sjeff pthread_spin_unlock(&srq->lock); 179219820Sjeff return err; 180219820Sjeff} 181219820Sjeff 182219820Sjeffint mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, 183219820Sjeff struct ibv_recv_wr *wr, 184219820Sjeff struct ibv_recv_wr **bad_wr) 185219820Sjeff{ 186219820Sjeff struct mthca_srq *srq = to_msrq(ibsrq); 187219820Sjeff int err = 0; 188219820Sjeff int ind; 189219820Sjeff int next_ind; 190219820Sjeff int nreq; 191219820Sjeff int i; 192219820Sjeff void *wqe; 193219820Sjeff 194219820Sjeff pthread_spin_lock(&srq->lock); 195219820Sjeff 196219820Sjeff for (nreq = 0; wr; ++nreq, wr = wr->next) { 197219820Sjeff ind = srq->first_free; 198219820Sjeff wqe = get_wqe(srq, ind); 199219820Sjeff next_ind = *wqe_to_link(wqe); 200219820Sjeff 201219820Sjeff if (next_ind < 0) { 202219820Sjeff err = -1; 203219820Sjeff *bad_wr = wr; 204219820Sjeff break; 205219820Sjeff } 206219820Sjeff 207219820Sjeff ((struct mthca_next_seg *) wqe)->ee_nds = 0; 208219820Sjeff /* flags field will always remain 0 */ 209219820Sjeff 210219820Sjeff wqe += sizeof (struct mthca_next_seg); 211219820Sjeff 212219820Sjeff if (wr->num_sge > srq->max_gs) { 213219820Sjeff err = -1; 214219820Sjeff *bad_wr = wr; 215219820Sjeff break; 216219820Sjeff } 217219820Sjeff 218219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 219219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 220219820Sjeff htonl(wr->sg_list[i].length); 221219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = 222219820Sjeff htonl(wr->sg_list[i].lkey); 223219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 224219820Sjeff htonll(wr->sg_list[i].addr); 225219820Sjeff wqe += sizeof (struct mthca_data_seg); 226219820Sjeff } 227219820Sjeff 228219820Sjeff if (i < srq->max_gs) { 229219820Sjeff ((struct mthca_data_seg *) wqe)->byte_count = 0; 230219820Sjeff ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); 231219820Sjeff ((struct mthca_data_seg *) wqe)->addr = 0; 232219820Sjeff } 233219820Sjeff 234219820Sjeff srq->wrid[ind] = wr->wr_id; 235219820Sjeff srq->first_free = next_ind; 236219820Sjeff } 237219820Sjeff 238219820Sjeff if (nreq) { 239219820Sjeff srq->counter += nreq; 240219820Sjeff 241219820Sjeff /* 242219820Sjeff * Make sure that descriptors are written before 243219820Sjeff * we write doorbell record. 244219820Sjeff */ 245219820Sjeff wmb(); 246219820Sjeff *srq->db = htonl(srq->counter); 247219820Sjeff } 248219820Sjeff 249219820Sjeff pthread_spin_unlock(&srq->lock); 250219820Sjeff return err; 251219820Sjeff} 252219820Sjeff 253219820Sjeffint mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 254219820Sjeff struct mthca_srq *srq) 255219820Sjeff{ 256219820Sjeff struct mthca_data_seg *scatter; 257219820Sjeff void *wqe; 258219820Sjeff int size; 259219820Sjeff int i; 260219820Sjeff 261219820Sjeff srq->wrid = malloc(srq->max * sizeof (uint64_t)); 262219820Sjeff if (!srq->wrid) 263219820Sjeff return -1; 264219820Sjeff 265219820Sjeff size = sizeof (struct mthca_next_seg) + 266219820Sjeff srq->max_gs * sizeof (struct mthca_data_seg); 267219820Sjeff 268219820Sjeff for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 269219820Sjeff ; /* nothing */ 270219820Sjeff 271219820Sjeff srq->buf_size = srq->max << srq->wqe_shift; 272219820Sjeff 273219820Sjeff if (mthca_alloc_buf(&srq->buf, 274219820Sjeff align(srq->buf_size, to_mdev(pd->context->device)->page_size), 275219820Sjeff to_mdev(pd->context->device)->page_size)) { 276219820Sjeff free(srq->wrid); 277219820Sjeff return -1; 278219820Sjeff } 279219820Sjeff 280219820Sjeff memset(srq->buf.buf, 0, srq->buf_size); 281219820Sjeff 282219820Sjeff /* 283219820Sjeff * Now initialize the SRQ buffer so that all of the WQEs are 284219820Sjeff * linked into the list of free WQEs. In addition, set the 285219820Sjeff * scatter list L_Keys to the sentry value of 0x100. 286219820Sjeff */ 287219820Sjeff 288219820Sjeff for (i = 0; i < srq->max; ++i) { 289219820Sjeff struct mthca_next_seg *next; 290219820Sjeff 291219820Sjeff next = wqe = get_wqe(srq, i); 292219820Sjeff 293219820Sjeff if (i < srq->max - 1) { 294219820Sjeff *wqe_to_link(wqe) = i + 1; 295219820Sjeff next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); 296219820Sjeff } else { 297219820Sjeff *wqe_to_link(wqe) = -1; 298219820Sjeff next->nda_op = 0; 299219820Sjeff } 300219820Sjeff 301219820Sjeff for (scatter = wqe + sizeof (struct mthca_next_seg); 302219820Sjeff (void *) scatter < wqe + (1 << srq->wqe_shift); 303219820Sjeff ++scatter) 304219820Sjeff scatter->lkey = htonl(MTHCA_INVAL_LKEY); 305219820Sjeff } 306219820Sjeff 307219820Sjeff srq->first_free = 0; 308219820Sjeff srq->last_free = srq->max - 1; 309219820Sjeff srq->last = get_wqe(srq, srq->max - 1); 310219820Sjeff 311219820Sjeff return 0; 312219820Sjeff} 313