1/* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#if HAVE_CONFIG_H 34# include <config.h> 35#endif /* HAVE_CONFIG_H */ 36 37#include <stdlib.h> 38#include <netinet/in.h> 39#include <pthread.h> 40#include <string.h> 41 42#include "mthca.h" 43#include "doorbell.h" 44#include "wqe.h" 45 46static void *get_wqe(struct mthca_srq *srq, int n) 47{ 48 return srq->buf.buf + (n << srq->wqe_shift); 49} 50 51/* 52 * Return a pointer to the location within a WQE that we're using as a 53 * link when the WQE is in the free list. We use the imm field at an 54 * offset of 12 bytes because in the Tavor case, posting a WQE may 55 * overwrite the next segment of the previous WQE, but a receive WQE 56 * will never touch the imm field. This avoids corrupting our free 57 * list if the previous WQE has already completed and been put on the 58 * free list when we post the next WQE. 59 */ 60static inline int *wqe_to_link(void *wqe) 61{ 62 return (int *) (wqe + 12); 63} 64 65void mthca_free_srq_wqe(struct mthca_srq *srq, int ind) 66{ 67 struct mthca_next_seg *last_free; 68 69 pthread_spin_lock(&srq->lock); 70 71 last_free = get_wqe(srq, srq->last_free); 72 *wqe_to_link(last_free) = ind; 73 last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); 74 *wqe_to_link(get_wqe(srq, ind)) = -1; 75 srq->last_free = ind; 76 77 pthread_spin_unlock(&srq->lock); 78} 79 80int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, 81 struct ibv_recv_wr *wr, 82 struct ibv_recv_wr **bad_wr) 83{ 84 struct mthca_srq *srq = to_msrq(ibsrq); 85 uint32_t doorbell[2]; 86 int err = 0; 87 int first_ind; 88 int ind; 89 int next_ind; 90 int nreq; 91 int i; 92 void *wqe; 93 void *prev_wqe; 94 95 pthread_spin_lock(&srq->lock); 96 97 first_ind = srq->first_free; 98 99 for (nreq = 0; wr; wr = wr->next) { 100 ind = srq->first_free; 101 wqe = get_wqe(srq, ind); 102 next_ind = *wqe_to_link(wqe); 103 104 if (next_ind < 0) { 105 err = -1; 106 *bad_wr = wr; 107 break; 108 } 109 110 prev_wqe = srq->last; 111 srq->last = wqe; 112 113 ((struct mthca_next_seg *) wqe)->ee_nds = 0; 114 /* flags field will always remain 0 */ 115 116 wqe += sizeof (struct mthca_next_seg); 117 118 if (wr->num_sge > srq->max_gs) { 119 err = -1; 120 *bad_wr = wr; 121 srq->last = prev_wqe; 122 break; 123 } 124 125 for (i = 0; i < wr->num_sge; ++i) { 126 ((struct mthca_data_seg *) wqe)->byte_count = 127 htonl(wr->sg_list[i].length); 128 ((struct mthca_data_seg *) wqe)->lkey = 129 htonl(wr->sg_list[i].lkey); 130 ((struct mthca_data_seg *) wqe)->addr = 131 htonll(wr->sg_list[i].addr); 132 wqe += sizeof (struct mthca_data_seg); 133 } 134 135 if (i < srq->max_gs) { 136 ((struct mthca_data_seg *) wqe)->byte_count = 0; 137 ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); 138 ((struct mthca_data_seg *) wqe)->addr = 0; 139 } 140 141 ((struct mthca_next_seg *) prev_wqe)->ee_nds = 142 htonl(MTHCA_NEXT_DBD); 143 144 srq->wrid[ind] = wr->wr_id; 145 srq->first_free = next_ind; 146 147 if (++nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) { 148 nreq = 0; 149 150 doorbell[0] = htonl(first_ind << srq->wqe_shift); 151 doorbell[1] = htonl(srq->srqn << 8); 152 153 /* 154 * Make sure that descriptors are written 155 * before doorbell is rung. 156 */ 157 wmb(); 158 159 mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); 160 161 first_ind = srq->first_free; 162 } 163 } 164 165 if (nreq) { 166 doorbell[0] = htonl(first_ind << srq->wqe_shift); 167 doorbell[1] = htonl((srq->srqn << 8) | nreq); 168 169 /* 170 * Make sure that descriptors are written before 171 * doorbell is rung. 172 */ 173 wmb(); 174 175 mthca_write64(doorbell, to_mctx(ibsrq->context), MTHCA_RECV_DOORBELL); 176 } 177 178 pthread_spin_unlock(&srq->lock); 179 return err; 180} 181 182int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, 183 struct ibv_recv_wr *wr, 184 struct ibv_recv_wr **bad_wr) 185{ 186 struct mthca_srq *srq = to_msrq(ibsrq); 187 int err = 0; 188 int ind; 189 int next_ind; 190 int nreq; 191 int i; 192 void *wqe; 193 194 pthread_spin_lock(&srq->lock); 195 196 for (nreq = 0; wr; ++nreq, wr = wr->next) { 197 ind = srq->first_free; 198 wqe = get_wqe(srq, ind); 199 next_ind = *wqe_to_link(wqe); 200 201 if (next_ind < 0) { 202 err = -1; 203 *bad_wr = wr; 204 break; 205 } 206 207 ((struct mthca_next_seg *) wqe)->ee_nds = 0; 208 /* flags field will always remain 0 */ 209 210 wqe += sizeof (struct mthca_next_seg); 211 212 if (wr->num_sge > srq->max_gs) { 213 err = -1; 214 *bad_wr = wr; 215 break; 216 } 217 218 for (i = 0; i < wr->num_sge; ++i) { 219 ((struct mthca_data_seg *) wqe)->byte_count = 220 htonl(wr->sg_list[i].length); 221 ((struct mthca_data_seg *) wqe)->lkey = 222 htonl(wr->sg_list[i].lkey); 223 ((struct mthca_data_seg *) wqe)->addr = 224 htonll(wr->sg_list[i].addr); 225 wqe += sizeof (struct mthca_data_seg); 226 } 227 228 if (i < srq->max_gs) { 229 ((struct mthca_data_seg *) wqe)->byte_count = 0; 230 ((struct mthca_data_seg *) wqe)->lkey = htonl(MTHCA_INVAL_LKEY); 231 ((struct mthca_data_seg *) wqe)->addr = 0; 232 } 233 234 srq->wrid[ind] = wr->wr_id; 235 srq->first_free = next_ind; 236 } 237 238 if (nreq) { 239 srq->counter += nreq; 240 241 /* 242 * Make sure that descriptors are written before 243 * we write doorbell record. 244 */ 245 wmb(); 246 *srq->db = htonl(srq->counter); 247 } 248 249 pthread_spin_unlock(&srq->lock); 250 return err; 251} 252 253int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 254 struct mthca_srq *srq) 255{ 256 struct mthca_data_seg *scatter; 257 void *wqe; 258 int size; 259 int i; 260 261 srq->wrid = malloc(srq->max * sizeof (uint64_t)); 262 if (!srq->wrid) 263 return -1; 264 265 size = sizeof (struct mthca_next_seg) + 266 srq->max_gs * sizeof (struct mthca_data_seg); 267 268 for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 269 ; /* nothing */ 270 271 srq->buf_size = srq->max << srq->wqe_shift; 272 273 if (mthca_alloc_buf(&srq->buf, 274 align(srq->buf_size, to_mdev(pd->context->device)->page_size), 275 to_mdev(pd->context->device)->page_size)) { 276 free(srq->wrid); 277 return -1; 278 } 279 280 memset(srq->buf.buf, 0, srq->buf_size); 281 282 /* 283 * Now initialize the SRQ buffer so that all of the WQEs are 284 * linked into the list of free WQEs. In addition, set the 285 * scatter list L_Keys to the sentry value of 0x100. 286 */ 287 288 for (i = 0; i < srq->max; ++i) { 289 struct mthca_next_seg *next; 290 291 next = wqe = get_wqe(srq, i); 292 293 if (i < srq->max - 1) { 294 *wqe_to_link(wqe) = i + 1; 295 next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); 296 } else { 297 *wqe_to_link(wqe) = -1; 298 next->nda_op = 0; 299 } 300 301 for (scatter = wqe + sizeof (struct mthca_next_seg); 302 (void *) scatter < wqe + (1 << srq->wqe_shift); 303 ++scatter) 304 scatter->lkey = htonl(MTHCA_INVAL_LKEY); 305 } 306 307 srq->first_free = 0; 308 srq->last_free = srq->max - 1; 309 srq->last = get_wqe(srq, srq->max - 1); 310 311 return 0; 312} 313