1/* 2 * Copyright (c) 2007 Cisco, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#if HAVE_CONFIG_H 34# include <config.h> 35#endif /* HAVE_CONFIG_H */ 36 37#include <stdlib.h> 38#include <netinet/in.h> 39#include <pthread.h> 40#include <string.h> 41 42#include "mlx4.h" 43#include "doorbell.h" 44#include "wqe.h" 45 46static void *get_wqe(struct mlx4_srq *srq, int n) 47{ 48 return srq->buf.buf + (n << srq->wqe_shift); 49} 50 51void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind) 52{ 53 struct mlx4_wqe_srq_next_seg *next; 54 55 pthread_spin_lock(&srq->lock); 56 57 next = get_wqe(srq, srq->tail); 58 next->next_wqe_index = htons(ind); 59 srq->tail = ind; 60 61 pthread_spin_unlock(&srq->lock); 62} 63 64int mlx4_post_srq_recv(struct ibv_srq *ibsrq, 65 struct ibv_recv_wr *wr, 66 struct ibv_recv_wr **bad_wr) 67{ 68 struct mlx4_srq *srq = to_msrq(ibsrq); 69 struct mlx4_wqe_srq_next_seg *next; 70 struct mlx4_wqe_data_seg *scat; 71 int err = 0; 72 int nreq; 73 int i; 74 75 pthread_spin_lock(&srq->lock); 76 77 for (nreq = 0; wr; ++nreq, wr = wr->next) { 78 if (wr->num_sge > srq->max_gs) { 79 err = -1; 80 *bad_wr = wr; 81 break; 82 } 83 84 if (srq->head == srq->tail) { 85 /* SRQ is full*/ 86 err = -1; 87 *bad_wr = wr; 88 break; 89 } 90 91 srq->wrid[srq->head] = wr->wr_id; 92 93 next = get_wqe(srq, srq->head); 94 srq->head = ntohs(next->next_wqe_index); 95 scat = (struct mlx4_wqe_data_seg *) (next + 1); 96 97 for (i = 0; i < wr->num_sge; ++i) { 98 scat[i].byte_count = htonl(wr->sg_list[i].length); 99 scat[i].lkey = htonl(wr->sg_list[i].lkey); 100 scat[i].addr = htonll(wr->sg_list[i].addr); 101 } 102 103 if (i < srq->max_gs) { 104 scat[i].byte_count = 0; 105 scat[i].lkey = htonl(MLX4_INVALID_LKEY); 106 scat[i].addr = 0; 107 } 108 } 109 110 if (nreq) { 111 srq->counter += nreq; 112 113 /* 114 * Make sure that descriptors are written before 115 * we write doorbell record. 116 */ 117 wmb(); 118 119 *srq->db = htonl(srq->counter); 120 } 121 122 pthread_spin_unlock(&srq->lock); 123 124 return err; 125} 126 127int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 128 struct mlx4_srq *srq) 129{ 130 struct mlx4_wqe_srq_next_seg *next; 131 struct mlx4_wqe_data_seg *scatter; 132 int size; 133 int buf_size; 134 int i; 135 136 srq->wrid = malloc(srq->max * sizeof (uint64_t)); 137 if (!srq->wrid) 138 return -1; 139 140 size = sizeof (struct mlx4_wqe_srq_next_seg) + 141 srq->max_gs * sizeof (struct mlx4_wqe_data_seg); 142 143 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 144 ; /* nothing */ 145 146 buf_size = srq->max << srq->wqe_shift; 147 148 if (mlx4_alloc_buf(&srq->buf, buf_size, 149 to_mdev(pd->context->device)->page_size)) { 150 free(srq->wrid); 151 return -1; 152 } 153 154 memset(srq->buf.buf, 0, buf_size); 155 156 /* 157 * Now initialize the SRQ buffer so that all of the WQEs are 158 * linked into the list of free WQEs. 159 */ 160 161 for (i = 0; i < srq->max; ++i) { 162 next = get_wqe(srq, i); 163 next->next_wqe_index = htons((i + 1) & (srq->max - 1)); 164 165 for (scatter = (void *) (next + 1); 166 (void *) scatter < (void *) next + (1 << srq->wqe_shift); 167 ++scatter) 168 scatter->lkey = htonl(MLX4_INVALID_LKEY); 169 } 170 171 srq->head = 0; 172 srq->tail = srq->max - 1; 173 174 return 0; 175} 176 177struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn) 178{ 179 int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 180 181 if (ctx->xrc_srq_table[tind].refcnt) 182 return ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask]; 183 else 184 return NULL; 185} 186 187int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn, 188 struct mlx4_srq *srq) 189{ 190 int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 191 int ret = 0; 192 193 pthread_mutex_lock(&ctx->xrc_srq_table_mutex); 194 195 if (!ctx->xrc_srq_table[tind].refcnt) { 196 ctx->xrc_srq_table[tind].table = calloc(ctx->xrc_srq_table_mask + 1, 197 sizeof(struct mlx4_srq *)); 198 if (!ctx->xrc_srq_table[tind].table) { 199 ret = -1; 200 goto out; 201 } 202 } 203 204 ++ctx->xrc_srq_table[tind].refcnt; 205 ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = srq; 206 207out: 208 pthread_mutex_unlock(&ctx->xrc_srq_table_mutex); 209 return ret; 210} 211 212void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn) 213{ 214 int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 215 216 pthread_mutex_lock(&ctx->xrc_srq_table_mutex); 217 218 if (!--ctx->xrc_srq_table[tind].refcnt) 219 free(ctx->xrc_srq_table[tind].table); 220 else 221 ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = NULL; 222 223 pthread_mutex_unlock(&ctx->xrc_srq_table_mutex); 224} 225 226