1219820Sjeff/* 2219820Sjeff * Copyright (c) 2007 Cisco, Inc. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff */ 32219820Sjeff 33219820Sjeff#if HAVE_CONFIG_H 34219820Sjeff# include <config.h> 35219820Sjeff#endif /* HAVE_CONFIG_H */ 36219820Sjeff 37219820Sjeff#include <stdlib.h> 38219820Sjeff#include <netinet/in.h> 39219820Sjeff#include <pthread.h> 40219820Sjeff#include <string.h> 41219820Sjeff 42219820Sjeff#include "mlx4.h" 43219820Sjeff#include "doorbell.h" 44219820Sjeff#include "wqe.h" 45219820Sjeff 46219820Sjeffstatic void *get_wqe(struct mlx4_srq *srq, int n) 47219820Sjeff{ 48219820Sjeff return srq->buf.buf + (n << srq->wqe_shift); 49219820Sjeff} 50219820Sjeff 51219820Sjeffvoid mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind) 52219820Sjeff{ 53219820Sjeff struct mlx4_wqe_srq_next_seg *next; 54219820Sjeff 55219820Sjeff pthread_spin_lock(&srq->lock); 56219820Sjeff 57219820Sjeff next = get_wqe(srq, srq->tail); 58219820Sjeff next->next_wqe_index = htons(ind); 59219820Sjeff srq->tail = ind; 60219820Sjeff 61219820Sjeff pthread_spin_unlock(&srq->lock); 62219820Sjeff} 63219820Sjeff 64219820Sjeffint mlx4_post_srq_recv(struct ibv_srq *ibsrq, 65219820Sjeff struct ibv_recv_wr *wr, 66219820Sjeff struct ibv_recv_wr **bad_wr) 67219820Sjeff{ 68219820Sjeff struct mlx4_srq *srq = to_msrq(ibsrq); 69219820Sjeff struct mlx4_wqe_srq_next_seg *next; 70219820Sjeff struct mlx4_wqe_data_seg *scat; 71219820Sjeff int err = 0; 72219820Sjeff int nreq; 73219820Sjeff int i; 74219820Sjeff 75219820Sjeff pthread_spin_lock(&srq->lock); 76219820Sjeff 77219820Sjeff for (nreq = 0; wr; ++nreq, wr = wr->next) { 78219820Sjeff if (wr->num_sge > srq->max_gs) { 79219820Sjeff err = -1; 80219820Sjeff *bad_wr = wr; 81219820Sjeff break; 82219820Sjeff } 83219820Sjeff 84219820Sjeff if (srq->head == srq->tail) { 85219820Sjeff /* SRQ is full*/ 86219820Sjeff err = -1; 87219820Sjeff *bad_wr = wr; 88219820Sjeff break; 89219820Sjeff } 90219820Sjeff 91219820Sjeff srq->wrid[srq->head] = wr->wr_id; 92219820Sjeff 93219820Sjeff next = get_wqe(srq, srq->head); 94219820Sjeff srq->head = ntohs(next->next_wqe_index); 95219820Sjeff scat = (struct mlx4_wqe_data_seg *) (next + 1); 96219820Sjeff 97219820Sjeff for (i = 0; i < wr->num_sge; ++i) { 98219820Sjeff scat[i].byte_count = htonl(wr->sg_list[i].length); 99219820Sjeff scat[i].lkey = htonl(wr->sg_list[i].lkey); 100219820Sjeff scat[i].addr = htonll(wr->sg_list[i].addr); 101219820Sjeff } 102219820Sjeff 103219820Sjeff if (i < srq->max_gs) { 104219820Sjeff scat[i].byte_count = 0; 105219820Sjeff scat[i].lkey = htonl(MLX4_INVALID_LKEY); 106219820Sjeff scat[i].addr = 0; 107219820Sjeff } 108219820Sjeff } 109219820Sjeff 110219820Sjeff if (nreq) { 111219820Sjeff srq->counter += nreq; 112219820Sjeff 113219820Sjeff /* 114219820Sjeff * Make sure that descriptors are written before 115219820Sjeff * we write doorbell record. 116219820Sjeff */ 117219820Sjeff wmb(); 118219820Sjeff 119219820Sjeff *srq->db = htonl(srq->counter); 120219820Sjeff } 121219820Sjeff 122219820Sjeff pthread_spin_unlock(&srq->lock); 123219820Sjeff 124219820Sjeff return err; 125219820Sjeff} 126219820Sjeff 127219820Sjeffint mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 128219820Sjeff struct mlx4_srq *srq) 129219820Sjeff{ 130219820Sjeff struct mlx4_wqe_srq_next_seg *next; 131219820Sjeff struct mlx4_wqe_data_seg *scatter; 132219820Sjeff int size; 133219820Sjeff int buf_size; 134219820Sjeff int i; 135219820Sjeff 136219820Sjeff srq->wrid = malloc(srq->max * sizeof (uint64_t)); 137219820Sjeff if (!srq->wrid) 138219820Sjeff return -1; 139219820Sjeff 140219820Sjeff size = sizeof (struct mlx4_wqe_srq_next_seg) + 141219820Sjeff srq->max_gs * sizeof (struct mlx4_wqe_data_seg); 142219820Sjeff 143219820Sjeff for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 144219820Sjeff ; /* nothing */ 145219820Sjeff 146219820Sjeff buf_size = srq->max << srq->wqe_shift; 147219820Sjeff 148219820Sjeff if (mlx4_alloc_buf(&srq->buf, buf_size, 149219820Sjeff to_mdev(pd->context->device)->page_size)) { 150219820Sjeff free(srq->wrid); 151219820Sjeff return -1; 152219820Sjeff } 153219820Sjeff 154219820Sjeff memset(srq->buf.buf, 0, buf_size); 155219820Sjeff 156219820Sjeff /* 157219820Sjeff * Now initialize the SRQ buffer so that all of the WQEs are 158219820Sjeff * linked into the list of free WQEs. 159219820Sjeff */ 160219820Sjeff 161219820Sjeff for (i = 0; i < srq->max; ++i) { 162219820Sjeff next = get_wqe(srq, i); 163219820Sjeff next->next_wqe_index = htons((i + 1) & (srq->max - 1)); 164219820Sjeff 165219820Sjeff for (scatter = (void *) (next + 1); 166219820Sjeff (void *) scatter < (void *) next + (1 << srq->wqe_shift); 167219820Sjeff ++scatter) 168219820Sjeff scatter->lkey = htonl(MLX4_INVALID_LKEY); 169219820Sjeff } 170219820Sjeff 171219820Sjeff srq->head = 0; 172219820Sjeff srq->tail = srq->max - 1; 173219820Sjeff 174219820Sjeff return 0; 175219820Sjeff} 176219820Sjeff 177219820Sjeffstruct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn) 178219820Sjeff{ 179219820Sjeff int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 180219820Sjeff 181219820Sjeff if (ctx->xrc_srq_table[tind].refcnt) 182219820Sjeff return ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask]; 183219820Sjeff else 184219820Sjeff return NULL; 185219820Sjeff} 186219820Sjeff 187219820Sjeffint mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn, 188219820Sjeff struct mlx4_srq *srq) 189219820Sjeff{ 190219820Sjeff int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 191219820Sjeff int ret = 0; 192219820Sjeff 193219820Sjeff pthread_mutex_lock(&ctx->xrc_srq_table_mutex); 194219820Sjeff 195219820Sjeff if (!ctx->xrc_srq_table[tind].refcnt) { 196219820Sjeff ctx->xrc_srq_table[tind].table = calloc(ctx->xrc_srq_table_mask + 1, 197219820Sjeff sizeof(struct mlx4_srq *)); 198219820Sjeff if (!ctx->xrc_srq_table[tind].table) { 199219820Sjeff ret = -1; 200219820Sjeff goto out; 201219820Sjeff } 202219820Sjeff } 203219820Sjeff 204219820Sjeff ++ctx->xrc_srq_table[tind].refcnt; 205219820Sjeff ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = srq; 206219820Sjeff 207219820Sjeffout: 208219820Sjeff pthread_mutex_unlock(&ctx->xrc_srq_table_mutex); 209219820Sjeff return ret; 210219820Sjeff} 211219820Sjeff 212219820Sjeffvoid mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn) 213219820Sjeff{ 214219820Sjeff int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift; 215219820Sjeff 216219820Sjeff pthread_mutex_lock(&ctx->xrc_srq_table_mutex); 217219820Sjeff 218219820Sjeff if (!--ctx->xrc_srq_table[tind].refcnt) 219219820Sjeff free(ctx->xrc_srq_table[tind].table); 220219820Sjeff else 221219820Sjeff ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = NULL; 222219820Sjeff 223219820Sjeff pthread_mutex_unlock(&ctx->xrc_srq_table_mutex); 224219820Sjeff} 225219820Sjeff 226