xref: /freebsd-10.0-release/contrib/ofed/libmlx4/fixes/xrc_consolidated_v2.patch

XRC implementation, consolidated (version 2):

xrc ops were moved to their own structure at the end of
struct ibv_context (to preserve binary compatibility).

Check for ibv_context.xrc_ops member via AC_CHECK_MEMBER

XRC QPs have MSB set in qp number, for identification in
completion handling.

Various bug fixes.
(OFED 1.3 commit 39fe7f47e8fc07f356098df048d00740ba585fc5)

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
---
V2:
1. checkpatch.pl cleanup
2. Changed xrc_ops to more ops
3. Check for xrc verbs in ibv_more_ops via AC_CHECK_MEMBER

diff --git a/configure.in b/configure.in
index 25f27f7..46a3a64 100644
--- a/configure.in
+++ b/configure.in
@@ -42,6 +42,12 @@ AC_CHECK_HEADER(valgrind/memcheck.h,
 dnl Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
 AC_CHECK_SIZEOF(long)
+AC_CHECK_MEMBER(struct ibv_context.more_ops,
+    [AC_DEFINE([HAVE_IBV_MORE_OPS], 1, [Define to 1 if more_ops is a member of ibv_context])],,
+    [#include <infiniband/verbs.h>])
+AC_CHECK_MEMBER(struct ibv_more_ops.create_xrc_srq,
+    [AC_DEFINE([HAVE_IBV_XRC_OPS], 1, [Define to 1 if have xrc ops])],,
+    [#include <infiniband/verbs.h>])
 
 dnl Checks for library functions
 AC_CHECK_FUNC(ibv_read_sysfs_file, [],
diff --git a/src/cq.c b/src/cq.c
index 68e16e9..c598b87 100644
--- a/src/cq.c
+++ b/src/cq.c
@@ -194,8 +194,9 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
 {
 	struct mlx4_wq *wq;
 	struct mlx4_cqe *cqe;
-	struct mlx4_srq *srq;
+	struct mlx4_srq *srq = NULL;
 	uint32_t qpn;
+	uint32_t srqn;
 	uint32_t g_mlpath_rqpn;
 	uint16_t wqe_index;
 	int is_error;
@@ -221,20 +223,29 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
 	is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
 		MLX4_CQE_OPCODE_ERROR;
 
-	if (!*cur_qp ||
-	    (ntohl(cqe->my_qpn) & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) {
+	if (qpn & MLX4_XRC_QPN_BIT && !is_send) {
+		srqn = ntohl(cqe->g_mlpath_rqpn) & 0xffffff;
+		/*
+		 * We do not have to take the XRC SRQ table lock here,
+		 * because CQs will be locked while XRC SRQs are removed
+		 * from the table.
+		 */
+		srq = mlx4_find_xrc_srq(to_mctx(cq->ibv_cq.context), srqn);
+		if (!srq)
+			return CQ_POLL_ERR;
+	} else if (!*cur_qp || (qpn & 0xffffff) != (*cur_qp)->ibv_qp.qp_num) {
 		/*
 		 * We do not have to take the QP table lock here,
 		 * because CQs will be locked while QPs are removed
 		 * from the table.
 		 */
 		*cur_qp = mlx4_find_qp(to_mctx(cq->ibv_cq.context),
-				       ntohl(cqe->my_qpn) & 0xffffff);
+				       qpn & 0xffffff);
 		if (!*cur_qp)
 			return CQ_POLL_ERR;
 	}
 
-	wc->qp_num = (*cur_qp)->ibv_qp.qp_num;
+	wc->qp_num = qpn & 0xffffff;
 
 	if (is_send) {
 		wq = &(*cur_qp)->sq;
@@ -242,6 +254,10 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
 		wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);
 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
 		++wq->tail;
+	} else if (srq) {
+		wqe_index = htons(cqe->wqe_index);
+		wc->wr_id = srq->wrid[wqe_index];
+		mlx4_free_srq_wqe(srq, wqe_index);
 	} else if ((*cur_qp)->ibv_qp.srq) {
 		srq = to_msrq((*cur_qp)->ibv_qp.srq);
 		wqe_index = htons(cqe->wqe_index);
@@ -387,6 +403,10 @@ void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
 	uint32_t prod_index;
 	uint8_t owner_bit;
 	int nfreed = 0;
+	int is_xrc_srq = 0;
+
+	if (srq && srq->ibv_srq.xrc_cq)
+		is_xrc_srq = 1;
 
 	/*
 	 * First we need to find the current producer index, so we
@@ -405,7 +425,12 @@ void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
 	 */
 	while ((int) --prod_index - (int) cq->cons_index >= 0) {
 		cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
-		if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
+		if (is_xrc_srq &&
+		    (ntohl(cqe->g_mlpath_rqpn & 0xffffff) == srq->srqn) &&
+		    !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
+			mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
+			++nfreed;
+		} else if ((ntohl(cqe->my_qpn) & 0xffffff) == qpn) {
 			if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
 				mlx4_free_srq_wqe(srq, ntohs(cqe->wqe_index));
 			++nfreed;
diff --git a/src/mlx4-abi.h b/src/mlx4-abi.h
index 20a40c9..1b1253c 100644
--- a/src/mlx4-abi.h
+++ b/src/mlx4-abi.h
@@ -68,6 +68,14 @@ struct mlx4_resize_cq {
 	__u64				buf_addr;
 };
 
+#ifdef HAVE_IBV_XRC_OPS
+struct mlx4_create_xrc_srq {
+	struct ibv_create_xrc_srq	ibv_cmd;
+	__u64				buf_addr;
+	__u64				db_addr;
+};
+#endif
+
 struct mlx4_create_srq {
 	struct ibv_create_srq		ibv_cmd;
 	__u64				buf_addr;
@@ -90,4 +98,12 @@ struct mlx4_create_qp {
 	__u8				reserved[5];
 };
 
+#ifdef HAVE_IBV_XRC_OPS
+struct mlx4_open_xrc_domain_resp {
+	struct ibv_open_xrc_domain_resp	ibv_resp;
+	__u32				xrcdn;
+	__u32				reserved;
+};
+#endif
+
 #endif /* MLX4_ABI_H */
diff --git a/src/mlx4.c b/src/mlx4.c
index 671e849..27ca75d 100644
--- a/src/mlx4.c
+++ b/src/mlx4.c
@@ -68,6 +68,16 @@ struct {
 	HCA(MELLANOX, 0x673c),	/* MT25408 "Hermon" QDR PCIe gen2 */
 };
 
+#ifdef HAVE_IBV_MORE_OPS
+static struct ibv_more_ops mlx4_more_ops = {
+#ifdef HAVE_IBV_XRC_OPS
+	.create_xrc_srq   = mlx4_create_xrc_srq,
+	.open_xrc_domain  = mlx4_open_xrc_domain,
+	.close_xrc_domain = mlx4_close_xrc_domain,
+#endif
+};
+#endif
+
 static struct ibv_context_ops mlx4_ctx_ops = {
 	.query_device  = mlx4_query_device,
 	.query_port    = mlx4_query_port,
@@ -124,6 +134,15 @@ static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_
 	for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i)
 		context->qp_table[i].refcnt = 0;
 
+	context->num_xrc_srqs = resp.qp_tab_size;
+	context->xrc_srq_table_shift = ffs(context->num_xrc_srqs) - 1
+				       - MLX4_XRC_SRQ_TABLE_BITS;
+	context->xrc_srq_table_mask = (1 << context->xrc_srq_table_shift) - 1;
+
+	pthread_mutex_init(&context->xrc_srq_table_mutex, NULL);
+	for (i = 0; i < MLX4_XRC_SRQ_TABLE_SIZE; ++i)
+		context->xrc_srq_table[i].refcnt = 0;
+
 	for (i = 0; i < MLX4_NUM_DB_TYPE; ++i)
 		context->db_list[i] = NULL;
 
@@ -156,6 +175,9 @@ static struct ibv_context *mlx4_alloc_context(struct ibv_device *ibdev, int cmd_
 	pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE);
 
 	context->ibv_ctx.ops = mlx4_ctx_ops;
+#ifdef HAVE_IBV_XRC_OPS
+	context->ibv_ctx.more_ops = &mlx4_more_ops;
+#endif
 
 	if (mlx4_query_device(&context->ibv_ctx, &dev_attrs))
 		goto query_free;
diff --git a/src/mlx4.h b/src/mlx4.h
index 8643d8f..3eadb98 100644
--- a/src/mlx4.h
+++ b/src/mlx4.h
@@ -79,6 +79,11 @@
 
 #endif
 
+#ifndef HAVE_IBV_MORE_OPS
+#undef HAVE_IBV_XRC_OPS
+#undef HAVE_IBV_CREATE_QP_EXP
+#endif
+
 #define HIDDEN		__attribute__((visibility ("hidden")))
 
 #define PFX		"mlx4: "
@@ -111,6 +116,16 @@ enum {
 	MLX4_QP_TABLE_MASK		= MLX4_QP_TABLE_SIZE - 1
 };
 
+enum {
+	MLX4_XRC_SRQ_TABLE_BITS		= 8,
+	MLX4_XRC_SRQ_TABLE_SIZE		= 1 << MLX4_XRC_SRQ_TABLE_BITS,
+	MLX4_XRC_SRQ_TABLE_MASK		= MLX4_XRC_SRQ_TABLE_SIZE - 1
+};
+
+enum {
+	MLX4_XRC_QPN_BIT		= (1 << 23)
+};
+
 enum mlx4_db_type {
 	MLX4_DB_TYPE_CQ,
 	MLX4_DB_TYPE_RQ,
@@ -174,6 +189,15 @@ struct mlx4_context {
 	int				max_sge;
 	int				max_cqe;
 
+	struct {
+		struct mlx4_srq       **table;
+		int			refcnt;
+	}				xrc_srq_table[MLX4_XRC_SRQ_TABLE_SIZE];
+	pthread_mutex_t			xrc_srq_table_mutex;
+	int				num_xrc_srqs;
+	int				xrc_srq_table_shift;
+	int				xrc_srq_table_mask;
+
 	struct mlx4_db_page	       *db_list[MLX4_NUM_DB_TYPE];
 	pthread_mutex_t			db_list_mutex;
 };
@@ -260,6 +284,11 @@ struct mlx4_ah {
 	struct mlx4_av			av;
 };
 
+struct mlx4_xrc_domain {
+	struct ibv_xrc_domain		ibv_xrcd;
+	uint32_t			xrcdn;
+};
+
 static inline unsigned long align(unsigned long val, unsigned long align)
 {
 	return (val + align - 1) & ~(align - 1);
@@ -304,6 +333,13 @@ static inline struct mlx4_ah *to_mah(struct ibv_ah *ibah)
 	return to_mxxx(ah, ah);
 }
 
+#ifdef HAVE_IBV_XRC_OPS
+static inline struct mlx4_xrc_domain *to_mxrcd(struct ibv_xrc_domain *ibxrcd)
+{
+	return to_mxxx(xrcd, xrc_domain);
+}
+#endif
+
 int mlx4_alloc_buf(struct mlx4_buf *buf, size_t size, int page_size);
 void mlx4_free_buf(struct mlx4_buf *buf);
 
@@ -350,6 +386,10 @@ void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind);
 int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
 		       struct ibv_recv_wr *wr,
 		       struct ibv_recv_wr **bad_wr);
+struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
+int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
+		       struct mlx4_srq *srq);
+void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn);
 
 struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr);
 int mlx4_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
@@ -380,5 +420,16 @@ int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr,
 int mlx4_alloc_av(struct mlx4_pd *pd, struct ibv_ah_attr *attr,
 		   struct mlx4_ah *ah);
 void mlx4_free_av(struct mlx4_ah *ah);
+#ifdef HAVE_IBV_XRC_OPS
+struct ibv_srq *mlx4_create_xrc_srq(struct ibv_pd *pd,
+				    struct ibv_xrc_domain *xrc_domain,
+				    struct ibv_cq *xrc_cq,
+				    struct ibv_srq_init_attr *attr);
+struct ibv_xrc_domain *mlx4_open_xrc_domain(struct ibv_context *context,
+					    int fd, int oflag);
+
+int mlx4_close_xrc_domain(struct ibv_xrc_domain *d);
+#endif
+
 
 #endif /* MLX4_H */
diff --git a/src/qp.c b/src/qp.c
index 01e8580..2f02430 100644
--- a/src/qp.c
+++ b/src/qp.c
@@ -226,7 +226,7 @@ int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
 		ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 		qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
 
-		ctrl->srcrb_flags =
+		ctrl->xrcrb_flags =
 			(wr->send_flags & IBV_SEND_SIGNALED ?
 			 htonl(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
 			(wr->send_flags & IBV_SEND_SOLICITED ?
@@ -243,6 +243,9 @@ int mlx4_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
 		size = sizeof *ctrl / 16;
 
 		switch (ibqp->qp_type) {
+		case IBV_QPT_XRC:
+			ctrl->xrcrb_flags |= htonl(wr->xrc_remote_srq_num << 8);
+			/* fall thru */
 		case IBV_QPT_RC:
 		case IBV_QPT_UC:
 			switch (wr->opcode) {
@@ -543,6 +546,7 @@ void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
 		size += sizeof (struct mlx4_wqe_raddr_seg);
 		break;
 
+	case IBV_QPT_XRC:
 	case IBV_QPT_RC:
 		size += sizeof (struct mlx4_wqe_raddr_seg);
 		/*
@@ -631,6 +635,7 @@ void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
 
 	case IBV_QPT_UC:
 	case IBV_QPT_RC:
+	case IBV_QPT_XRC:
 		wqe_size -= sizeof (struct mlx4_wqe_raddr_seg);
 		break;
 
diff --git a/src/srq.c b/src/srq.c
index ba2ceb9..1350792 100644
--- a/src/srq.c
+++ b/src/srq.c
@@ -167,3 +167,53 @@ int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
 
 	return 0;
 }
+
+struct mlx4_srq *mlx4_find_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)
+{
+	int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
+
+	if (ctx->xrc_srq_table[tind].refcnt)
+		return ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask];
+	else
+		return NULL;
+}
+
+int mlx4_store_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn,
+		       struct mlx4_srq *srq)
+{
+	int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
+	int ret = 0;
+
+	pthread_mutex_lock(&ctx->xrc_srq_table_mutex);
+
+	if (!ctx->xrc_srq_table[tind].refcnt) {
+		ctx->xrc_srq_table[tind].table = calloc(ctx->xrc_srq_table_mask + 1,
+							sizeof(struct mlx4_srq *));
+		if (!ctx->xrc_srq_table[tind].table) {
+			ret = -1;
+			goto out;
+		}
+	}
+
+	++ctx->xrc_srq_table[tind].refcnt;
+	ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = srq;
+
+out:
+	pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);
+	return ret;
+}
+
+void mlx4_clear_xrc_srq(struct mlx4_context *ctx, uint32_t xrc_srqn)
+{
+	int tind = (xrc_srqn & (ctx->num_xrc_srqs - 1)) >> ctx->xrc_srq_table_shift;
+
+	pthread_mutex_lock(&ctx->xrc_srq_table_mutex);
+
+	if (!--ctx->xrc_srq_table[tind].refcnt)
+		free(ctx->xrc_srq_table[tind].table);
+	else
+		ctx->xrc_srq_table[tind].table[xrc_srqn & ctx->xrc_srq_table_mask] = NULL;
+
+	pthread_mutex_unlock(&ctx->xrc_srq_table_mutex);
+}
+
diff --git a/src/verbs.c b/src/verbs.c
index 400050c..b7c9c8e 100644
--- a/src/verbs.c
+++ b/src/verbs.c
@@ -368,18 +368,36 @@ int mlx4_query_srq(struct ibv_srq *srq,
 	return ibv_cmd_query_srq(srq, attr, &cmd, sizeof cmd);
 }
 
-int mlx4_destroy_srq(struct ibv_srq *srq)
+int mlx4_destroy_srq(struct ibv_srq *ibsrq)
 {
+	struct mlx4_srq *srq = to_msrq(ibsrq);
+	struct mlx4_cq *mcq = NULL;
 	int ret;
 
-	ret = ibv_cmd_destroy_srq(srq);
-	if (ret)
+	if (ibsrq->xrc_cq) {
+		/* is an xrc_srq */
+		mcq = to_mcq(ibsrq->xrc_cq);
+		mlx4_cq_clean(mcq, 0, srq);
+		pthread_spin_lock(&mcq->lock);
+		mlx4_clear_xrc_srq(to_mctx(ibsrq->context), srq->srqn);
+		pthread_spin_unlock(&mcq->lock);
+	}
+
+	ret = ibv_cmd_destroy_srq(ibsrq);
+	if (ret) {
+		if (ibsrq->xrc_cq) {
+			pthread_spin_lock(&mcq->lock);
+			mlx4_store_xrc_srq(to_mctx(ibsrq->context),
+					   srq->srqn, srq);
+			pthread_spin_unlock(&mcq->lock);
+		}
 		return ret;
+	}
 
-	mlx4_free_db(to_mctx(srq->context), MLX4_DB_TYPE_RQ, to_msrq(srq)->db);
-	mlx4_free_buf(&to_msrq(srq)->buf);
-	free(to_msrq(srq)->wrid);
-	free(to_msrq(srq));
+	mlx4_free_db(to_mctx(ibsrq->context), MLX4_DB_TYPE_RQ, srq->db);
+	mlx4_free_buf(&srq->buf);
+	free(srq->wrid);
+	free(srq);
 
 	return 0;
 }
@@ -415,7 +433,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 	qp->sq.wqe_cnt = align_queue_size(attr->cap.max_send_wr + qp->sq_spare_wqes);
 	qp->rq.wqe_cnt = align_queue_size(attr->cap.max_recv_wr);
 
-	if (attr->srq)
+	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
 		attr->cap.max_recv_wr = qp->rq.wqe_cnt = 0;
 	else {
 		if (attr->cap.max_recv_sge < 1)
@@ -433,7 +451,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 	    pthread_spin_init(&qp->rq.lock, PTHREAD_PROCESS_PRIVATE))
 		goto err_free;
 
-	if (!attr->srq) {
+	if (!attr->srq && attr->qp_type != IBV_QPT_XRC) {
 		qp->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ);
 		if (!qp->db)
 			goto err_free;
@@ -442,7 +460,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 	}
 
 	cmd.buf_addr	    = (uintptr_t) qp->buf.buf;
-	if (attr->srq)
+	if (attr->srq || attr->qp_type == IBV_QPT_XRC)
 		cmd.db_addr = 0;
 	else
 		cmd.db_addr = (uintptr_t) qp->db;
@@ -485,7 +503,7 @@ err_destroy:
 
 err_rq_db:
 	pthread_mutex_unlock(&to_mctx(pd->context)->qp_table_mutex);
-	if (!attr->srq)
+	if (!attr->srq && attr->qp_type != IBV_QPT_XRC)
 		mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, qp->db);
 
 err_free:
@@ -544,7 +562,7 @@ int mlx4_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 			mlx4_cq_clean(to_mcq(qp->send_cq), qp->qp_num, NULL);
 
 		mlx4_init_qp_indices(to_mqp(qp));
-		if (!qp->srq)
+		if (!qp->srq && qp->qp_type != IBV_QPT_XRC)
 			*to_mqp(qp)->db = 0;
 	}
 
@@ -603,7 +621,7 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp)
 	mlx4_unlock_cqs(ibqp);
 	pthread_mutex_unlock(&to_mctx(ibqp->context)->qp_table_mutex);
 
-	if (!ibqp->srq)
+	if (!ibqp->srq && ibqp->qp_type != IBV_QPT_XRC)
 		mlx4_free_db(to_mctx(ibqp->context), MLX4_DB_TYPE_RQ, qp->db);
 	free(qp->sq.wrid);
 	if (qp->rq.wqe_cnt)
@@ -661,3 +679,103 @@ int mlx4_detach_mcast(struct ibv_qp *qp, union ibv_gid *gid, uint16_t lid)
 
 	return 0;
 }
+
+#ifdef HAVE_IBV_XRC_OPS
+struct ibv_srq *mlx4_create_xrc_srq(struct ibv_pd *pd,
+				    struct ibv_xrc_domain *xrc_domain,
+				    struct ibv_cq *xrc_cq,
+				    struct ibv_srq_init_attr *attr)
+{
+	struct mlx4_create_xrc_srq  cmd;
+	struct mlx4_create_srq_resp resp;
+	struct mlx4_srq		   *srq;
+	int			    ret;
+
+	/* Sanity check SRQ size before proceeding */
+	if (attr->attr.max_wr > 1 << 16 || attr->attr.max_sge > 64)
+		return NULL;
+
+	srq = malloc(sizeof *srq);
+	if (!srq)
+		return NULL;
+
+	if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
+		goto err;
+
+	srq->max     = align_queue_size(attr->attr.max_wr + 1);
+	srq->max_gs  = attr->attr.max_sge;
+	srq->counter = 0;
+
+	if (mlx4_alloc_srq_buf(pd, &attr->attr, srq))
+		goto err;
+
+	srq->db = mlx4_alloc_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ);
+	if (!srq->db)
+		goto err_free;
+
+	*srq->db = 0;
+
+	cmd.buf_addr = (uintptr_t) srq->buf.buf;
+	cmd.db_addr  = (uintptr_t) srq->db;
+
+	ret = ibv_cmd_create_xrc_srq(pd, &srq->ibv_srq, attr,
+				     xrc_domain->handle,
+				     xrc_cq->handle,
+				     &cmd.ibv_cmd, sizeof cmd,
+				     &resp.ibv_resp, sizeof resp);
+	if (ret)
+		goto err_db;
+
+	srq->ibv_srq.xrc_srq_num = srq->srqn = resp.srqn;
+
+	ret = mlx4_store_xrc_srq(to_mctx(pd->context), srq->ibv_srq.xrc_srq_num, srq);
+	if (ret)
+		goto err_destroy;
+
+	return &srq->ibv_srq;
+
+err_destroy:
+	ibv_cmd_destroy_srq(&srq->ibv_srq);
+
+err_db:
+	mlx4_free_db(to_mctx(pd->context), MLX4_DB_TYPE_RQ, srq->db);
+
+err_free:
+	free(srq->wrid);
+	mlx4_free_buf(&srq->buf);
+
+err:
+	free(srq);
+
+	return NULL;
+}
+
+struct ibv_xrc_domain *mlx4_open_xrc_domain(struct ibv_context *context,
+					    int fd, int oflag)
+{
+	int ret;
+	struct mlx4_open_xrc_domain_resp resp;
+	struct mlx4_xrc_domain *xrcd;
+
+	xrcd = malloc(sizeof *xrcd);
+	if (!xrcd)
+		return NULL;
+
+	ret = ibv_cmd_open_xrc_domain(context, fd, oflag, &xrcd->ibv_xrcd,
+				      &resp.ibv_resp, sizeof resp);
+	if (ret) {
+		free(xrcd);
+		return NULL;
+	}
+
+	xrcd->xrcdn = resp.xrcdn;
+	return &xrcd->ibv_xrcd;
+}
+
+int mlx4_close_xrc_domain(struct ibv_xrc_domain *d)
+{
+	ibv_cmd_close_xrc_domain(d);
+	free(d);
+	return 0;
+}
+#endif
diff --git a/src/wqe.h b/src/wqe.h
index 6f7f309..fa2f8ac 100644
--- a/src/wqe.h
+++ b/src/wqe.h
@@ -65,7 +65,7 @@ struct mlx4_wqe_ctrl_seg {
 	 * [1]   SE (solicited event)
 	 * [0]   FL (force loopback)
 	 */
-	uint32_t		srcrb_flags;
+	uint32_t		xrcrb_flags;
 	/*
 	 * imm is immediate data for send/RDMA write w/ immediate;
 	 * also invalidation key for send with invalidate; input