xref: /freebsd-10.2-release/contrib/ofed/libmlx4/fixes/fix_inline_size.patch

From 6f9c7b5c83a832884b3e47a6ce52145991073586 Mon Sep 17 00:00:00 2001
From: Eli Cohen <eli@mellanox.co.il>
Date: Sun, 19 Sep 2010 11:28:40 +0200
Subject: [PATCH] libmlx4: fix possible inline size

The current driver checks required inline size by making sure it does not
exceed 1024. This is wrong since the whole WQE is limited to 1008 bytes.
Moreover, a more careful claculation is required to avoid cases where the
application requests inline support in a certain size that when used later
could cause connections to stall due to bad WQEs. This patch takes into account
the size of the WQE, the segements used to create a WQE and the overhead
incured by the inline segments themselves.

Signed-off-by: Eli Cohen <eli@mellanox.co.il>
---
 src/verbs.c |   45 ++++++++++++++++++++++++++++++++++++++++-----
 1 files changed, 40 insertions(+), 5 deletions(-)

Index: libmlx4/src/verbs.c
===================================================================
--- libmlx4.orig/src/verbs.c	2010-09-29 11:10:17.691587848 +0200
+++ libmlx4/src/verbs.c	2010-09-29 11:16:11.031586721 +0200
@@ -402,6 +402,44 @@ int mlx4_destroy_srq(struct ibv_srq *ibs
 	return 0;
 }
 
+static int verify_sizes(struct ibv_qp_init_attr *attr, struct mlx4_context *context)
+{
+	int size;
+	int nsegs;
+
+	if (attr->cap.max_send_wr     > context->max_qp_wr ||
+	    attr->cap.max_recv_wr     > context->max_qp_wr ||
+	    attr->cap.max_send_sge    > context->max_sge   ||
+	    attr->cap.max_recv_sge    > context->max_sge)
+		return -1;
+
+	if (attr->cap.max_inline_data) {
+		nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
+		size = MLX4_MAX_WQE_SIZE - nsegs * sizeof (struct mlx4_wqe_inline_seg);
+		switch (attr->qp_type) {
+		case IBV_QPT_UD:
+			size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
+				 sizeof (struct mlx4_wqe_datagram_seg));
+			break;
+
+		case IBV_QPT_RC:
+		case IBV_QPT_UC:
+		case IBV_QPT_XRC:
+			size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
+				 sizeof (struct mlx4_wqe_raddr_seg));
+			break;
+
+		default:
+			return 0;
+		}
+
+		if (attr->cap.max_inline_data > size)
+			return -1;
+	}
+
+	return 0;
+}
+
 struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
 {
 	struct mlx4_create_qp     cmd;
@@ -412,11 +450,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv
 
 
 	/* Sanity check QP size before proceeding */
-	if (attr->cap.max_send_wr     > context->max_qp_wr ||
-	    attr->cap.max_recv_wr     > context->max_qp_wr ||
-	    attr->cap.max_send_sge    > context->max_sge   ||
-	    attr->cap.max_recv_sge    > context->max_sge   ||
-	    attr->cap.max_inline_data > 1024)
+	if (verify_sizes(attr, context))
 		return NULL;
 
 	qp = malloc(sizeof *qp);
Index: libmlx4/src/mlx4.h
===================================================================
--- libmlx4.orig/src/mlx4.h	2010-09-29 11:10:17.691587848 +0200
+++ libmlx4/src/mlx4.h	2010-09-29 11:11:35.559586971 +0200
@@ -159,6 +159,10 @@ enum {
 	MLX4_CQE_OPCODE_RESIZE		= 0x16,
 };
 
+enum {
+	MLX4_MAX_WQE_SIZE = 1008
+};
+
 struct mlx4_device {
 	struct ibv_device		ibv_dev;
 	int				page_size;
@@ -410,6 +414,7 @@ int mlx4_post_recv(struct ibv_qp *ibqp, 
 			  struct ibv_recv_wr **bad_wr);
 void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
 			   struct mlx4_qp *qp);
+int num_inline_segs(int data, enum ibv_qp_type type);
 int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
 		       enum ibv_qp_type type, struct mlx4_qp *qp);
 void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
Index: libmlx4/src/qp.c
===================================================================
--- libmlx4.orig/src/qp.c	2010-09-29 11:10:17.691587848 +0200
+++ libmlx4/src/qp.c	2010-09-29 11:12:14.931587492 +0200
@@ -505,7 +505,7 @@ out:
 	return ret;
 }
 
-static int num_inline_segs(int data, enum ibv_qp_type type)
+int num_inline_segs(int data, enum ibv_qp_type type)
 {
 	/*
 	 * Inline data segments are not allowed to cross 64 byte
@@ -634,7 +634,8 @@ void mlx4_set_sq_sizes(struct mlx4_qp *q
 	int wqe_size;
 	struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);
 
-	wqe_size = (1 << qp->sq.wqe_shift) - sizeof (struct mlx4_wqe_ctrl_seg);
+	wqe_size = min((1 << qp->sq.wqe_shift), MLX4_MAX_WQE_SIZE) -
+		sizeof (struct mlx4_wqe_ctrl_seg);
 	switch (type) {
 	case IBV_QPT_UD:
 		wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);