1[PATCHv7 1/2] libmlx4: Add RoCEE support 2 3Modify libmlx4 to support RoCEE. The change involves retrieving the MAC address 4of a port based on its GID through a new system call, ibv_cmd_get_mac(), and 5embedding the MAC in the address vector representation of mlx4. 6 7Signed-off-by: Eli Cohen <eli@mellanox.co.il> 8--- 9 src/mlx4.h | 3 +++ 10 src/qp.c | 2 ++ 11 src/verbs.c | 29 +++++++++++++++++++++++++++++ 12 src/wqe.h | 3 ++- 13 4 files changed, 36 insertions(+), 1 deletions(-) 14 15Index: libmlx4/src/mlx4.h 16=================================================================== 17--- libmlx4.orig/src/mlx4.h 2010-08-23 08:07:47.599964446 +0300 18+++ libmlx4/src/mlx4.h 2010-08-23 08:08:32.039462057 +0300 19@@ -277,11 +277,15 @@ struct mlx4_av { 20 uint8_t hop_limit; 21 uint32_t sl_tclass_flowlabel; 22 uint8_t dgid[16]; 23+ uint8_t mac[8]; 24 }; 25 26 struct mlx4_ah { 27 struct ibv_ah ibv_ah; 28 struct mlx4_av av; 29+ uint16_t vlan; 30+ uint8_t mac[6]; 31+ uint8_t tagged; 32 }; 33 34 struct mlx4_xrc_domain { 35Index: libmlx4/src/qp.c 36=================================================================== 37--- libmlx4.orig/src/qp.c 2010-08-23 08:07:46.283963844 +0300 38+++ libmlx4/src/qp.c 2010-08-23 08:08:32.039462057 +0300 39@@ -143,6 +143,8 @@ static void set_datagram_seg(struct mlx4 40 memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); 41 dseg->dqpn = htonl(wr->wr.ud.remote_qpn); 42 dseg->qkey = htonl(wr->wr.ud.remote_qkey); 43+ dseg->vlan = htons(to_mah(wr->wr.ud.ah)->vlan); 44+ memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6); 45 } 46 47 static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg) 48@@ -284,6 +286,11 @@ int mlx4_post_send(struct ibv_qp *ibqp, 49 set_datagram_seg(wqe, wr); 50 wqe += sizeof (struct mlx4_wqe_datagram_seg); 51 size += sizeof (struct mlx4_wqe_datagram_seg) / 16; 52+ if (to_mah(wr->wr.ud.ah)->tagged) { 53+ ctrl->ins_vlan = 1 << 6; 54+ ctrl->vlan_tag = htons(to_mah(wr->wr.ud.ah)->vlan); 55+ } 56+ 57 break; 58 59 default: 60@@ -396,7 +403,7 @@ out: 61 62 if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) { 63 ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8); 64- *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn; 65+ *(uint32_t *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; 66 /* 67 * Make sure that descriptor is written to memory 68 * before writing to BlueFlame page. 69Index: libmlx4/src/verbs.c 70=================================================================== 71--- libmlx4.orig/src/verbs.c 2010-08-23 08:07:48.451964305 +0300 72+++ libmlx4/src/verbs.c 2010-08-23 08:08:32.039462057 +0300 73@@ -643,12 +643,14 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp) 74 struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 75 { 76 struct mlx4_ah *ah; 77+ struct ibv_port_attr port_attr; 78+ uint8_t is_mcast; 79 80 ah = malloc(sizeof *ah); 81 if (!ah) 82 return NULL; 83 84- memset(&ah->av, 0, sizeof ah->av); 85+ memset(ah, 0, sizeof *ah); 86 87 ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24)); 88 ah->av.g_slid = attr->src_path_bits; 89@@ -668,7 +670,32 @@ struct ibv_ah *mlx4_create_ah(struct ibv 90 memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); 91 } 92 93+ if (ibv_query_port(pd->context, attr->port_num, &port_attr)) 94+ goto err; 95+ 96+ if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { 97+ if (ibv_resolve_eth_gid(pd, attr->port_num, 98+ (union ibv_gid *)ah->av.dgid, 99+ attr->grh.sgid_index, 100+ ah->mac, &ah->vlan, 101+ &ah->tagged, &is_mcast)) 102+ goto err; 103+ 104+ if (is_mcast) { 105+ ah->av.dlid = htons(0xc000); 106+ ah->av.port_pd |= htonl(1 << 31); 107+ } 108+ if (ah->tagged) { 109+ ah->av.port_pd |= htonl(1 << 29); 110+ ah->vlan |= (attr->sl & 7) << 13; 111+ } 112+ } 113+ 114+ 115 return &ah->ibv_ah; 116+err: 117+ free(ah); 118+ return NULL; 119 } 120 121 int mlx4_destroy_ah(struct ibv_ah *ah) 122Index: libmlx4/src/wqe.h 123=================================================================== 124--- libmlx4.orig/src/wqe.h 2010-08-23 08:07:46.287962570 +0300 125+++ libmlx4/src/wqe.h 2010-08-23 08:07:50.231963413 +0300 126@@ -54,7 +54,8 @@ enum { 127 128 struct mlx4_wqe_ctrl_seg { 129 uint32_t owner_opcode; 130- uint8_t reserved[3]; 131+ uint16_t vlan_tag; 132+ uint8_t ins_vlan; 133 uint8_t fence_size; 134 /* 135 * High 24 bits are SRC remote buffer; low 8 bits are flags: 136@@ -78,7 +79,8 @@ struct mlx4_wqe_datagram_seg { 137 uint32_t av[8]; 138 uint32_t dqpn; 139 uint32_t qkey; 140- uint32_t reserved[2]; 141+ uint16_t vlan; 142+ uint8_t mac[6]; 143 }; 144 145 struct mlx4_wqe_data_seg { 146