1219820Sjeff[PATCHv7 1/2] libmlx4: Add RoCEE support 2219820Sjeff 3219820SjeffModify libmlx4 to support RoCEE. The change involves retrieving the MAC address 4219820Sjeffof a port based on its GID through a new system call, ibv_cmd_get_mac(), and 5219820Sjeffembedding the MAC in the address vector representation of mlx4. 6219820Sjeff 7219820SjeffSigned-off-by: Eli Cohen <eli@mellanox.co.il> 8219820Sjeff--- 9219820Sjeff src/mlx4.h | 3 +++ 10219820Sjeff src/qp.c | 2 ++ 11219820Sjeff src/verbs.c | 29 +++++++++++++++++++++++++++++ 12219820Sjeff src/wqe.h | 3 ++- 13219820Sjeff 4 files changed, 36 insertions(+), 1 deletions(-) 14219820Sjeff 15219820SjeffIndex: libmlx4/src/mlx4.h 16219820Sjeff=================================================================== 17219820Sjeff--- libmlx4.orig/src/mlx4.h 2010-08-23 08:07:47.599964446 +0300 18219820Sjeff+++ libmlx4/src/mlx4.h 2010-08-23 08:08:32.039462057 +0300 19219820Sjeff@@ -277,11 +277,15 @@ struct mlx4_av { 20219820Sjeff uint8_t hop_limit; 21219820Sjeff uint32_t sl_tclass_flowlabel; 22219820Sjeff uint8_t dgid[16]; 23219820Sjeff+ uint8_t mac[8]; 24219820Sjeff }; 25219820Sjeff 26219820Sjeff struct mlx4_ah { 27219820Sjeff struct ibv_ah ibv_ah; 28219820Sjeff struct mlx4_av av; 29219820Sjeff+ uint16_t vlan; 30219820Sjeff+ uint8_t mac[6]; 31219820Sjeff+ uint8_t tagged; 32219820Sjeff }; 33219820Sjeff 34219820Sjeff struct mlx4_xrc_domain { 35219820SjeffIndex: libmlx4/src/qp.c 36219820Sjeff=================================================================== 37219820Sjeff--- libmlx4.orig/src/qp.c 2010-08-23 08:07:46.283963844 +0300 38219820Sjeff+++ libmlx4/src/qp.c 2010-08-23 08:08:32.039462057 +0300 39219820Sjeff@@ -143,6 +143,8 @@ static void set_datagram_seg(struct mlx4 40219820Sjeff memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av)); 41219820Sjeff dseg->dqpn = htonl(wr->wr.ud.remote_qpn); 42219820Sjeff dseg->qkey = htonl(wr->wr.ud.remote_qkey); 43219820Sjeff+ dseg->vlan = htons(to_mah(wr->wr.ud.ah)->vlan); 44219820Sjeff+ memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->mac, 6); 45219820Sjeff } 46219820Sjeff 47219820Sjeff static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg) 48219820Sjeff@@ -284,6 +286,11 @@ int mlx4_post_send(struct ibv_qp *ibqp, 49219820Sjeff set_datagram_seg(wqe, wr); 50219820Sjeff wqe += sizeof (struct mlx4_wqe_datagram_seg); 51219820Sjeff size += sizeof (struct mlx4_wqe_datagram_seg) / 16; 52219820Sjeff+ if (to_mah(wr->wr.ud.ah)->tagged) { 53219820Sjeff+ ctrl->ins_vlan = 1 << 6; 54219820Sjeff+ ctrl->vlan_tag = htons(to_mah(wr->wr.ud.ah)->vlan); 55219820Sjeff+ } 56219820Sjeff+ 57219820Sjeff break; 58219820Sjeff 59219820Sjeff default: 60219820Sjeff@@ -396,7 +403,7 @@ out: 61219820Sjeff 62219820Sjeff if (nreq == 1 && inl && size > 1 && size < ctx->bf_buf_size / 16) { 63219820Sjeff ctrl->owner_opcode |= htonl((qp->sq.head & 0xffff) << 8); 64219820Sjeff- *(uint32_t *) ctrl->reserved |= qp->doorbell_qpn; 65219820Sjeff+ *(uint32_t *) (&ctrl->vlan_tag) |= qp->doorbell_qpn; 66219820Sjeff /* 67219820Sjeff * Make sure that descriptor is written to memory 68219820Sjeff * before writing to BlueFlame page. 69219820SjeffIndex: libmlx4/src/verbs.c 70219820Sjeff=================================================================== 71219820Sjeff--- libmlx4.orig/src/verbs.c 2010-08-23 08:07:48.451964305 +0300 72219820Sjeff+++ libmlx4/src/verbs.c 2010-08-23 08:08:32.039462057 +0300 73219820Sjeff@@ -643,12 +643,14 @@ int mlx4_destroy_qp(struct ibv_qp *ibqp) 74219820Sjeff struct ibv_ah *mlx4_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) 75219820Sjeff { 76219820Sjeff struct mlx4_ah *ah; 77219820Sjeff+ struct ibv_port_attr port_attr; 78219820Sjeff+ uint8_t is_mcast; 79219820Sjeff 80219820Sjeff ah = malloc(sizeof *ah); 81219820Sjeff if (!ah) 82219820Sjeff return NULL; 83219820Sjeff 84219820Sjeff- memset(&ah->av, 0, sizeof ah->av); 85219820Sjeff+ memset(ah, 0, sizeof *ah); 86219820Sjeff 87219820Sjeff ah->av.port_pd = htonl(to_mpd(pd)->pdn | (attr->port_num << 24)); 88219820Sjeff ah->av.g_slid = attr->src_path_bits; 89219820Sjeff@@ -668,7 +670,32 @@ struct ibv_ah *mlx4_create_ah(struct ibv 90219820Sjeff memcpy(ah->av.dgid, attr->grh.dgid.raw, 16); 91219820Sjeff } 92219820Sjeff 93219820Sjeff+ if (ibv_query_port(pd->context, attr->port_num, &port_attr)) 94219820Sjeff+ goto err; 95219820Sjeff+ 96219820Sjeff+ if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { 97219820Sjeff+ if (ibv_resolve_eth_gid(pd, attr->port_num, 98219820Sjeff+ (union ibv_gid *)ah->av.dgid, 99219820Sjeff+ attr->grh.sgid_index, 100219820Sjeff+ ah->mac, &ah->vlan, 101219820Sjeff+ &ah->tagged, &is_mcast)) 102219820Sjeff+ goto err; 103219820Sjeff+ 104219820Sjeff+ if (is_mcast) { 105219820Sjeff+ ah->av.dlid = htons(0xc000); 106219820Sjeff+ ah->av.port_pd |= htonl(1 << 31); 107219820Sjeff+ } 108219820Sjeff+ if (ah->tagged) { 109219820Sjeff+ ah->av.port_pd |= htonl(1 << 29); 110219820Sjeff+ ah->vlan |= (attr->sl & 7) << 13; 111219820Sjeff+ } 112219820Sjeff+ } 113219820Sjeff+ 114219820Sjeff+ 115219820Sjeff return &ah->ibv_ah; 116219820Sjeff+err: 117219820Sjeff+ free(ah); 118219820Sjeff+ return NULL; 119219820Sjeff } 120219820Sjeff 121219820Sjeff int mlx4_destroy_ah(struct ibv_ah *ah) 122219820SjeffIndex: libmlx4/src/wqe.h 123219820Sjeff=================================================================== 124219820Sjeff--- libmlx4.orig/src/wqe.h 2010-08-23 08:07:46.287962570 +0300 125219820Sjeff+++ libmlx4/src/wqe.h 2010-08-23 08:07:50.231963413 +0300 126219820Sjeff@@ -54,7 +54,8 @@ enum { 127219820Sjeff 128219820Sjeff struct mlx4_wqe_ctrl_seg { 129219820Sjeff uint32_t owner_opcode; 130219820Sjeff- uint8_t reserved[3]; 131219820Sjeff+ uint16_t vlan_tag; 132219820Sjeff+ uint8_t ins_vlan; 133219820Sjeff uint8_t fence_size; 134219820Sjeff /* 135219820Sjeff * High 24 bits are SRC remote buffer; low 8 bits are flags: 136219820Sjeff@@ -78,7 +79,8 @@ struct mlx4_wqe_datagram_seg { 137219820Sjeff uint32_t av[8]; 138219820Sjeff uint32_t dqpn; 139219820Sjeff uint32_t qkey; 140219820Sjeff- uint32_t reserved[2]; 141219820Sjeff+ uint16_t vlan; 142219820Sjeff+ uint8_t mac[6]; 143219820Sjeff }; 144219820Sjeff 145219820Sjeff struct mlx4_wqe_data_seg { 146