1#include <linux/gfp.h> 2 3#include <barrelfish/sys_debug.h> 4 5#include <mlx4ib.h> 6 7#define BUFF_SIZE 100 8#define MAX_POLL 2000 9void test_ib(struct ib_device *device) { 10 struct ib_pd *pd; 11 struct ib_cq *cq; 12 struct ib_mr *mr; 13 struct ib_qp *qp; 14 void *buff_vaddr; 15 u64 buff_paddr = 0; 16 int mr_flags; 17 struct ib_qp_init_attr attr; 18 int ret; 19 int flags; 20 int uninitialized_var( poll_result); 21 struct ib_qp_attr qp_attr; 22 struct ib_send_wr sr; 23 struct ib_sge sge; 24 struct ib_send_wr *bad_wr = NULL; 25 struct ib_wc wc; 26 uint64_t current, ticks_per_msec; 27 int opcode = IB_WR_RDMA_READ; 28 29 pd = ib_alloc_pd(device); 30 if (!pd) 31 printf("failed to alloc PD\n"); 32 33 cq = ib_create_cq(device, NULL, NULL, NULL, 1, 0); 34 if (!cq) 35 printf("failed to create CQ\n"); 36 37 buff_vaddr = dma_alloc(BUFF_SIZE, &buff_paddr); 38 if (!buff_vaddr) 39 printf("failed to alloc buff\n"); 40 41 mr_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ 42 | IB_ACCESS_REMOTE_WRITE; 43 44 mr = ib_get_dma_mr(pd, mr_flags); 45 printf( 46 "MR was registered with addr=0x%lx, lkey=0x%x, rkey=0x%x, flags=0x%x\n", 47 buff_paddr, mr->lkey, mr->rkey, mr_flags); 48 /*mr = mlx4_ib_reg_user_mr(pd, (u64) buff, BUFF_SIZE, (u64) buff, mr_flags, NULL, 49 0);*/ 50 51 memset(&attr, 0, sizeof attr); 52 attr.create_flags = 0; 53 attr.event_handler = NULL; 54 attr.qp_context = NULL; 55 attr.send_cq = cq; 56 attr.recv_cq = cq; 57 attr.srq = NULL; 58 attr.sq_sig_type = IB_SIGNAL_ALL_WR; 59 attr.qp_type = IB_QPT_RC; 60 attr.xrcd = NULL; 61 attr.create_flags = 0; 62 63 attr.cap.max_send_wr = 1; 64 attr.cap.max_recv_wr = 1; 65 attr.cap.max_send_sge = 1; 66 attr.cap.max_recv_sge = 1; 67 attr.cap.max_inline_data = 0; 68 qp = ib_create_qp(pd, &attr); 69 printf("QP was created, QP number=0x%x\n", qp->qp_num); 70 71 memset(&qp_attr, 0, sizeof qp_attr); 72 qp_attr.qp_state = IB_QPS_INIT; 73 qp_attr.port_num = 2; 74 qp_attr.pkey_index = 0; 75 qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ 76 | IB_ACCESS_REMOTE_WRITE; 77 flags = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS; 78 ret = ib_modify_qp(qp, &qp_attr, flags); 79 if (ret) 80 printf("failed to modify QP state to INIT\n"); 81 82 memset(&qp_attr, 0, sizeof(qp_attr)); 83 qp_attr.qp_state = IB_QPS_RTR; 84 qp_attr.path_mtu = IB_MTU_256; 85 qp_attr.dest_qp_num = dest_qp_num; //XXX: need to be HARDWIRED 86 qp_attr.rq_psn = 0; 87 qp_attr.max_dest_rd_atomic = 1; 88 qp_attr.min_rnr_timer = 0x12; 89 qp_attr.ah_attr.ah_flags = 0; 90 qp_attr.ah_attr.dlid = dest_lid; //XXX: need to be HARDWIRED 91 qp_attr.ah_attr.sl = 0; 92 qp_attr.ah_attr.src_path_bits = 0; 93 qp_attr.ah_attr.port_num = 2; 94 flags = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN 95 | IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 96 ret = ib_modify_qp(qp, &qp_attr, flags); 97 if (ret) 98 printf("failed to modify QP state to RTS\n"); 99 100 memset(&qp_attr, 0, sizeof(qp_attr)); 101 qp_attr.qp_state = IB_QPS_RTS; 102 qp_attr.timeout = 0x12; 103 qp_attr.retry_cnt = 6; 104 qp_attr.rnr_retry = 0; 105 qp_attr.sq_psn = 0; 106 qp_attr.max_rd_atomic = 1; 107 flags = IB_QP_STATE | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY 108 | IB_QP_SQ_PSN | IB_QP_MAX_QP_RD_ATOMIC; 109 ret = ib_modify_qp(qp, &qp_attr, flags); 110 if (ret) 111 printf("failed to modify QP state to RTS\n"); 112 113 /* prepare the scatter/gather entry */ 114 memset(&sge, 0, sizeof(sge)); 115 sge.addr = buff_paddr; 116 sge.length = BUFF_SIZE; 117 sge.lkey = mr->lkey; 118 /* prepare the send work request */ 119 memset(&sr, 0, sizeof(sr)); 120 sr.next = NULL; 121 sr.wr_id = 0; 122 sr.sg_list = &sge; 123 sr.num_sge = 1; 124 sr.opcode = opcode; 125 sr.send_flags = IB_SEND_SIGNALED; 126 if (opcode != IB_WR_SEND) { 127 /*XXX: the r_address and r_key has to be hardwired*/ 128 sr.wr.rdma.remote_addr = dest_addr; 129 sr.wr.rdma.rkey = dest_rkey; 130 } 131 /* there is a Receive Request in the responder side, so we won't get any into RNR flow */ 132 ret = ib_post_send(qp, &sr, &bad_wr); 133 if (ret) { 134 printf("failed to post SR\n"); 135 } else { 136 switch (opcode) { 137 case IB_WR_SEND: 138 printf("Send Request was posted\n"); 139 break; 140 case IB_WR_RDMA_READ: 141 printf("RDMA Read Request was posted\n"); 142 break; 143 case IB_WR_RDMA_WRITE: 144 printf("RDMA Write Request was posted\n"); 145 break; 146 default: 147 printf("Unknown Request was posted\n"); 148 break; 149 } 150 } 151 152 current = rdtsc(); 153 sys_debug_get_tsc_per_ms(&ticks_per_msec); 154 while (poll_result == 0 && rdtsc() < current + MAX_POLL * ticks_per_msec) { 155 poll_result = ib_poll_cq(cq, 1, &wc); 156 } 157 if (poll_result < 0) { 158 /* poll CQ failed */ 159 printf("poll CQ failed\n"); 160 } else if (poll_result == 0) { 161 /* the CQ is empty */ 162 printf("completion wasn't found in the CQ after timeout\n"); 163 } else { 164 /* CQE found */ 165 printf("completion was found in CQ with status 0x%x\n", wc.status); 166 /* check the completion status (here we don't care about the completion opcode */ 167 if (wc.status != IB_WC_SUCCESS) { 168 printf( 169 "got bad completion with status: 0x%x, vendor syndrome: 0x%x\n", 170 wc.status, wc.vendor_err); 171 } 172 } 173 174 printf("Contents of client's buffer: %s\n", (char *) buff_vaddr); 175} 176