1#include <linux/gfp.h>
2
3#include <barrelfish/sys_debug.h>
4
5#include <mlx4ib.h>
6
7#define BUFF_SIZE 100
8#define MAX_POLL 2000
9void test_ib(struct ib_device *device) {
10	struct ib_pd *pd;
11	struct ib_cq *cq;
12	struct ib_mr *mr;
13	struct ib_qp *qp;
14	void *buff_vaddr;
15	u64 buff_paddr = 0;
16	int mr_flags;
17	struct ib_qp_init_attr attr;
18	int ret;
19	int flags;
20	int uninitialized_var( poll_result);
21	struct ib_qp_attr qp_attr;
22	struct ib_send_wr sr;
23	struct ib_sge sge;
24	struct ib_send_wr *bad_wr = NULL;
25	struct ib_wc wc;
26	uint64_t current, ticks_per_msec;
27	int opcode = IB_WR_RDMA_READ;
28
29	pd = ib_alloc_pd(device);
30	if (!pd)
31		printf("failed to alloc PD\n");
32
33	cq = ib_create_cq(device, NULL, NULL, NULL, 1, 0);
34	if (!cq)
35		printf("failed to create CQ\n");
36
37	buff_vaddr = dma_alloc(BUFF_SIZE, &buff_paddr);
38	if (!buff_vaddr)
39		printf("failed to alloc buff\n");
40
41	mr_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ
42			| IB_ACCESS_REMOTE_WRITE;
43
44	mr = ib_get_dma_mr(pd, mr_flags);
45	printf(
46			"MR was registered with addr=0x%lx, lkey=0x%x, rkey=0x%x, flags=0x%x\n",
47			buff_paddr, mr->lkey, mr->rkey, mr_flags);
48	/*mr = mlx4_ib_reg_user_mr(pd, (u64) buff, BUFF_SIZE, (u64) buff, mr_flags, NULL,
49	 0);*/
50
51	memset(&attr, 0, sizeof attr);
52	attr.create_flags = 0;
53	attr.event_handler = NULL;
54	attr.qp_context = NULL;
55	attr.send_cq = cq;
56	attr.recv_cq = cq;
57	attr.srq = NULL;
58	attr.sq_sig_type = IB_SIGNAL_ALL_WR;
59	attr.qp_type = IB_QPT_RC;
60	attr.xrcd = NULL;
61	attr.create_flags = 0;
62
63	attr.cap.max_send_wr = 1;
64	attr.cap.max_recv_wr = 1;
65	attr.cap.max_send_sge = 1;
66	attr.cap.max_recv_sge = 1;
67	attr.cap.max_inline_data = 0;
68	qp = ib_create_qp(pd, &attr);
69	printf("QP was created, QP number=0x%x\n", qp->qp_num);
70
71	memset(&qp_attr, 0, sizeof qp_attr);
72	qp_attr.qp_state = IB_QPS_INIT;
73	qp_attr.port_num = 2;
74	qp_attr.pkey_index = 0;
75	qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ
76			| IB_ACCESS_REMOTE_WRITE;
77	flags = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;
78	ret = ib_modify_qp(qp, &qp_attr, flags);
79	if (ret)
80		printf("failed to modify QP state to INIT\n");
81
82	memset(&qp_attr, 0, sizeof(qp_attr));
83	qp_attr.qp_state = IB_QPS_RTR;
84	qp_attr.path_mtu = IB_MTU_256;
85	qp_attr.dest_qp_num = dest_qp_num; //XXX: need to be HARDWIRED
86	qp_attr.rq_psn = 0;
87	qp_attr.max_dest_rd_atomic = 1;
88	qp_attr.min_rnr_timer = 0x12;
89	qp_attr.ah_attr.ah_flags = 0;
90	qp_attr.ah_attr.dlid = dest_lid; //XXX: need to be HARDWIRED
91	qp_attr.ah_attr.sl = 0;
92	qp_attr.ah_attr.src_path_bits = 0;
93	qp_attr.ah_attr.port_num = 2;
94	flags = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN
95			| IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;
96	ret = ib_modify_qp(qp, &qp_attr, flags);
97	if (ret)
98		printf("failed to modify QP state to RTS\n");
99
100	memset(&qp_attr, 0, sizeof(qp_attr));
101	qp_attr.qp_state = IB_QPS_RTS;
102	qp_attr.timeout = 0x12;
103	qp_attr.retry_cnt = 6;
104	qp_attr.rnr_retry = 0;
105	qp_attr.sq_psn = 0;
106	qp_attr.max_rd_atomic = 1;
107	flags = IB_QP_STATE | IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY
108			| IB_QP_SQ_PSN | IB_QP_MAX_QP_RD_ATOMIC;
109	ret = ib_modify_qp(qp, &qp_attr, flags);
110	if (ret)
111		printf("failed to modify QP state to RTS\n");
112
113	/* prepare the scatter/gather entry */
114	memset(&sge, 0, sizeof(sge));
115	sge.addr = buff_paddr;
116	sge.length = BUFF_SIZE;
117	sge.lkey = mr->lkey;
118	/* prepare the send work request */
119	memset(&sr, 0, sizeof(sr));
120	sr.next = NULL;
121	sr.wr_id = 0;
122	sr.sg_list = &sge;
123	sr.num_sge = 1;
124	sr.opcode = opcode;
125	sr.send_flags = IB_SEND_SIGNALED;
126	if (opcode != IB_WR_SEND) {
127		/*XXX: the r_address and r_key has to be hardwired*/
128		sr.wr.rdma.remote_addr = dest_addr;
129		sr.wr.rdma.rkey = dest_rkey;
130	}
131	/* there is a Receive Request in the responder side, so we won't get any into RNR flow */
132	ret = ib_post_send(qp, &sr, &bad_wr);
133	if (ret) {
134		printf("failed to post SR\n");
135	} else {
136		switch (opcode) {
137		case IB_WR_SEND:
138			printf("Send Request was posted\n");
139			break;
140		case IB_WR_RDMA_READ:
141			printf("RDMA Read Request was posted\n");
142			break;
143		case IB_WR_RDMA_WRITE:
144			printf("RDMA Write Request was posted\n");
145			break;
146		default:
147			printf("Unknown Request was posted\n");
148			break;
149		}
150	}
151
152	current = rdtsc();
153	sys_debug_get_tsc_per_ms(&ticks_per_msec);
154	while (poll_result == 0 && rdtsc() < current + MAX_POLL * ticks_per_msec) {
155		poll_result = ib_poll_cq(cq, 1, &wc);
156	}
157	if (poll_result < 0) {
158		/* poll CQ failed */
159		printf("poll CQ failed\n");
160	} else if (poll_result == 0) {
161		/* the CQ is empty */
162		printf("completion wasn't found in the CQ after timeout\n");
163	} else {
164		/* CQE found */
165		printf("completion was found in CQ with status 0x%x\n", wc.status);
166		/* check the completion status (here we don't care about the completion opcode */
167		if (wc.status != IB_WC_SUCCESS) {
168			printf(
169					"got bad completion with status: 0x%x, vendor syndrome: 0x%x\n",
170					wc.status, wc.vendor_err);
171		}
172	}
173
174	printf("Contents of client's buffer: %s\n", (char *) buff_vaddr);
175}
176