1/*
2 * Copyright (c) 2006-2009 Voltaire, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *        copyright notice, this list of conditions and the following
16 *        disclaimer.
17 *
18 *      - Redistributions in binary form must reproduce the above
19 *        copyright notice, this list of conditions and the following
20 *        disclaimer in the documentation and/or other materials
21 *        provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34#if HAVE_CONFIG_H
35#  include <config.h>
36#endif				/* HAVE_CONFIG_H */
37
38#include <stdio.h>
39#include <string.h>
40#include <errno.h>
41#include <inttypes.h>
42
43#include <infiniband/umad.h>
44#include <infiniband/mad.h>
45#include <infiniband/iba/ib_types.h>
46
47#include "ibdiag_common.h"
48
49#define info(fmt, ...) fprintf(stderr, "INFO: " fmt, ## __VA_ARGS__ )
50#define err(fmt, ...) fprintf(stderr, "ERR: " fmt, ## __VA_ARGS__ )
51#ifdef NOISY_DEBUG
52#define dbg(fmt, ...) fprintf(stderr, "DBG: " fmt, ## __VA_ARGS__ )
53#else
54#define dbg(fmt, ...)
55#endif
56
57#define TMO 100
58
59static ibmad_gid_t mgid_ipoib = {
60	0xff, 0x12, 0x40, 0x1b, 0xff, 0xff, 0x00, 0x00,
61	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
62};
63
64struct ibmad_port *srcport;
65
66uint64_t build_mcm_rec(uint8_t * data, ibmad_gid_t mgid, ibmad_gid_t port_gid)
67{
68	memset(data, 0, IB_SA_DATA_SIZE);
69	mad_set_array(data, 0, IB_SA_MCM_MGID_F, mgid);
70	mad_set_array(data, 0, IB_SA_MCM_PORTGID_F, port_gid);
71	mad_set_field(data, 0, IB_SA_MCM_JOIN_STATE_F, 1);
72
73	return IB_MCR_COMPMASK_MGID | IB_MCR_COMPMASK_PORT_GID |
74	    IB_MCR_COMPMASK_JOIN_STATE;
75}
76
77static void build_mcm_rec_umad(void *umad, ib_portid_t * dport, int method,
78			       uint64_t comp_mask, uint8_t * data)
79{
80	ib_rpc_t rpc;
81
82	memset(&rpc, 0, sizeof(rpc));
83	rpc.mgtclass = IB_SA_CLASS;
84	rpc.method = method;
85	rpc.attr.id = IB_SA_ATTR_MCRECORD;
86	rpc.attr.mod = 0;	// ???
87	rpc.mask = comp_mask;
88	rpc.datasz = IB_SA_DATA_SIZE;
89	rpc.dataoffs = IB_SA_DATA_OFFS;
90
91	mad_build_pkt(umad, &rpc, dport, NULL, data);
92}
93
94static int rereg_send(int port, int agent, ib_portid_t * dport,
95		      uint8_t * umad, int len, int method, ibmad_gid_t port_gid)
96{
97	uint8_t data[IB_SA_DATA_SIZE];
98	uint64_t comp_mask;
99
100	comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
101
102	build_mcm_rec_umad(umad, dport, method, comp_mask, data);
103	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
104		err("umad_send %s failed: %s\n",
105		    (method == IB_MAD_METHOD_GET) ? "query" : "non query",
106		    strerror(errno));
107		return -1;
108	}
109	dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method,
110	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
111
112	return 0;
113}
114
115static int rereg_port_gid(int port, int agent, ib_portid_t * dport,
116			  uint8_t * umad, int len, ibmad_gid_t port_gid)
117{
118	uint8_t data[IB_SA_DATA_SIZE];
119	uint64_t comp_mask;
120
121	comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
122
123	build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE, comp_mask, data);
124	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
125		err("umad_send leave failed: %s\n", strerror(errno));
126		return -1;
127	}
128	dbg("umad_send leave: tid = 0x%016" PRIx64 "\n",
129	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
130
131	build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET, comp_mask, data);
132	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
133		err("umad_send join failed: %s\n", strerror(errno));
134		return -1;
135	}
136	dbg("umad_send join: tid = 0x%016" PRIx64 "\n",
137	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
138
139	return 0;
140}
141
142struct guid_trid {
143	ibmad_gid_t gid;
144	uint64_t guid;
145	uint64_t trid;
146};
147
148static int rereg_send_all(int port, int agent, ib_portid_t * dport,
149			  struct guid_trid *list, unsigned cnt)
150{
151	uint8_t *umad;
152	int len = umad_size() + 256;
153	unsigned i;
154	int ret;
155
156	info("rereg_send_all... cnt = %u\n", cnt);
157
158	umad = calloc(1, len);
159	if (!umad) {
160		err("cannot alloc mem for umad: %s\n", strerror(errno));
161		return -1;
162	}
163
164	for (i = 0; i < cnt; i++) {
165		ret =
166		    rereg_port_gid(port, agent, dport, umad, len, list[i].gid);
167		if (ret < 0) {
168			err("rereg_send_all: rereg_port_gid 0x%016" PRIx64
169			    " failed\n", ntohll(list[i].guid));
170			continue;
171		}
172		list[i].trid = mad_get_field64(umad_get_mad(umad), 0,
173					       IB_MAD_TRID_F);
174	}
175
176	info("rereg_send_all: sent %u requests\n", cnt * 2);
177
178	free(umad);
179
180	return 0;
181}
182
183static int rereg_recv(int port, int agent, ib_portid_t * dport,
184		      uint8_t * umad, int length, int tmo)
185{
186	int ret, retry = 0;
187	int len = length;
188
189	while ((ret = umad_recv(port, umad, &len, tmo)) < 0 &&
190	       errno == ETIMEDOUT) {
191		if (retry++ > 3)
192			return 0;
193	}
194	if (ret < 0) {
195		err("umad_recv %d failed: %s\n", ret, strerror(errno));
196		return -1;
197	}
198	dbg("umad_recv (retries %d), tid = 0x%016" PRIx64
199	    ": len = %d, status = %d\n", retry,
200	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), len,
201	    umad_status(umad));
202
203	return 1;
204}
205
206static int rereg_recv_all(int port, int agent, ib_portid_t * dport,
207			  struct guid_trid *list, unsigned cnt)
208{
209	uint8_t *umad, *mad;
210	int len = umad_size() + 256;
211	uint64_t trid;
212	unsigned n, method, status;
213	unsigned i;
214
215	info("rereg_recv_all...\n");
216
217	umad = calloc(1, len);
218	if (!umad) {
219		err("cannot alloc mem for umad: %s\n", strerror(errno));
220		return -1;
221	}
222
223	n = 0;
224	while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) {
225		dbg("rereg_recv_all: done %d\n", n);
226		n++;
227		mad = umad_get_mad(umad);
228
229		method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
230		status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
231
232		if (status)
233			dbg("MAD status %x, method %x\n", status, method);
234
235		if (status &&
236		    (method & 0x7f) == (IB_MAD_METHOD_GET_RESPONSE & 0x7f)) {
237			trid = mad_get_field64(mad, 0, IB_MAD_TRID_F);
238			for (i = 0; i < cnt; i++)
239				if (trid == list[i].trid)
240					break;
241			if (i == cnt) {
242				err("cannot find trid 0x%016" PRIx64 "\n",
243				    trid);
244				continue;
245			}
246			info("guid 0x%016" PRIx64
247			     ": method = %x status = %x. Resending\n",
248			     ntohll(list[i].guid), method, status);
249			rereg_port_gid(port, agent, dport, umad, len,
250				       list[i].gid);
251			list[i].trid =
252			    mad_get_field64(umad_get_mad(umad), 0,
253					    IB_MAD_TRID_F);
254		}
255	}
256
257	info("rereg_recv_all: got %u responses\n", n);
258
259	free(umad);
260	return 0;
261}
262
263static int rereg_query_all(int port, int agent, ib_portid_t * dport,
264			   struct guid_trid *list, unsigned cnt)
265{
266	uint8_t *umad, *mad;
267	int len = umad_size() + 256;
268	unsigned method, status;
269	unsigned i;
270	int ret;
271
272	info("rereg_query_all...\n");
273
274	umad = calloc(1, len);
275	if (!umad) {
276		err("cannot alloc mem for umad: %s\n", strerror(errno));
277		return -1;
278	}
279
280	for (i = 0; i < cnt; i++) {
281		ret = rereg_send(port, agent, dport, umad, len,
282				 IB_MAD_METHOD_GET, list[i].gid);
283		if (ret < 0) {
284			err("query_all: rereg_send failed.\n");
285			continue;
286		}
287
288		ret = rereg_recv(port, agent, dport, umad, len, TMO);
289		if (ret < 0) {
290			err("query_all: rereg_recv failed.\n");
291			continue;
292		}
293
294		mad = umad_get_mad(umad);
295
296		method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
297		status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
298
299		if (status)
300			info("guid 0x%016" PRIx64 ": status %x, method %x\n",
301			     ntohll(list[i].guid), status, method);
302	}
303
304	info("rereg_query_all: %u queried.\n", cnt);
305
306	free(umad);
307	return 0;
308}
309
310#define MAX_CLIENTS 50
311
312static int rereg_and_test_port(char *guid_file, int port, int agent,
313			       ib_portid_t * dport, int timeout)
314{
315	char line[256];
316	FILE *f;
317	ibmad_gid_t port_gid;
318	uint64_t prefix = htonll(0xfe80000000000000ull);
319	uint64_t guid = htonll(0x0002c90200223825ull);
320	struct guid_trid *list;
321	int i = 0;
322
323	list = calloc(MAX_CLIENTS, sizeof(*list));
324	if (!list) {
325		err("cannot alloc mem for guid/trid list: %s\n",
326		    strerror(errno));
327		return -1;
328	}
329
330	f = fopen(guid_file, "r");
331	if (!f) {
332		err("cannot open %s: %s\n", guid_file, strerror(errno));
333		return -1;
334	}
335
336	while (fgets(line, sizeof(line), f)) {
337		guid = strtoull(line, NULL, 0);
338		guid = htonll(guid);
339		memcpy(&port_gid[0], &prefix, 8);
340		memcpy(&port_gid[8], &guid, 8);
341
342		list[i].guid = guid;
343		memcpy(list[i].gid, port_gid, sizeof(list[i].gid));
344		list[i].trid = 0;
345		if (++i >= MAX_CLIENTS)
346			break;
347	}
348	fclose(f);
349
350	rereg_send_all(port, agent, dport, list, i);
351	rereg_recv_all(port, agent, dport, list, i);
352
353	rereg_query_all(port, agent, dport, list, i);
354
355	free(list);
356	return 0;
357}
358
359int main(int argc, char **argv)
360{
361	char *guid_file = "port_guids.list";
362	int mgmt_classes[2] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS };
363	ib_portid_t dport_id;
364	int port, agent;
365	uint8_t *umad, *mad;
366	int len;
367
368	if (argc > 1)
369		guid_file = argv[1];
370
371	srcport = mad_rpc_open_port(NULL, 0, mgmt_classes, 2);
372	if (!srcport)
373		err("Failed to open port");
374
375	resolve_sm_portid(NULL, 0, &dport_id);
376	dport_id.qp = 1;
377	if (!dport_id.qkey)
378		dport_id.qkey = IB_DEFAULT_QP1_QKEY;
379
380	len = umad_size() + 256;
381	umad = calloc(1, len);
382	if (!umad) {
383		err("cannot alloc mem for umad: %s\n", strerror(errno));
384		return -1;
385	}
386	port = mad_rpc_portid(srcport);
387
388	agent = umad_register(port, IB_SA_CLASS, 2, 0, NULL);
389
390	rereg_and_test_port(guid_file, port, agent, &dport_id, TMO);
391	mad = umad_get_mad(umad);
392
393	free(umad);
394	umad_unregister(port, agent);
395	umad_close_port(port);
396	umad_done();
397
398	return 0;
399}
400