1/*
2 * Copyright (c) 2004-2009 Voltaire Inc.  All rights reserved.
3 * Copyright (c) 2009 HNR Consulting.  All rights reserved.
4 * Copyright (c) 2011 Mellanox Technologies LTD.  All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses.  You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 *     Redistribution and use in source and binary forms, with or
13 *     without modification, are permitted provided that the following
14 *     conditions are met:
15 *
16 *      - Redistributions of source code must retain the above
17 *        copyright notice, this list of conditions and the following
18 *        disclaimer.
19 *
20 *      - Redistributions in binary form must reproduce the above
21 *        copyright notice, this list of conditions and the following
22 *        disclaimer in the documentation and/or other materials
23 *        provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36#if HAVE_CONFIG_H
37#  include <config.h>
38#endif				/* HAVE_CONFIG_H */
39
40#include <stdio.h>
41#include <stdlib.h>
42#include <string.h>
43#include <errno.h>
44
45#include <infiniband/umad.h>
46#include <infiniband/mad.h>
47
48#include "mad_internal.h"
49
50int ibdebug;
51
52static struct ibmad_port mad_port;
53struct ibmad_port *ibmp = &mad_port;
54
55static int iberrs;
56
57int madrpc_retries = MAD_DEF_RETRIES;
58int madrpc_timeout = MAD_DEF_TIMEOUT_MS;
59
60static void *save_mad;
61static int save_mad_len = 256;
62
63#undef DEBUG
64#define DEBUG	if (ibdebug)	IBWARN
65#define ERRS(fmt, ...) do {	\
66	if (iberrs || ibdebug)	\
67		IBWARN(fmt, ## __VA_ARGS__); \
68} while (0)
69
70#define MAD_TID(mad)	(*((uint64_t *)((char *)(mad) + 8)))
71
72void madrpc_show_errors(int set)
73{
74	iberrs = set;
75}
76
77void madrpc_save_mad(void *madbuf, int len)
78{
79	save_mad = madbuf;
80	save_mad_len = len;
81}
82
83int madrpc_set_retries(int retries)
84{
85	if (retries > 0)
86		madrpc_retries = retries;
87	return madrpc_retries;
88}
89
90int madrpc_set_timeout(int timeout)
91{
92	madrpc_timeout = timeout;
93	return 0;
94}
95
96void mad_rpc_set_retries(struct ibmad_port *port, int retries)
97{
98	port->retries = retries;
99}
100
101void mad_rpc_set_timeout(struct ibmad_port *port, int timeout)
102{
103	port->timeout = timeout;
104}
105
106int madrpc_def_timeout(void)
107{
108	return madrpc_timeout;
109}
110
111int madrpc_portid(void)
112{
113	return ibmp->port_id;
114}
115
116int mad_rpc_portid(struct ibmad_port *srcport)
117{
118	return srcport->port_id;
119}
120
121int mad_rpc_class_agent(struct ibmad_port *port, int class)
122{
123	if (class < 1 || class >= MAX_CLASS)
124		return -1;
125	return port->class_agents[class];
126}
127
128static int
129_do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
130	   int timeout, int max_retries, int *p_error)
131{
132	uint32_t trid;		/* only low 32 bits - see mad_trid() */
133	int retries;
134	int length, status;
135
136	if (ibdebug > 1) {
137		IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
138		xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
139	}
140
141	if (save_mad) {
142		memcpy(save_mad, umad_get_mad(sndbuf),
143		       save_mad_len < len ? save_mad_len : len);
144		save_mad = 0;
145	}
146
147	if (max_retries <= 0) {
148		errno = EINVAL;
149		*p_error = EINVAL;
150		ERRS("max_retries %d <= 0", max_retries);
151		return -1;
152	}
153
154	trid =
155	    (uint32_t) mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
156
157	for (retries = 0; retries < max_retries; retries++) {
158		if (retries)
159			ERRS("retry %d (timeout %d ms)", retries, timeout);
160
161		length = len;
162		if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
163			IBWARN("send failed; %s", strerror(errno));
164			return -1;
165		}
166
167		/* Use same timeout on receive side just in case */
168		/* send packet is lost somewhere. */
169		do {
170			length = len;
171			if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
172				IBWARN("recv failed: %s", strerror(errno));
173				return -1;
174			}
175
176			if (ibdebug > 2)
177				umad_addr_dump(umad_get_mad_addr(rcvbuf));
178			if (ibdebug > 1) {
179				IBWARN("rcv buf:");
180				xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf),
181				      IB_MAD_SIZE);
182			}
183		} while ((uint32_t)
184			 mad_get_field64(umad_get_mad(rcvbuf), 0,
185					 IB_MAD_TRID_F) != trid);
186
187		status = umad_status(rcvbuf);
188		if (!status)
189			return length;	/* done */
190		if (status == ENOMEM)
191			return length;
192	}
193
194	errno = status;
195	*p_error = ETIMEDOUT;
196	ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
197	return -1;
198}
199
200static int redirect_port(ib_portid_t * port, uint8_t * mad)
201{
202	port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F);
203	if (!port->lid) {
204		IBWARN("GID-based redirection is not supported");
205		return -1;
206	}
207
208	port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F);
209	port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F);
210	port->sl = (uint8_t) mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F);
211
212	/* TODO: Reverse map redirection P_Key to P_Key index */
213
214	if (ibdebug)
215		IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x",
216		       port->lid, port->qp, port->qkey, port->sl);
217
218	return 0;
219}
220
221void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc,
222	      ib_portid_t * dport, void *payload, void *rcvdata)
223{
224	int status, len;
225	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
226	ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
227	int error = 0;
228
229	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
230		rpcv1->error = 0;
231	do {
232		len = 0;
233		memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
234
235		if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
236			return NULL;
237
238		if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
239				      port->class_agents[rpc->mgtclass & 0xff],
240				      len, mad_get_timeout(port, rpc->timeout),
241				      mad_get_retries(port), &error)) < 0) {
242			if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) ==
243			    IB_MAD_RPC_VERSION1)
244				rpcv1->error = error;
245			IBWARN("_do_madrpc failed; dport (%s)",
246			       portid2str(dport));
247			return NULL;
248		}
249
250		mad = umad_get_mad(rcvbuf);
251		status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F);
252
253		/* check for exact match instead of only the redirect bit;
254		 * that way, weird statuses cause an error, too */
255		if (status == IB_MAD_STS_REDIRECT) {
256			/* update dport for next request and retry */
257			/* bail if redirection fails */
258			if (redirect_port(dport, mad))
259				break;
260		} else
261			break;
262	} while (1);
263
264	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
265		rpcv1->error = error;
266	rpc->rstatus = status;
267
268	if (status != 0) {
269		ERRS("MAD completed with error status 0x%x; dport (%s)",
270		     status, portid2str(dport));
271		errno = EIO;
272		return NULL;
273	}
274
275	if (ibdebug) {
276		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
277		xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
278	}
279
280	if (rcvdata)
281		memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
282
283	return rcvdata;
284}
285
286void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc,
287		   ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data)
288{
289	int status, len;
290	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
291	ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
292	int error = 0;
293
294	memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
295
296	DEBUG("rmpp %p data %p", rmpp, data);
297
298	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
299		rpcv1->error = 0;
300	if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
301		return NULL;
302
303	if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
304			      port->class_agents[rpc->mgtclass & 0xff],
305			      len, mad_get_timeout(port, rpc->timeout),
306			      mad_get_retries(port), &error)) < 0) {
307		if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
308			rpcv1->error = error;
309		IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
310		return NULL;
311	}
312
313	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
314		rpcv1->error = error;
315
316	mad = umad_get_mad(rcvbuf);
317
318	if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
319		ERRS("MAD completed with error status 0x%x; dport (%s)",
320		     status, portid2str(dport));
321		errno = EIO;
322		return NULL;
323	}
324
325	if (ibdebug) {
326		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
327		xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
328		      rpc->datasz);
329	}
330
331	if (rmpp) {
332		rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
333		if ((rmpp->flags & 0x3) &&
334		    mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
335			IBWARN("bad rmpp version");
336			return NULL;
337		}
338		rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
339		rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
340		DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
341		rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
342		rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
343	}
344
345	if (data)
346		memcpy(data, mad + rpc->dataoffs, rpc->datasz);
347
348	rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
349
350	return data;
351}
352
353void *madrpc(ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata)
354{
355	return mad_rpc(ibmp, rpc, dport, payload, rcvdata);
356}
357
358void *madrpc_rmpp(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp,
359		  void *data)
360{
361	return mad_rpc_rmpp(ibmp, rpc, dport, rmpp, data);
362}
363
364void
365madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
366{
367	int fd;
368
369	if (umad_init() < 0)
370		IBPANIC("can't init UMAD library");
371
372	if ((fd = umad_open_port(dev_name, dev_port)) < 0)
373		IBPANIC("can't open UMAD port (%s:%d)",
374		dev_name ? dev_name : "(nil)", dev_port);
375
376	if (num_classes >= MAX_CLASS)
377		IBPANIC("too many classes %d requested", num_classes);
378
379	ibmp->port_id = fd;
380	memset(ibmp->class_agents, 0xff, sizeof ibmp->class_agents);
381	while (num_classes--) {
382		uint8_t rmpp_version = 0;
383		int mgmt = *mgmt_classes++;
384
385		if (mgmt == IB_SA_CLASS)
386			rmpp_version = 1;
387		if (mad_register_client_via(mgmt, rmpp_version, ibmp) < 0)
388			IBPANIC("client_register for mgmt class %d failed",
389				mgmt);
390	}
391}
392
393struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port,
394				     int *mgmt_classes, int num_classes)
395{
396	struct ibmad_port *p;
397	int port_id;
398
399	if (num_classes >= MAX_CLASS) {
400		IBWARN("too many classes %d requested", num_classes);
401		errno = EINVAL;
402		return NULL;
403	}
404
405	if (umad_init() < 0) {
406		IBWARN("can't init UMAD library");
407		errno = ENODEV;
408		return NULL;
409	}
410
411	p = malloc(sizeof(*p));
412	if (!p) {
413		errno = ENOMEM;
414		return NULL;
415	}
416	memset(p, 0, sizeof(*p));
417
418	if ((port_id = umad_open_port(dev_name, dev_port)) < 0) {
419		IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
420		if (!errno)
421			errno = EIO;
422		free(p);
423		return NULL;
424	}
425
426	p->port_id = port_id;
427	if (dev_name)
428		strncpy(p->ca_name, dev_name, sizeof p->ca_name - 1);
429	p->portnum = dev_port;
430
431	memset(p->class_agents, 0xff, sizeof p->class_agents);
432	while (num_classes--) {
433		uint8_t rmpp_version = 0;
434		int mgmt = *mgmt_classes++;
435
436		if (mgmt == IB_SA_CLASS)
437			rmpp_version = 1;
438		if (mgmt < 0 || mgmt >= MAX_CLASS ||
439		    mad_register_client_via(mgmt, rmpp_version, p) < 0) {
440			IBWARN("client_register for mgmt %d failed", mgmt);
441			if (!errno)
442				errno = EINVAL;
443			umad_close_port(port_id);
444			free(p);
445			return NULL;
446		}
447	}
448
449	return p;
450}
451
452void mad_rpc_close_port(struct ibmad_port *port)
453{
454	umad_close_port(port->port_id);
455	free(port);
456}
457