1/*
2 * Copyright (c) 2004-2006 Voltaire Inc.  All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *        copyright notice, this list of conditions and the following
16 *        disclaimer.
17 *
18 *      - Redistributions in binary form must reproduce the above
19 *        copyright notice, this list of conditions and the following
20 *        disclaimer in the documentation and/or other materials
21 *        provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34#if HAVE_CONFIG_H
35#  include <config.h>
36#endif /* HAVE_CONFIG_H */
37
38#include <stdio.h>
39#include <stdlib.h>
40#include <unistd.h>
41#include <pthread.h>
42#include <sys/time.h>
43#include <string.h>
44#include <errno.h>
45
46#include <infiniband/umad.h>
47#include "mad.h"
48
49#define MAX_CLASS 256
50
51struct ibmad_port {
52	int port_id;  /* file descriptor returned by umad_open() */
53	int class_agents[MAX_CLASS]; /* class2agent mapper */
54};
55
56int ibdebug;
57
58static int mad_portid = -1;
59static int iberrs;
60
61static int madrpc_retries = MAD_DEF_RETRIES;
62static int def_madrpc_timeout = MAD_DEF_TIMEOUT_MS;
63static void *save_mad;
64static int save_mad_len = 256;
65
66#undef DEBUG
67#define DEBUG	if (ibdebug)	IBWARN
68#define ERRS	if (iberrs || ibdebug)	IBWARN
69
70#define MAD_TID(mad)	(*((uint64_t *)((char *)(mad) + 8)))
71
72void
73madrpc_show_errors(int set)
74{
75	iberrs = set;
76}
77
78void
79madrpc_save_mad(void *madbuf, int len)
80{
81	save_mad = madbuf;
82	save_mad_len = len;
83}
84
85int
86madrpc_set_retries(int retries)
87{
88	if (retries > 0)
89		madrpc_retries = retries;
90	return madrpc_retries;
91}
92
93int
94madrpc_set_timeout(int timeout)
95{
96	def_madrpc_timeout = timeout;
97	return 0;
98}
99
100int
101madrpc_def_timeout(void)
102{
103	return def_madrpc_timeout;
104}
105
106int
107madrpc_portid(void)
108{
109	return mad_portid;
110}
111
112static int
113_do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
114	   int timeout)
115{
116	uint32_t trid; /* only low 32 bits */
117	int retries;
118	int length, status;
119
120	if (!timeout)
121		timeout = def_madrpc_timeout;
122
123	if (ibdebug > 1) {
124		IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
125		xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
126	}
127
128	if (save_mad) {
129		memcpy(save_mad, umad_get_mad(sndbuf),
130		       save_mad_len < len ? save_mad_len : len);
131		save_mad = 0;
132	}
133
134	trid = mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
135
136	for (retries = 0; retries < madrpc_retries; retries++) {
137		if (retries) {
138			ERRS("retry %d (timeout %d ms)", retries, timeout);
139		}
140
141		length = len;
142		if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
143			IBWARN("send failed; %s", strerror(errno));
144			return -1;
145		}
146
147		/* Use same timeout on receive side just in case */
148		/* send packet is lost somewhere. */
149		do {
150			if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
151				IBWARN("recv failed: %s", strerror(errno));
152				return -1;
153			}
154
155			if (ibdebug > 1) {
156				IBWARN("rcv buf:");
157				xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE);
158			}
159		} while ((uint32_t)mad_get_field64(umad_get_mad(rcvbuf), 0, IB_MAD_TRID_F) != trid);
160
161		status = umad_status(rcvbuf);
162		if (!status)
163			return length;		/* done */
164		if (status == ENOMEM)
165			return length;
166	}
167
168	ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
169	return -1;
170}
171
172void *
173mad_rpc(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport, void *payload,
174	void *rcvdata)
175{
176	const struct ibmad_port *p = port_id;
177	int status, len;
178	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
179
180	len = 0;
181	memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
182
183	if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
184		return 0;
185
186	if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
187			      p->class_agents[rpc->mgtclass],
188			      len, rpc->timeout)) < 0) {
189		IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
190		return 0;
191	}
192
193	mad = umad_get_mad(rcvbuf);
194
195	if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) {
196		ERRS("MAD completed with error status 0x%x; dport (%s)",
197			status, portid2str(dport));
198		return 0;
199	}
200
201	if (ibdebug) {
202		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
203		xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
204	}
205
206	if (rcvdata)
207		memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
208
209	return rcvdata;
210}
211
212void *
213mad_rpc_rmpp(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport,
214	     ib_rmpp_hdr_t *rmpp, void *data)
215{
216	const struct ibmad_port *p = port_id;
217	int status, len;
218	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
219
220	memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
221
222	DEBUG("rmpp %p data %p", rmpp, data);
223
224	if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
225		return 0;
226
227	if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
228			      p->class_agents[rpc->mgtclass],
229			      len, rpc->timeout)) < 0) {
230		IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
231		return 0;
232	}
233
234	mad = umad_get_mad(rcvbuf);
235
236	if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
237		ERRS("MAD completed with error status 0x%x; dport (%s)",
238			status, portid2str(dport));
239		return 0;
240	}
241
242	if (ibdebug) {
243		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
244		xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
245		      rpc->datasz);
246	}
247
248	if (rmpp) {
249		rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
250		if ((rmpp->flags & 0x3) &&
251		    mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
252			IBWARN("bad rmpp version");
253			return 0;
254		}
255		rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
256		rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
257		DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
258		rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
259		rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
260	}
261
262	if (data)
263		memcpy(data, mad + rpc->dataoffs, rpc->datasz);
264
265	rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
266
267	return data;
268}
269
270void *
271madrpc(ib_rpc_t *rpc, ib_portid_t *dport, void *payload, void *rcvdata)
272{
273	struct ibmad_port port;
274
275	port.port_id = mad_portid;
276	port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass);
277	return mad_rpc(&port, rpc, dport, payload, rcvdata);
278}
279
280void *
281madrpc_rmpp(ib_rpc_t *rpc, ib_portid_t *dport, ib_rmpp_hdr_t *rmpp, void *data)
282{
283	struct ibmad_port port;
284
285	port.port_id = mad_portid;
286	port.class_agents[rpc->mgtclass] = mad_class_agent(rpc->mgtclass);
287	return mad_rpc_rmpp(&port, rpc, dport, rmpp, data);
288}
289
290static pthread_mutex_t rpclock = PTHREAD_MUTEX_INITIALIZER;
291
292void
293madrpc_lock(void)
294{
295	pthread_mutex_lock(&rpclock);
296}
297
298void
299madrpc_unlock(void)
300{
301	pthread_mutex_unlock(&rpclock);
302}
303
304void
305madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
306{
307	if (umad_init() < 0)
308		IBPANIC("can't init UMAD library");
309
310	if ((mad_portid = umad_open_port(dev_name, dev_port)) < 0)
311		IBPANIC("can't open UMAD port (%s:%d)", dev_name, dev_port);
312
313	if (num_classes >= MAX_CLASS)
314		IBPANIC("too many classes %d requested", num_classes);
315
316	while (num_classes--) {
317		int rmpp_version = 0;
318		int mgmt = *mgmt_classes++;
319
320		if (mgmt == IB_SA_CLASS)
321			rmpp_version = 1;
322		if (mad_register_client(mgmt, rmpp_version) < 0)
323			IBPANIC("client_register for mgmt class %d failed", mgmt);
324	}
325}
326
327void *
328mad_rpc_open_port(char *dev_name, int dev_port,
329		  int *mgmt_classes, int num_classes)
330{
331	struct ibmad_port *p;
332	int port_id;
333
334	if (num_classes >= MAX_CLASS) {
335		IBWARN("too many classes %d requested", num_classes);
336		errno = EINVAL;
337		return NULL;
338	}
339
340	if (umad_init() < 0) {
341		IBWARN("can't init UMAD library");
342		errno = ENODEV;
343		return NULL;
344	}
345
346	p = malloc(sizeof(*p));
347	if (!p) {
348		errno = ENOMEM;
349		return NULL;
350	}
351	memset(p, 0, sizeof(*p));
352
353	if ((port_id = umad_open_port(dev_name, dev_port)) < 0) {
354		IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
355		if (!errno)
356			errno = EIO;
357		free(p);
358		return NULL;
359	}
360
361	while (num_classes--) {
362		int rmpp_version = 0;
363		int mgmt = *mgmt_classes++;
364		int agent;
365
366		if (mgmt == IB_SA_CLASS)
367			rmpp_version = 1;
368		if (mgmt < 0 || mgmt >= MAX_CLASS ||
369		    (agent = mad_register_port_client(port_id, mgmt,
370						      rmpp_version)) < 0) {
371			IBWARN("client_register for mgmt %d failed", mgmt);
372			if(!errno)
373				errno = EINVAL;
374			umad_close_port(port_id);
375  			free(p);
376  			return NULL;
377		}
378		p->class_agents[mgmt] = agent;
379	}
380
381	p->port_id = port_id;
382	return p;
383}
384
385void
386mad_rpc_close_port(void *port_id)
387{
388	struct ibmad_port *p = port_id;
389
390	umad_close_port(p->port_id);
391	free(p);
392}
393
394uint8_t *
395sa_call(void *rcvbuf, ib_portid_t *portid, ib_sa_call_t *sa, unsigned timeout)
396{
397	struct ibmad_port port;
398
399	port.port_id = mad_portid;
400	port.class_agents[IB_SA_CLASS] = mad_class_agent(IB_SA_CLASS);
401	return sa_rpc_call(&port, rcvbuf, portid, sa, timeout);
402}
403