1/*
2 * Copyright (c) 2010 Lawrence Livermore National Laboratory
3 * Copyright (c) 2011 Mellanox Technologies LTD.  All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses.  You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 *     Redistribution and use in source and binary forms, with or
12 *     without modification, are permitted provided that the following
13 *     conditions are met:
14 *
15 *      - Redistributions of source code must retain the above
16 *        copyright notice, this list of conditions and the following
17 *        disclaimer.
18 *
19 *      - Redistributions in binary form must reproduce the above
20 *        copyright notice, this list of conditions and the following
21 *        disclaimer in the documentation and/or other materials
22 *        provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35#if HAVE_CONFIG_H
36#  include <config.h>
37#endif				/* HAVE_CONFIG_H */
38
39#include <errno.h>
40#include <infiniband/ibnetdisc.h>
41#include <infiniband/umad.h>
42#include "internal.h"
43
44extern int mlnx_ext_port_info_err(smp_engine_t * engine, ibnd_smp_t * smp,
45				  uint8_t * mad, void *cb_data);
46
47static void queue_smp(smp_engine_t * engine, ibnd_smp_t * smp)
48{
49	smp->qnext = NULL;
50	if (!engine->smp_queue_head) {
51		engine->smp_queue_head = smp;
52		engine->smp_queue_tail = smp;
53	} else {
54		engine->smp_queue_tail->qnext = smp;
55		engine->smp_queue_tail = smp;
56	}
57}
58
59static ibnd_smp_t *get_smp(smp_engine_t * engine)
60{
61	ibnd_smp_t *head = engine->smp_queue_head;
62	ibnd_smp_t *tail = engine->smp_queue_tail;
63	ibnd_smp_t *rc = head;
64	if (head) {
65		if (tail == head)
66			engine->smp_queue_tail = NULL;
67		engine->smp_queue_head = head->qnext;
68	}
69	return rc;
70}
71
72static int send_smp(ibnd_smp_t * smp, smp_engine_t * engine)
73{
74	int rc = 0;
75	uint8_t umad[1024];
76	ib_rpc_t *rpc = &smp->rpc;
77	int agent = 0;
78
79	memset(umad, 0, umad_size() + IB_MAD_SIZE);
80
81	if (rpc->mgtclass == IB_SMI_CLASS) {
82		agent = engine->smi_agent;
83	} else if (rpc->mgtclass == IB_SMI_DIRECT_CLASS) {
84		agent = engine->smi_dir_agent;
85	} else {
86		IBND_ERROR("Invalid class for RPC\n");
87		return (-EIO);
88	}
89
90	if ((rc = mad_build_pkt(umad, &smp->rpc, &smp->path, NULL, NULL))
91	    < 0) {
92		IBND_ERROR("mad_build_pkt failed; %d\n", rc);
93		return rc;
94	}
95
96	if ((rc = umad_send(engine->umad_fd, agent, umad, IB_MAD_SIZE,
97			    engine->cfg->timeout_ms, engine->cfg->retries)) < 0) {
98		IBND_ERROR("send failed; %d\n", rc);
99		return rc;
100	}
101
102	return 0;
103}
104
105static int process_smp_queue(smp_engine_t * engine)
106{
107	int rc = 0;
108	ibnd_smp_t *smp;
109	while (cl_qmap_count(&engine->smps_on_wire)
110	       < engine->cfg->max_smps) {
111		smp = get_smp(engine);
112		if (!smp)
113			return 0;
114
115		if ((rc = send_smp(smp, engine)) != 0) {
116			free(smp);
117			return rc;
118		}
119		cl_qmap_insert(&engine->smps_on_wire, (uint32_t) smp->rpc.trid,
120			       (cl_map_item_t *) smp);
121		engine->total_smps++;
122	}
123	return 0;
124}
125
126int issue_smp(smp_engine_t * engine, ib_portid_t * portid,
127	      unsigned attrid, unsigned mod, smp_comp_cb_t cb, void *cb_data)
128{
129	ibnd_smp_t *smp = calloc(1, sizeof *smp);
130	if (!smp) {
131		IBND_ERROR("OOM\n");
132		return -ENOMEM;
133	}
134
135	smp->cb = cb;
136	smp->cb_data = cb_data;
137	smp->path = *portid;
138	smp->rpc.method = IB_MAD_METHOD_GET;
139	smp->rpc.attr.id = attrid;
140	smp->rpc.attr.mod = mod;
141	smp->rpc.timeout = engine->cfg->timeout_ms;
142	smp->rpc.datasz = IB_SMP_DATA_SIZE;
143	smp->rpc.dataoffs = IB_SMP_DATA_OFFS;
144	smp->rpc.trid = mad_trid();
145	smp->rpc.mkey = engine->cfg->mkey;
146
147	if (portid->lid <= 0 || portid->drpath.drslid == 0xffff ||
148	    portid->drpath.drdlid == 0xffff)
149		smp->rpc.mgtclass = IB_SMI_DIRECT_CLASS;	/* direct SMI */
150	else
151		smp->rpc.mgtclass = IB_SMI_CLASS;	/* Lid routed SMI */
152
153	portid->sl = 0;
154	portid->qp = 0;
155
156	queue_smp(engine, smp);
157	return process_smp_queue(engine);
158}
159
160static int process_one_recv(smp_engine_t * engine)
161{
162	int rc = 0;
163	int status = 0;
164	ibnd_smp_t *smp;
165	uint8_t *mad;
166	uint32_t trid;
167	uint8_t umad[sizeof(struct ib_user_mad) + IB_MAD_SIZE];
168	int length = umad_size() + IB_MAD_SIZE;
169
170	memset(umad, 0, sizeof(umad));
171
172	/* wait for the next message */
173	if ((rc = umad_recv(engine->umad_fd, umad, &length,
174			    -1)) < 0) {
175		IBND_ERROR("umad_recv failed: %d\n", rc);
176		return -1;
177	}
178
179	mad = umad_get_mad(umad);
180	trid = (uint32_t) mad_get_field64(mad, 0, IB_MAD_TRID_F);
181
182	smp = (ibnd_smp_t *) cl_qmap_remove(&engine->smps_on_wire, trid);
183	if ((cl_map_item_t *) smp == cl_qmap_end(&engine->smps_on_wire)) {
184		IBND_ERROR("Failed to find matching smp for trid (%x)\n", trid);
185		return -1;
186	}
187
188	rc = process_smp_queue(engine);
189	if (rc)
190		goto error;
191
192	if ((status = umad_status(umad))) {
193		IBND_ERROR("umad (%s Attr 0x%x:%u) bad status %d; %s\n",
194			   portid2str(&smp->path), smp->rpc.attr.id,
195			   smp->rpc.attr.mod, status, strerror(status));
196		if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
197			rc = mlnx_ext_port_info_err(engine, smp, mad,
198						    smp->cb_data);
199	} else if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F))) {
200		IBND_ERROR("mad (%s Attr 0x%x:%u) bad status 0x%x\n",
201			   portid2str(&smp->path), smp->rpc.attr.id,
202			   smp->rpc.attr.mod, status);
203		if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
204			rc = mlnx_ext_port_info_err(engine, smp, mad,
205						    smp->cb_data);
206	} else
207		rc = smp->cb(engine, smp, mad, smp->cb_data);
208
209error:
210	free(smp);
211	return rc;
212}
213
214int smp_engine_init(smp_engine_t * engine, char * ca_name, int ca_port,
215		    void *user_data, ibnd_config_t *cfg)
216{
217	memset(engine, 0, sizeof(*engine));
218
219	if (umad_init() < 0) {
220		IBND_ERROR("umad_init failed\n");
221		return -EIO;
222	}
223
224	engine->umad_fd = umad_open_port(ca_name, ca_port);
225	if (engine->umad_fd < 0) {
226		IBND_ERROR("can't open UMAD port (%s:%d)\n", ca_name, ca_port);
227		return -EIO;
228	}
229
230	if ((engine->smi_agent = umad_register(engine->umad_fd,
231	     IB_SMI_CLASS, 1, 0, 0)) < 0) {
232		IBND_ERROR("Failed to register SMI agent on (%s:%d)\n",
233			   ca_name, ca_port);
234		goto eio_close;
235	}
236
237	if ((engine->smi_dir_agent = umad_register(engine->umad_fd,
238	     IB_SMI_DIRECT_CLASS, 1, 0, 0)) < 0) {
239		IBND_ERROR("Failed to register SMI_DIRECT agent on (%s:%d)\n",
240			   ca_name, ca_port);
241		goto eio_close;
242	}
243
244	engine->user_data = user_data;
245	cl_qmap_init(&engine->smps_on_wire);
246	engine->cfg = cfg;
247	return (0);
248
249eio_close:
250	umad_close_port(engine->umad_fd);
251	return (-EIO);
252}
253
254void smp_engine_destroy(smp_engine_t * engine)
255{
256	cl_map_item_t *item;
257	ibnd_smp_t *smp;
258
259	/* remove queued smps */
260	smp = get_smp(engine);
261	if (smp)
262		IBND_ERROR("outstanding SMP's\n");
263	for ( /* */ ; smp; smp = get_smp(engine))
264		free(smp);
265
266	/* remove smps from the wire queue */
267	item = cl_qmap_head(&engine->smps_on_wire);
268	if (item != cl_qmap_end(&engine->smps_on_wire))
269		IBND_ERROR("outstanding SMP's on wire\n");
270	for ( /* */ ; item != cl_qmap_end(&engine->smps_on_wire);
271	     item = cl_qmap_head(&engine->smps_on_wire)) {
272		cl_qmap_remove_item(&engine->smps_on_wire, item);
273		free(item);
274	}
275
276	umad_close_port(engine->umad_fd);
277}
278
279int process_mads(smp_engine_t * engine)
280{
281	int rc;
282	while (!cl_is_qmap_empty(&engine->smps_on_wire))
283		if ((rc = process_one_recv(engine)) != 0)
284			return rc;
285	return 0;
286}
287