1/*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses.  You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 *     Redistribution and use in source and binary forms, with or
13 *     without modification, are permitted provided that the following
14 *     conditions are met:
15 *
16 *      - Redistributions of source code must retain the above
17 *        copyright notice, this list of conditions and the following
18 *        disclaimer.
19 *
20 *      - Redistributions in binary form must reproduce the above
21 *        copyright notice, this list of conditions and the following
22 *        disclaimer in the documentation and/or other materials
23 *        provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36/*  AUTHOR                 Edward Bortnikov
37 *
38 *  DESCRIPTION
39 *     The lower-level MAD transport interface implementation
40 *     that allows sending a single MAD/receiving a callback
41 *     when a single MAD is received.
42 */
43
44#if HAVE_CONFIG_H
45#  include <config.h>
46#endif				/* HAVE_CONFIG_H */
47
48#include <sys/types.h>
49#include <sys/stat.h>
50#include <sys/ioctl.h>
51#include <fcntl.h>
52#include <stddef.h>
53#include <errno.h>
54#include <stdlib.h>
55#include <string.h>
56
57#include <vendor/osm_vendor_api.h>
58#include <vendor/osm_vendor_mlx_transport.h>
59#include <vendor/osm_vendor_mlx_transport_anafa.h>
60#include <vendor/osm_vendor_mlx_dispatcher.h>
61#include <vendor/osm_vendor_mlx_svc.h>
62#include <vendor/osm_ts_useraccess.h>
63
64static void
65__osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
66					  IN struct ib_mad *p_mad,
67					  IN uint8_t is_smi,
68					  OUT osm_mad_addr_t * p_mad_addr);
69
70static void
71__osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t *
72					  p_mad_addr, IN uint8_t is_smi,
73					  OUT struct ib_mad *p_mad);
74
75void __osmv_TOPSPIN_ANAFA_receiver_thr(void *p_ctx)
76{
77	int ts_ret_code;
78	struct ib_mad mad;
79	osm_mad_addr_t mad_addr;
80	osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
81	ib_api_status_t status = IB_SUCCESS;
82
83	OSM_LOG_ENTER(p_bo->p_vendor->p_log);
84
85	/* Make sure the p_bo object is still relevant */
86	if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
87		return;
88
89	/* we set the type of cancelation for this thread */
90	/* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
91
92	while (1) {
93		/* Make sure the p_bo object is still relevant */
94		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
95			return;
96
97		/* we read one mad at a time and pass it to the read callback function */
98		ts_ret_code =
99		    read(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
100								  p_transp_mgr))->
101			 device_fd, &mad, sizeof(mad));
102
103		/* Make sure the p_bo object is still relevant */
104		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
105			return;
106
107		if (ts_ret_code != sizeof(mad)) {
108			osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
109				"__osmv_TOPSPIN_ANAFA_receiver_thr: ERR 6903: "
110				"error with read, bytes = %d\n", ts_ret_code);
111			break;
112		} else {
113			osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
114				"__osmv_TOPSPIN_ANAFA_receiver_thr: "
115				"MAD QPN:%d SLID:0x%04x class:0x%02x "
116				"method:0x%02x attr:0x%04x status:0x%04x "
117				"tid:0x%016" PRIx64 "\n",
118				mad.dqpn,
119				cl_ntoh16(mad.slid),
120				mad.mgmt_class,
121				mad.r_method,
122				cl_ntoh16(mad.attribute_id),
123				cl_ntoh16(mad.status),
124				cl_ntoh64(mad.transaction_id));
125
126			/* first arrange an address */
127			__osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr
128			    (p_bo->p_vendor, &mad,
129			     (((ib_mad_t *) & mad)->mgmt_class ==
130			      IB_MCLASS_SUBN_LID)
131			     || (((ib_mad_t *) & mad)->mgmt_class ==
132				 IB_MCLASS_SUBN_DIR), &mad_addr);
133
134			/* call the receiver callback */
135
136			status =
137			    osmv_dispatch_mad((osm_bind_handle_t) p_bo,
138					      (void *)&mad, &mad_addr);
139
140			/* Make sure the p_bo object is still relevant */
141			if (p_bo->magic_ptr != p_bo)
142				return;
143
144			if (IB_INTERRUPTED == status) {
145
146				osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
147					"__osmv_TOPSPIN_ANAFA_receiver_thr: "
148					"The bind handle %p is being closed. "
149					"Breaking the loop.\n", p_bo);
150				break;
151			}
152		}
153	}
154
155	OSM_LOG_EXIT(p_bo->p_vendor->p_log);
156}
157
158/*
159 * NAME
160 *   osmv_transport_init
161 *
162 * DESCRIPTION
163 *   Setup the MAD transport infrastructure (filters, callbacks etc).
164 */
165
166ib_api_status_t
167osmv_transport_init(IN osm_bind_info_t * p_info,
168		    IN char hca_id[VENDOR_HCA_MAXNAMES],
169		    IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
170{
171	cl_status_t cl_st;
172
173	int ts_ioctl_ret;
174	int device_fd;
175	char *device_file = "/dev/ts_ua0";
176	osm_ts_user_mad_filter filter;
177	osmv_TOPSPIN_ANAFA_transport_mgr_t *p_mgr;
178	osmv_TOPSPIN_ANAFA_transport_info_t *p_tpot_info;
179	p_tpot_info =
180	    (osmv_TOPSPIN_ANAFA_transport_info_t *) p_bo->p_vendor->
181	    p_transport_info;
182
183	p_mgr = malloc(sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
184	if (!p_mgr) {
185		return IB_INSUFFICIENT_MEMORY;
186	}
187
188	memset(p_mgr, 0, sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
189
190	/* open TopSpin file device */
191	device_fd = open(device_file, O_RDWR);
192	if (device_fd < 0) {
193		fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
194			device_file, errno);
195		return IB_ERROR;
196	}
197	p_mgr->device_fd = device_fd;
198
199	/*
200	 * Create the MAD filter on this file handle.
201	 */
202
203	filter.port = 0;	/* Victor */
204	filter.direction = TS_IB_MAD_DIRECTION_IN;
205	filter.mask =
206	    TS_IB_MAD_FILTER_DIRECTION |
207	    TS_IB_MAD_FILTER_PORT |
208	    TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
209
210	switch (p_info->mad_class) {
211	case IB_MCLASS_SUBN_LID:
212	case IB_MCLASS_SUBN_DIR:
213		filter.qpn = 0;
214		filter.mgmt_class = IB_MCLASS_SUBN_LID;
215		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
216		if (ts_ioctl_ret < 0) {
217			return IB_ERROR;
218		}
219
220		filter.mgmt_class = IB_MCLASS_SUBN_DIR;
221		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
222		if (ts_ioctl_ret < 0) {
223			return IB_ERROR;
224		}
225
226		break;
227
228	case IB_MCLASS_SUBN_ADM:
229	default:
230		filter.qpn = 1;
231		filter.mgmt_class = p_info->mad_class;
232		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
233		if (ts_ioctl_ret < 0) {
234			return IB_ERROR;
235		}
236		break;
237	}
238
239	p_bo->p_transp_mgr = p_mgr;
240
241	/* Initialize the magic_ptr to the pointer of the p_bo info.
242	   This will be used to signal when the object is being destroyed, so no
243	   real action will be done then. */
244	p_bo->magic_ptr = p_bo;
245
246	/* init receiver thread */
247	cl_st =
248	    cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_ANAFA_receiver_thr,
249			   (void *)p_bo, "osmv TOPSPIN_ANAFA rcv thr");
250
251	return (ib_api_status_t) cl_st;
252}
253
254/*
255 * NAME
256 *   osmv_transport_send_mad
257 *
258 * DESCRIPTION
259 *   Send a single MAD (256 byte)
260 */
261
262ib_api_status_t
263osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
264			IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
265{
266
267	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
268	osm_vendor_t const *p_vend = p_bo->p_vendor;
269	struct ib_mad ts_mad = { 0 };
270	int ret;
271	ib_api_status_t status;
272
273	const ib_mad_t *p_mad_hdr = p_mad;
274
275	OSM_LOG_ENTER(p_vend->p_log);
276
277	/* Make sure the p_bo object is still relevant */
278	if (p_bo->magic_ptr != p_bo)
279		return IB_INVALID_CALLBACK;
280
281	/*
282	 * Copy the MAD over to the sent mad
283	 */
284	memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
285
286	/*
287	 * For all sends other than directed route SM MADs,
288	 * acquire an address vector for the destination.
289	 */
290	if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
291
292		__osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(p_mad_addr,
293							  p_mad_hdr->
294							  mgmt_class ==
295							  IB_MCLASS_SUBN_LID,
296							  &ts_mad);
297	} else {
298		/* is a directed route - we need to construct a permissive address */
299		/* we do not need port number since it is part of the mad_hndl */
300		ts_mad.dlid = IB_LID_PERMISSIVE;
301		ts_mad.slid = IB_LID_PERMISSIVE;
302	}
303	if ((p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_DIR) ||
304	    (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_LID)) {
305		ts_mad.sqpn = 0;
306		ts_mad.dqpn = 0;
307	} else {
308		ts_mad.sqpn = 1;
309		ts_mad.dqpn = 1;
310	}
311
312	/* ts_mad.port = p_bo->port_num; */
313	ts_mad.port = 0;	/* Victor */
314
315	/* send it */
316	ret =
317	    write(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
318							   p_transp_mgr))->
319		  device_fd, &ts_mad, sizeof(ts_mad));
320
321	if (ret != sizeof(ts_mad)) {
322		osm_log(p_vend->p_log, OSM_LOG_ERROR,
323			"osmv_transport_mad_send: ERR 6904: "
324			"Error sending mad (%d).\n", ret);
325		status = IB_ERROR;
326		goto Exit;
327	}
328
329	status = IB_SUCCESS;
330
331Exit:
332	OSM_LOG_EXIT(p_vend->p_log);
333	return (status);
334}
335
336void osmv_transport_done(IN const osm_bind_handle_t h_bind)
337{
338	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
339	osmv_TOPSPIN_ANAFA_transport_mgr_t *p_tpot_mgr =
340	    (osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->p_transp_mgr);
341
342	CL_ASSERT(p_bo);
343
344	/* First of all - zero out the magic_ptr, so if a callback is called -
345	   it'll know that we are currently closing down, and will not handle the
346	   mad. */
347	p_bo->magic_ptr = 0;
348
349	/* usleep(3000000); */
350
351	/* pthread_cancel (p_tpot_mgr->receiver.osd.id); */
352	cl_thread_destroy(&(p_tpot_mgr->receiver));
353	free(p_tpot_mgr);
354}
355
356static void
357__osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
358					  IN uint8_t is_smi,
359					  OUT struct ib_mad *p_mad)
360{
361
362	/* For global destination or Multicast address: */
363	p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
364	p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
365	if (is_smi) {
366		p_mad->sqpn = 0;
367		p_mad->dqpn = 0;
368	} else {
369		p_mad->sqpn = 1;
370		p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
371	}
372	/*
373	   HACK we limit to the first PKey Index assuming it will
374	   always be the default PKey
375	 */
376	p_mad->pkey_index = 0;
377}
378
379static void
380__osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
381					  IN struct ib_mad *p_mad,
382					  IN uint8_t is_smi,
383					  OUT osm_mad_addr_t * p_mad_addr)
384{
385	p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
386	p_mad_addr->static_rate = 0;
387	p_mad_addr->path_bits = 0;
388	if (is_smi) {
389		/* SMI */
390		p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
391		p_mad_addr->addr_type.smi.port_num = p_mad->port;
392	} else {
393		/* GSI */
394		p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
395		p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
396		p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
397		p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
398
399		p_mad_addr->addr_type.gsi.global_route = FALSE;
400		/* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
401		/*
402		   if (p_mad_addr->addr_type.gsi.global_route)
403		   {
404		   p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
405		   ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
406		   p_rcv_desc->grh.traffic_class,
407		   p_rcv_desc->grh.flow_label);
408		   p_mad_addr->addr_type.gsi.grh_info.hop_limit =  p_rcv_desc->grh.hop_limit;
409		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
410		   &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
411		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
412		   p_rcv_desc->grh.dgid,  sizeof(ib_net64_t));
413		   }
414		 */
415	}
416}
417