1/* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36/* AUTHOR Eitan Zahavi 37 * 38 * DESCRIPTION 39 * The lower-level MAD transport interface implementation 40 * that allows sending a single MAD/receiving a callback 41 * when a single MAD is received. 42 */ 43 44#if HAVE_CONFIG_H 45# include <config.h> 46#endif /* HAVE_CONFIG_H */ 47 48#include <sys/types.h> 49#include <sys/stat.h> 50#include <sys/ioctl.h> 51#include <fcntl.h> 52#include <errno.h> 53#include <stdlib.h> 54#include <string.h> 55 56#include <vendor/osm_vendor_api.h> 57#include <vendor/osm_vendor_mlx_transport.h> 58#include <vendor/osm_vendor_mlx_dispatcher.h> 59#include <vendor/osm_vendor_mlx_svc.h> 60#include <complib/cl_thread.h> 61 62/* the simulator messages definition */ 63#include <ibmgtsim/ibms_client_api.h> 64 65typedef struct _osmv_ibms_transport_mgr { 66 ibms_conn_handle_t conHdl; /* the connection handle we talk to */ 67 ibms_bind_msg_t filter; /* the bind message defining the filtering */ 68 cl_thread_t receiver; /* the thread waiting for incomming messages */ 69} osmv_ibms_transport_mgr_t; 70 71static void 72__osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 73 IN struct _ibms_mad_addr *p_ibms_addr, 74 IN uint8_t is_smi, 75 OUT osm_mad_addr_t * p_osm_addr); 76 77static void 78__osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr, 79 IN uint8_t is_smi, 80 OUT struct _ibms_mad_addr *p_ibms_addr); 81 82/* this is the callback function the "server" will call on incoming 83 messages */ 84void __osmv_ibms_receiver_callback(void *p_ctx, ibms_mad_msg_t * p_mad) 85{ 86 osm_mad_addr_t mad_addr; 87 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx; 88 ib_api_status_t status = IB_SUCCESS; 89 90 /* Make sure the p_bo object is still relevant */ 91 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 92 return; 93 94 { 95 OSM_LOG_ENTER(p_bo->p_vendor->p_log); 96 97 /* some logging */ 98 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 99 "__osmv_ibms_receiver_callback: " 100 "MAD QPN:%d SLID:0x%04x class:0x%02x " 101 "method:0x%02x attr:0x%04x status:0x%04x " 102 "tid:0x%016" PRIx64 "\n", 103 p_mad->addr.dqpn, 104 cl_ntoh16(p_mad->addr.slid), 105 p_mad->header.mgmt_class, 106 p_mad->header.method, 107 cl_ntoh16(p_mad->header.attr_id), 108 cl_ntoh16(p_mad->header.status), 109 cl_ntoh64(p_mad->header.trans_id)); 110 111 /* first arrange an address */ 112 __osmv_ibms_mad_addr_to_osm_addr(p_bo->p_vendor, 113 &p_mad->addr, 114 (((ib_mad_t *) & p_mad-> 115 header)->mgmt_class == 116 IB_MCLASS_SUBN_LID) 117 || 118 (((ib_mad_t *) & p_mad-> 119 header)->mgmt_class == 120 IB_MCLASS_SUBN_DIR), 121 &mad_addr); 122 123 /* call the receiver callback */ 124 125 status = 126 osmv_dispatch_mad((osm_bind_handle_t) p_bo, 127 (void *)&p_mad->header, &mad_addr); 128 129 OSM_LOG_EXIT(p_bo->p_vendor->p_log); 130 } 131} 132 133ib_api_status_t 134osm_vendor_get_guid_by_ca_and_port(IN osm_vendor_t * const p_vend, 135 IN char *hca_id, 136 IN uint32_t port_num, 137 OUT uint64_t * p_port_guid); 138 139/* 140 * NAME 141 * osmv_transport_init 142 * 143 * DESCRIPTION 144 * Setup the MAD transport infrastructure (filters, callbacks etc). 145 */ 146 147ib_api_status_t 148osmv_transport_init(IN osm_bind_info_t * p_info, 149 IN char hca_id[VENDOR_HCA_MAXNAMES], 150 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo) 151{ 152 ibms_conn_handle_t conHdl; /* the connection we talk to the simulator through */ 153 osmv_ibms_transport_mgr_t *p_mgr = 154 malloc(sizeof(osmv_ibms_transport_mgr_t)); 155 int qpn; 156 int ibms_status; 157 uint64_t port_guid; 158 159 if (!p_mgr) { 160 return IB_INSUFFICIENT_MEMORY; 161 } 162 163 memset(p_mgr, 0, sizeof(osmv_ibms_transport_mgr_t)); 164 165 /* create the client socket connected to the simulator */ 166 /* also perform the "connect" message - such that we 167 validate the target guid */ 168 if (osm_vendor_get_guid_by_ca_and_port 169 (p_bo->p_vendor, hca_id, p_bo->port_num, &port_guid)) { 170 return IB_INVALID_GUID; 171 } 172 173 conHdl = 174 ibms_connect(port_guid, __osmv_ibms_receiver_callback, 175 (void *)p_bo); 176 if (!conHdl) { 177 printf("fail to connect to the server.\n"); 178 exit(1); 179 } 180 181 /* 182 * Create the MAD filter on this file handle. 183 */ 184 185 p_mgr->filter.port = p_bo->port_num; 186 p_mgr->filter.only_input = 1; 187 p_mgr->filter.mask = 188 IBMS_BIND_MASK_PORT | 189 IBMS_BIND_MASK_INPUT | IBMS_BIND_MASK_QP | IBMS_BIND_MASK_CLASS; 190 191 switch (p_info->mad_class) { 192 case IB_MCLASS_SUBN_LID: 193 case IB_MCLASS_SUBN_DIR: 194 qpn = 0; 195 p_mgr->filter.qpn = qpn; 196 p_mgr->filter.mgt_class = IB_MCLASS_SUBN_LID; 197 ibms_status = ibms_bind(conHdl, &p_mgr->filter); 198 if (ibms_status) { 199 return IB_ERROR; 200 } 201 202 p_mgr->filter.mgt_class = IB_MCLASS_SUBN_DIR; 203 ibms_status = ibms_bind(conHdl, &p_mgr->filter); 204 if (ibms_status) { 205 return IB_ERROR; 206 } 207 208 break; 209 210 case IB_MCLASS_SUBN_ADM: 211 default: 212 qpn = 1; 213 p_mgr->filter.qpn = qpn; 214 p_mgr->filter.mgt_class = p_info->mad_class; 215 ibms_status = ibms_bind(conHdl, &p_mgr->filter); 216 if (ibms_status) { 217 return IB_ERROR; 218 } 219 break; 220 } 221 222 p_mgr->conHdl = conHdl; 223 224 p_bo->p_transp_mgr = p_mgr; 225 226 /* Initialize the magic_ptr to the pointer of the p_bo info. 227 This will be used to signal when the object is being destroyed, so no 228 real action will be done then. */ 229 p_bo->magic_ptr = p_bo; 230 231 return IB_SUCCESS; 232} 233 234/* 235 * NAME 236 * osmv_transport_send_mad 237 * 238 * DESCRIPTION 239 * Send a single MAD (256 byte) 240 */ 241 242ib_api_status_t 243osmv_transport_mad_send(IN const osm_bind_handle_t h_bind, 244 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr) 245{ 246 247 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 248 osm_vendor_t const *p_vend = p_bo->p_vendor; 249 int ret; 250 ibms_mad_msg_t mad_msg; 251 ib_api_status_t status; 252 253 const ib_mad_t *p_mad_hdr = p_mad; 254 255 OSM_LOG_ENTER(p_vend->p_log); 256 257 memset(&mad_msg, 0, sizeof(mad_msg)); 258 259 /* Make sure the p_bo object is still relevant */ 260 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 261 return IB_INVALID_CALLBACK; 262 263 /* 264 * Copy the MAD over to the sent mad 265 */ 266 memcpy(&mad_msg.header, p_mad_hdr, MAD_BLOCK_SIZE); 267 268 /* 269 * For all sends other than directed route SM MADs, 270 * acquire an address vector for the destination. 271 */ 272 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) { 273 274 __osmv_ibms_osm_addr_to_mad_addr(p_mad_addr, 275 p_mad_hdr->mgmt_class == 276 IB_MCLASS_SUBN_LID, 277 &mad_msg.addr); 278 } else { 279 /* is a directed route - we need to construct a permissive address */ 280 /* we do not need port number since it is part of the mad_hndl */ 281 mad_msg.addr.dlid = IB_LID_PERMISSIVE; 282 mad_msg.addr.slid = IB_LID_PERMISSIVE; 283 mad_msg.addr.sqpn = 0; 284 mad_msg.addr.dqpn = 0; 285 } 286 287 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 288 "osmv_transport_mad_send: " 289 "Sending QPN:%d DLID:0x%04x class:0x%02x " 290 "method:0x%02x attr:0x%04x status:0x%04x " 291 "tid:0x%016" PRIx64 "\n", 292 mad_msg.addr.dqpn, 293 cl_ntoh16(mad_msg.addr.dlid), 294 mad_msg.header.mgmt_class, 295 mad_msg.header.method, 296 cl_ntoh16(mad_msg.header.attr_id), 297 cl_ntoh16(mad_msg.header.status), 298 cl_ntoh64(mad_msg.header.trans_id) 299 ); 300 301 /* send it */ 302 ret = 303 ibms_send(((osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr))-> 304 conHdl, &mad_msg); 305 if (ret) { 306 osm_log(p_vend->p_log, OSM_LOG_ERROR, 307 "osmv_transport_mad_send: ERR 5304: " 308 "Error sending mad (%d).\n", ret); 309 status = IB_ERROR; 310 goto Exit; 311 } 312 313 status = IB_SUCCESS; 314 315Exit: 316 OSM_LOG_EXIT(p_vend->p_log); 317 return (status); 318} 319 320void osmv_transport_done(IN const osm_bind_handle_t h_bind) 321{ 322 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 323 osmv_ibms_transport_mgr_t *p_tpot_mgr = 324 (osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr); 325 326 CL_ASSERT(p_bo); 327 328 /* First of all - zero out the magic_ptr, so if a callback is called - 329 it'll know that we are currently closing down, and will not handle the 330 mad. */ 331 p_bo->magic_ptr = 0; 332 /* usleep(3000000); */ 333 334 ibms_disconnect(p_tpot_mgr->conHdl); 335 336 /* seems the only way to abort a blocking read is to make it read something */ 337 free(p_tpot_mgr); 338} 339 340static void 341__osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr, 342 IN uint8_t is_smi, 343 OUT struct _ibms_mad_addr *p_ibms_addr) 344{ 345 346 /* For global destination or Multicast address: */ 347 p_ibms_addr->dlid = cl_ntoh16(p_osm_addr->dest_lid); 348 p_ibms_addr->sl = p_osm_addr->addr_type.gsi.service_level; 349 if (is_smi) { 350 p_ibms_addr->sqpn = 0; 351 p_ibms_addr->dqpn = 0; 352 } else { 353 p_ibms_addr->sqpn = 1; 354 p_ibms_addr->dqpn = 355 cl_ntoh32(p_osm_addr->addr_type.gsi.remote_qp); 356 } 357 /* 358 HACK we limit to the first PKey Index assuming it will 359 always be the default PKey 360 */ 361 p_ibms_addr->pkey_index = 0; 362} 363 364static void 365__osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 366 IN struct _ibms_mad_addr *p_ibms_addr, 367 IN uint8_t is_smi, 368 OUT osm_mad_addr_t * p_osm_addr) 369{ 370 memset(p_osm_addr, 0, sizeof(osm_mad_addr_t)); 371 p_osm_addr->dest_lid = cl_hton16(p_ibms_addr->slid); 372 p_osm_addr->static_rate = 0; 373 p_osm_addr->path_bits = 0; 374 if (is_smi) { 375 /* SMI */ 376 p_osm_addr->addr_type.smi.source_lid = 377 cl_hton16(p_ibms_addr->slid); 378 p_osm_addr->addr_type.smi.port_num = 1; /* TODO add if required p_ibms_addr->port; */ 379 } else { 380 /* GSI */ 381 p_osm_addr->addr_type.gsi.remote_qp = 382 cl_ntoh32(p_ibms_addr->sqpn); 383 p_osm_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 384 p_osm_addr->addr_type.gsi.pkey_ix = p_ibms_addr->pkey_index; 385 p_osm_addr->addr_type.gsi.service_level = p_ibms_addr->sl; 386 387 p_osm_addr->addr_type.gsi.global_route = FALSE; 388 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */ 389 /* 390 if (p_osm_addr->addr_type.gsi.global_route) 391 { 392 p_osm_addr->addr_type.gsi.grh_info.ver_class_flow = 393 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version, 394 p_rcv_desc->grh.traffic_class, 395 p_rcv_desc->grh.flow_label); 396 p_osm_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit; 397 memcpy(&p_osm_addr->addr_type.gsi.grh_info.src_gid.raw, 398 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 399 memcpy(&p_osm_addr->addr_type.gsi.grh_info.dest_gid.raw, 400 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 401 } 402 */ 403 } 404} 405 406/* 407 * NAME osm_vendor_set_sm 408 * 409 * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit 410 * according to the value given (TRUE or FALSE). 411 */ 412 413void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 414{ 415 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 416 osm_vendor_t const *p_vend = p_bo->p_vendor; 417 int ret; 418 ibms_cap_msg_t cap_msg; 419 420 OSM_LOG_ENTER(p_vend->p_log); 421 422 cap_msg.mask = IB_PORT_CAP_IS_SM; 423 if (is_sm_val) 424 cap_msg.capabilities = IB_PORT_CAP_IS_SM; 425 else 426 cap_msg.capabilities = 0; 427 428 ret = ibms_set_cap(((osmv_ibms_transport_mgr_t *) (p_bo-> 429 p_transp_mgr))-> 430 conHdl, &cap_msg); 431 432 if (ret) { 433 osm_log(p_vend->p_log, OSM_LOG_ERROR, 434 "osm_vendor_set_sm: ERR 5312: " 435 "Unable set 'IS_SM' bit to:%u in port attributes.\n", 436 is_sm_val); 437 } 438 OSM_LOG_EXIT(p_vend->p_log); 439} 440