osm_vendor_ts.c revision 219820
1251839Sbapt/* 2251839Sbapt * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3251839Sbapt * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4251839Sbapt * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5251839Sbapt * 6251839Sbapt * This software is available to you under a choice of one of two 7251839Sbapt * licenses. You may choose to be licensed under the terms of the GNU 8251839Sbapt * General Public License (GPL) Version 2, available from the file 9251839Sbapt * COPYING in the main directory of this source tree, or the 10251839Sbapt * OpenIB.org BSD license below: 11251839Sbapt * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36#undef __init 37#if HAVE_CONFIG_H 38# include <config.h> 39#endif /* HAVE_CONFIG_H */ 40 41#include <stdlib.h> 42#include <string.h> 43#include <vendor/osm_vendor_ts.h> 44#include <vendor/osm_vendor_api.h> 45#include <vendor/osm_ts_useraccess.h> 46#include <opensm/osm_subnet.h> 47#include <opensm/osm_opensm.h> 48 49/* 50 Since a race can accure on requests. Meaning - a response is received before 51 the send_callback is called - we will save both the madw_p and the fact 52 whether or not it is a response. A race can occure only on requests that did 53 not fail, and then the madw_p will be put back in the pool before the 54 callback. 55*/ 56uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw) 57{ 58 uint64_t wrid = 0; 59 60 CL_ASSERT(p_madw->p_mad); 61 62 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *)); 63 wrid = (wrid << 1) | 64 ib_mad_is_response(p_madw->p_mad) | 65 (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS); 66 return wrid; 67} 68 69void 70__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid, 71 OUT uint8_t * is_resp, 72 OUT osm_madw_t ** pp_madw) 73{ 74 *is_resp = wrid & 0x0000000000000001; 75 wrid = wrid >> 1; 76 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *)); 77} 78 79/********************************************************************** 80 * TS MAD to OSM ADDRESS VECTOR 81 **********************************************************************/ 82void 83__osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend, 84 IN struct ib_mad *p_mad, 85 IN uint8_t is_smi, 86 OUT osm_mad_addr_t * p_mad_addr) 87{ 88 p_mad_addr->dest_lid = cl_hton16(p_mad->slid); 89 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */ 90 p_mad_addr->path_bits = 0; /* HACK - no way to know in TS */ 91 if (is_smi) { 92 /* SMI */ 93 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid); 94 p_mad_addr->addr_type.smi.port_num = p_mad->port; 95 } else { 96 /* GSI */ 97 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn; 98 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 99 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index; 100 p_mad_addr->addr_type.gsi.service_level = 0; /* HACK no way to know */ 101 102 p_mad_addr->addr_type.gsi.global_route = FALSE; /* HACK no way to know */ 103 /* copy the GRH data if relevant */ 104 /* 105 if (p_mad_addr->addr_type.gsi.global_route) 106 { 107 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 108 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version, 109 p_rcv_desc->grh.traffic_class, 110 p_rcv_desc->grh.flow_label); 111 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit; 112 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 113 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 114 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 115 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 116 } 117 */ 118 } 119} 120 121/********************************************************************** 122 * OSM ADDR VECTOR TO TS MAD: 123 **********************************************************************/ 124void 125__osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr, 126 IN uint8_t is_smi, OUT struct ib_mad *p_mad) 127{ 128 129 /* For global destination or Multicast address: */ 130 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid); 131 p_mad->sl = 0; 132 if (is_smi) { 133 p_mad->sqpn = 0; 134 p_mad->dqpn = 0; 135 } else { 136 p_mad->sqpn = 1; 137 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp; 138 } 139} 140 141/********************************************************************** 142 **********************************************************************/ 143void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind) 144{ 145 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 146 osm_vendor_t *p_vend = p_bind->p_vend; 147 VAPI_ret_t status; 148 VAPI_hca_attr_t attr_mod; 149 VAPI_hca_attr_mask_t attr_mask; 150 151 OSM_LOG_ENTER(p_vend->p_log); 152 153 memset(&attr_mod, 0, sizeof(attr_mod)); 154 memset(&attr_mask, 0, sizeof(attr_mask)); 155 156 attr_mod.is_sm = FALSE; 157 attr_mask = HCA_ATTR_IS_SM; 158 159 status = 160 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 161 &attr_mask); 162 if (status != VAPI_OK) { 163 osm_log(p_vend->p_log, OSM_LOG_ERROR, 164 "__osm_vendor_clear_sm: ERR 5021: " 165 "Unable set 'IS_SM' bit in port attributes (%d).\n", 166 status); 167 } 168 169 OSM_LOG_EXIT(p_vend->p_log); 170} 171 172/********************************************************************** 173 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT 174 **********************************************************************/ 175void osm_vendor_construct(IN osm_vendor_t * const p_vend) 176{ 177 memset(p_vend, 0, sizeof(*p_vend)); 178 cl_thread_construct(&(p_vend->smi_bind.poller)); 179 cl_thread_construct(&(p_vend->gsi_bind.poller)); 180} 181 182/********************************************************************** 183 * DEALOCATE osm_vendor_t 184 **********************************************************************/ 185void osm_vendor_destroy(IN osm_vendor_t * const p_vend) 186{ 187 OSM_LOG_ENTER(p_vend->p_log); 188 osm_transaction_mgr_destroy(p_vend); 189 190 /* Destroy the poller threads */ 191 /* HACK: can you destroy an un-initialized thread ? */ 192 pthread_cancel(p_vend->smi_bind.poller.osd.id); 193 pthread_cancel(p_vend->gsi_bind.poller.osd.id); 194 cl_thread_destroy(&(p_vend->smi_bind.poller)); 195 cl_thread_destroy(&(p_vend->gsi_bind.poller)); 196 OSM_LOG_EXIT(p_vend->p_log); 197} 198 199/********************************************************************** 200DEALLOCATE A POINTER TO osm_vendor_t 201**********************************************************************/ 202void osm_vendor_delete(IN osm_vendor_t ** const pp_vend) 203{ 204 CL_ASSERT(pp_vend); 205 206 osm_vendor_destroy(*pp_vend); 207 free(*pp_vend); 208 *pp_vend = NULL; 209} 210 211/********************************************************************** 212 Initializes the vendor: 213**********************************************************************/ 214 215ib_api_status_t 216osm_vendor_init(IN osm_vendor_t * const p_vend, 217 IN osm_log_t * const p_log, IN const uint32_t timeout) 218{ 219 ib_api_status_t status = IB_SUCCESS; 220 221 OSM_LOG_ENTER(p_log); 222 223 p_vend->p_log = p_log; 224 p_vend->p_transaction_mgr = NULL; 225 osm_transaction_mgr_init(p_vend); 226 p_vend->timeout = timeout; 227 228 /* we use the file handle to track the binding */ 229 p_vend->smi_bind.ul_dev_fd = -1; 230 p_vend->gsi_bind.ul_dev_fd = -1; 231 232 OSM_LOG_EXIT(p_log); 233 return (status); 234} 235 236/********************************************************************** 237 * Create and Initialize osm_vendor_t Object 238 **********************************************************************/ 239osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log, 240 IN const uint32_t timeout) 241{ 242 ib_api_status_t status; 243 osm_vendor_t *p_vend; 244 245 OSM_LOG_ENTER(p_log); 246 247 CL_ASSERT(p_log); 248 249 p_vend = malloc(sizeof(*p_vend)); 250 if (p_vend != NULL) { 251 memset(p_vend, 0, sizeof(*p_vend)); 252 253 status = osm_vendor_init(p_vend, p_log, timeout); 254 if (status != IB_SUCCESS) { 255 osm_vendor_delete(&p_vend); 256 } 257 } else { 258 osm_log(p_vend->p_log, OSM_LOG_ERROR, 259 "osm_vendor_new: ERR 5007: " 260 "Fail to allocate vendor object.\n"); 261 } 262 263 OSM_LOG_EXIT(p_log); 264 return (p_vend); 265} 266 267/********************************************************************** 268 * TS RCV Thread callback 269 * HACK: - we need to make this support arbitrary size mads. 270 **********************************************************************/ 271void 272__osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind, 273 IN osm_mad_addr_t * p_mad_addr, 274 IN uint32_t mad_size, IN void *p_mad) 275{ 276 ib_api_status_t status; 277 osm_madw_t *p_req_madw = NULL; 278 osm_madw_t *p_madw; 279 osm_vend_wrap_t *p_new_vw; 280 ib_mad_t *p_mad_buf; 281 osm_log_t *const p_log = p_bind->p_vend->p_log; 282 283 OSM_LOG_ENTER(p_log); 284 285 /* if it is a response MAD we mustbe able to get the request */ 286 if (ib_mad_is_response((ib_mad_t *) p_mad)) { 287 /* can we find a matching madw by this payload TID */ 288 status = 289 osm_transaction_mgr_get_madw_for_tid(p_bind->p_vend, 290 (ib_mad_t *) p_mad, 291 &p_req_madw); 292 if (status != IB_SUCCESS) { 293 osm_log(p_log, OSM_LOG_ERROR, 294 "__osm_ts_rcv_callback: ERR 5008: " 295 "Error obtaining request madw by TID (%d).\n", 296 status); 297 p_req_madw = NULL; 298 } 299 300 if (p_req_madw == NULL) { 301 osm_log(p_log, OSM_LOG_ERROR, 302 "__osm_ts_rcv_callback: ERR 5009: " 303 "Fail to obtain request madw for receined MAD. Aborting CB.\n"); 304 goto Exit; 305 } 306 } 307 308 /* do we have a request ??? */ 309 if (p_req_madw == NULL) { 310 311 /* if not - get new osm_madw and arrange it. */ 312 /* create the new madw in the pool */ 313 p_madw = osm_mad_pool_get(p_bind->p_osm_pool, 314 (osm_bind_handle_t) p_bind, 315 mad_size, p_mad_addr); 316 if (p_madw == NULL) { 317 osm_log(p_log, OSM_LOG_ERROR, 318 "__osm_ts_rcv_callback: ERR 5010: " 319 "Error request for a new madw.\n"); 320 goto Exit; 321 } 322 /* HACK: we cust to avoid the const ??? */ 323 p_mad_buf = (void *)p_madw->p_mad; 324 } else { 325 /* we have the madw defined during the send and stored in the vend_wrap */ 326 /* we need to make sure the wrapper is correctly init there */ 327 CL_ASSERT(p_req_madw->vend_wrap.p_resp_madw != 0); 328 p_madw = p_req_madw->vend_wrap.p_resp_madw; 329 330 CL_ASSERT(p_madw->h_bind); 331 p_mad_buf = 332 osm_vendor_get(p_madw->h_bind, mad_size, 333 &p_madw->vend_wrap); 334 335 if (p_mad_buf == NULL) { 336 osm_log(p_log, OSM_LOG_ERROR, 337 "__osm_ts_rcv_callback: ERR 5011: " 338 "Unable to acquire wire MAD.\n"); 339 340 goto Exit; 341 } 342 343 /* 344 Finally, attach the wire MAD to this wrapper. 345 */ 346 osm_madw_set_mad(p_madw, p_mad_buf); 347 } 348 349 /* init some fields of the vendor wrapper */ 350 p_new_vw = osm_madw_get_vend_ptr(p_madw); 351 p_new_vw->h_bind = p_bind; 352 p_new_vw->size = mad_size; 353 p_new_vw->p_resp_madw = NULL; 354 p_new_vw->p_mad_buf = p_mad_buf; 355 356 memcpy(p_new_vw->p_mad_buf, p_mad, mad_size); 357 358 /* attach the buffer to the wrapper */ 359 p_madw->p_mad = p_mad_buf; 360 361 /* we can also make sure we marked the size and bind on the returned madw */ 362 p_madw->h_bind = p_new_vw->h_bind; 363 364 /* call the CB */ 365 (*(osm_vend_mad_recv_callback_t) p_bind->rcv_callback) 366 (p_madw, p_bind->client_context, p_req_madw); 367 368Exit: 369 OSM_LOG_EXIT(p_log); 370} 371 372/********************************************************************** 373 * TS Send callback : invoked after each send 374 * 375 **********************************************************************/ 376void 377__osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p, 378 IN boolean_t is_resp, 379 IN osm_madw_t * madw_p, IN IB_comp_status_t status) 380{ 381 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 382 osm_vend_wrap_t *p_vw; 383 384 OSM_LOG_ENTER(p_log); 385 386 osm_log(p_log, OSM_LOG_DEBUG, 387 "__osm_ts_send_callback: INFO 1008: " 388 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp); 389 390 /* we need to handle requests and responses differently */ 391 if (is_resp) { 392 if (status != IB_COMP_SUCCESS) { 393 osm_log(p_log, OSM_LOG_ERROR, 394 "__osm_ts_send_callback: ERR 5012: " 395 "Error Sending Response MADW:%p.\n", madw_p); 396 } else { 397 osm_log(p_log, OSM_LOG_DEBUG, 398 "__osm_ts_send_callback: DBG 1008: " 399 "Completed Sending Response MADW:%p.\n", 400 madw_p); 401 } 402 403 /* if we are a response - we need to clean it up */ 404 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p); 405 } else { 406 407 /* this call back is invoked on completion of send - error or not */ 408 if (status != IB_COMP_SUCCESS) { 409 410 osm_log(p_log, OSM_LOG_ERROR, 411 "__osm_ts_send_callback: ERR 5013: " 412 "Received an Error from IB_MGT Send (%d).\n", 413 status); 414 415 p_vw = osm_madw_get_vend_ptr(madw_p); 416 CL_ASSERT(p_vw); 417 418 /* 419 Return any wrappers to the pool that may have been 420 pre-emptively allocated to handle a receive. 421 */ 422 if (p_vw->p_resp_madw) { 423 osm_mad_pool_put(bind_info_p->p_osm_pool, 424 p_vw->p_resp_madw); 425 p_vw->p_resp_madw = NULL; 426 } 427 428 /* invoke the CB */ 429 (*(osm_vend_mad_send_err_callback_t) bind_info_p-> 430 send_err_callback) 431 (bind_info_p->client_context, madw_p); 432 } else { 433 /* successful request send - do nothing - the response will need the 434 out mad */ 435 osm_log(p_log, OSM_LOG_DEBUG, 436 "__osm_ts_send_callback: DBG 1008: " 437 "Completed Sending Request MADW:%p.\n", madw_p); 438 } 439 } 440 441 OSM_LOG_EXIT(p_log); 442} 443 444/********************************************************************** 445 * Poller thread: 446 * Always receive 256byte mads from the devcie file 447 **********************************************************************/ 448void __osm_vendor_ts_poller(IN void *p_ptr) 449{ 450 int ts_ret_code; 451 struct ib_mad mad; 452 osm_mad_addr_t mad_addr; 453 osm_ts_bind_info_t *const p_bind = (osm_ts_bind_info_t *) p_ptr; 454 455 OSM_LOG_ENTER(p_bind->p_vend->p_log); 456 /* we set the type of cancelation for this thread */ 457 pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); 458 459 while (1) { 460 /* we read one mad at a time and pass it to the read callback function */ 461 ts_ret_code = read(p_bind->ul_dev_fd, &mad, sizeof(mad)); 462 if (ts_ret_code != sizeof(mad)) { 463 osm_log(p_bind->p_vend->p_log, OSM_LOG_ERROR, 464 "__osm_vendor_ts_poller: ERR 5003: " 465 "error with read, bytes = %d, errno = %d\n", 466 ts_ret_code, errno); 467 } else { 468 osm_log(p_bind->p_vend->p_log, OSM_LOG_DEBUG, 469 "__osm_vendor_ts_poller: " 470 "MAD QPN:%d SLID:0x%04x class:0x%02x " 471 "__osm_vendor_ts_poller:0x%02x attr:0x%04x status:0x%04x " 472 "__osm_vendor_ts_poller:0x%016" PRIx64 "\n", 473 cl_ntoh32(mad.dqpn), 474 cl_ntoh16(mad.slid), 475 mad.mgmt_class, 476 mad.r_method, 477 cl_ntoh16(mad.attribute_id), 478 cl_ntoh16(mad.status), 479 cl_ntoh64(mad.transaction_id)); 480 481 /* first arrange an address */ 482 __osm_ts_conv_mad_rcv_desc_to_osm_addr(p_bind->p_vend, 483 &mad, 484 (((ib_mad_t *) & 485 mad)-> 486 mgmt_class == 487 IB_MCLASS_SUBN_LID) 488 || 489 (((ib_mad_t *) & 490 mad)-> 491 mgmt_class == 492 IB_MCLASS_SUBN_DIR), 493 &mad_addr); 494 495 /* call the receiver callback */ 496 /* HACK: this should be replaced with a call to the RMPP Assembly ... */ 497 __osm_ts_rcv_callback(p_bind, &mad_addr, 256, &mad); 498 } 499 } 500 501 OSM_LOG_EXIT(p_bind->p_vend->p_log); 502} 503 504/********************************************************************** 505 * BINDs a callback (rcv and send error) for a given class and method 506 * defined by the given: osm_bind_info_t 507 **********************************************************************/ 508osm_bind_handle_t 509osm_vendor_bind(IN osm_vendor_t * const p_vend, 510 IN osm_bind_info_t * const p_user_bind, 511 IN osm_mad_pool_t * const p_mad_pool, 512 IN osm_vend_mad_recv_callback_t mad_recv_callback, 513 IN osm_vend_mad_send_err_callback_t send_err_callback, 514 IN void *context) 515{ 516 ib_net64_t port_guid; 517 osm_ts_bind_info_t *p_bind = NULL; 518 VAPI_hca_hndl_t hca_hndl; 519 VAPI_hca_id_t hca_id; 520 uint32_t port_num; 521 ib_api_status_t status; 522 int device_fd; 523 char device_file[16]; 524 osm_ts_user_mad_filter filter; 525 int ts_ioctl_ret; 526 int qpn; 527 528 OSM_LOG_ENTER(p_vend->p_log); 529 530 CL_ASSERT(p_mad_pool); 531 532 port_guid = p_user_bind->port_guid; 533 534 osm_log(p_vend->p_log, OSM_LOG_INFO, 535 "osm_vendor_bind: " 536 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid)); 537 538 switch (p_user_bind->mad_class) { 539 case IB_MCLASS_SUBN_LID: 540 case IB_MCLASS_SUBN_DIR: 541 p_bind = &(p_vend->smi_bind); 542 qpn = 0; 543 break; 544 545 case IB_MCLASS_SUBN_ADM: 546 default: 547 p_bind = &(p_vend->gsi_bind); 548 qpn = 1; 549 break; 550 } 551 552 /* Make sure we did not previously opened the file */ 553 if (p_bind->ul_dev_fd >= 0) { 554 osm_log(p_vend->p_log, OSM_LOG_ERROR, 555 "osm_vendor_bind: ERR 5004: " 556 "Already binded to port %u\n", p_bind->port_num); 557 goto Exit; 558 } 559 560 /* 561 We need to figure out what is the TS file name to attach to. 562 I guess it is following the index of the port in the table of 563 ports. 564 */ 565 566 /* obtain the hca name and port num from the guid */ 567 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 568 "osm_vendor_bind: " 569 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n", 570 cl_ntoh64(port_guid)); 571 status = 572 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl, 573 &hca_id, &port_num); 574 if (status != IB_SUCCESS) { 575 osm_log(p_vend->p_log, OSM_LOG_ERROR, 576 "osm_vendor_bind: ERR 5005: " 577 "Fail to find port number of port guid:0x%016" PRIx64 578 "\n", port_guid); 579 goto Exit; 580 } 581 582 /* the file name is just /dev/ts_ua0: */ 583 strcpy(device_file, "/dev/ts_ua0"); 584 585 osm_log(p_vend->p_log, OSM_LOG_ERROR, 586 "osm_vendor_bind: " "Opening TS UL dev file:%s\n", device_file); 587 588 /* Open the file ... */ 589 device_fd = open(device_file, O_RDWR); 590 if (device_fd < 0) { 591 osm_log(p_vend->p_log, OSM_LOG_ERROR, 592 "osm_vendor_bind: ERR 5006: " 593 "Fail to open TS UL dev file:%s\n", device_file); 594 goto Exit; 595 } 596 597 /* track this bind request info */ 598 p_bind->ul_dev_fd = device_fd; 599 p_bind->port_num = port_num; 600 p_bind->p_vend = p_vend; 601 p_bind->client_context = context; 602 p_bind->rcv_callback = mad_recv_callback; 603 p_bind->send_err_callback = send_err_callback; 604 p_bind->p_osm_pool = p_mad_pool; 605 p_bind->hca_hndl = hca_hndl; 606 607 /* 608 * Create the MAD filter on this file handle. 609 */ 610 filter.port = port_num; 611 612 filter.qpn = qpn; 613 filter.mgmt_class = p_user_bind->mad_class; 614 filter.direction = TS_IB_MAD_DIRECTION_IN; 615 filter.mask = 616 TS_IB_MAD_FILTER_DIRECTION | 617 TS_IB_MAD_FILTER_PORT | 618 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS; 619 620 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter); 621 if (ts_ioctl_ret < 0) { 622 osm_log(p_vend->p_log, OSM_LOG_ERROR, 623 "osm_vendor_bind: ERR 5014: " 624 "Fail to register MAD filter with err:%u\n", 625 ts_ioctl_ret); 626 goto Exit; 627 } 628 629 /* Initialize the listener thread for this port */ 630 status = cl_thread_init(&p_bind->poller, 631 __osm_vendor_ts_poller, p_bind, 632 "osm ts poller"); 633 if (status != IB_SUCCESS) 634 goto Exit; 635 636Exit: 637 OSM_LOG_EXIT(p_vend->p_log); 638 return ((osm_bind_handle_t) p_bind); 639} 640 641/********************************************************************** 642Get a mad from the lower level. 643The osm_vend_wrap_t is a wrapper used to connect the mad to the response. 644**********************************************************************/ 645ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind, 646 IN const uint32_t mad_size, 647 IN osm_vend_wrap_t * const p_vw) 648{ 649 ib_mad_t *p_mad; 650 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 651 osm_vendor_t *p_vend = p_bind->p_vend; 652 653 OSM_LOG_ENTER(p_vend->p_log); 654 655 CL_ASSERT(p_vw); 656 657 p_vw->size = mad_size; 658 659 /* allocate it */ 660 p_mad = (ib_mad_t *) malloc(p_vw->size); 661 if (p_mad == NULL) { 662 osm_log(p_vend->p_log, OSM_LOG_ERROR, 663 "osm_vendor_get: ERR 5022: " 664 "Error Obtaining MAD buffer.\n"); 665 goto Exit; 666 } 667 668 memset(p_mad, 0, p_vw->size); 669 670 /* track locally */ 671 p_vw->p_mad_buf = p_mad; 672 p_vw->h_bind = h_bind; 673 p_vw->p_resp_madw = NULL; 674 675 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 676 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 677 "osm_vendor_get: " 678 "Acquired MAD %p, size = %u.\n", p_mad, p_vw->size); 679 } 680 681Exit: 682 OSM_LOG_EXIT(p_vend->p_log); 683 return (p_mad); 684} 685 686/********************************************************************** 687 * Return a MAD by providing it's wrapper object. 688 **********************************************************************/ 689void 690osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw) 691{ 692 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 693 osm_vendor_t *p_vend = p_bind->p_vend; 694 osm_madw_t *p_madw; 695 696 OSM_LOG_ENTER(p_vend->p_log); 697 698 CL_ASSERT(p_vw); 699 CL_ASSERT(p_vw->p_mad_buf); 700 701 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 702 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 703 "osm_vendor_put: " "Retiring MAD %p.\n", 704 p_vw->p_mad_buf); 705 } 706 707 /* 708 * We moved the removal of the transaction to immediatly after 709 * it was looked up. 710 */ 711 712 /* free the mad but the wrapper is part of the madw object */ 713 free(p_vw->p_mad_buf); 714 p_vw->p_mad_buf = NULL; 715 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap); 716 p_madw->p_mad = NULL; 717 718 OSM_LOG_EXIT(p_vend->p_log); 719} 720 721/********************************************************************** 722Actually Send a MAD 723 724MADs are buffers of type: struct ib_mad - so they are limited by size. 725This is for internal use by osm_vendor_send and the transaction mgr 726retry too. 727**********************************************************************/ 728ib_api_status_t 729osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind, IN osm_madw_t * const p_madw) 730{ 731 osm_vendor_t *const p_vend = p_bind->p_vend; 732 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); 733 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); 734 struct ib_mad ts_mad; 735 int ret; 736 ib_api_status_t status; 737 738 OSM_LOG_ENTER(p_vend->p_log); 739 740 /* 741 * Copy the MAD over to the sent mad 742 */ 743 memcpy(&ts_mad, p_mad, 256); 744 745 /* 746 * For all sends other than directed route SM MADs, 747 * acquire an address vector for the destination. 748 */ 749 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 750 __osm_ts_conv_osm_addr_to_ts_addr(p_mad_addr, 751 p_mad->mgmt_class == 752 IB_MCLASS_SUBN_LID, &ts_mad); 753 } else { 754 /* is a directed route - we need to construct a permissive address */ 755 /* we do not need port number since it is part of the mad_hndl */ 756 ts_mad.dlid = IB_LID_PERMISSIVE; 757 ts_mad.slid = IB_LID_PERMISSIVE; 758 } 759 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 760 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 761 ts_mad.sqpn = 0; 762 ts_mad.dqpn = 0; 763 } else { 764 ts_mad.sqpn = 1; 765 ts_mad.dqpn = 1; 766 } 767 ts_mad.port = p_bind->port_num; 768 769 /* send it */ 770 ret = write(p_bind->ul_dev_fd, &ts_mad, sizeof(ts_mad)); 771 772 if (ret != sizeof(ts_mad)) { 773 osm_log(p_vend->p_log, OSM_LOG_ERROR, 774 "osm_ts_send_mad: ERR 5026: " 775 "Error sending mad (%d).\n", ret); 776 status = IB_ERROR; 777 goto Exit; 778 } 779 780 status = IB_SUCCESS; 781 782Exit: 783 OSM_LOG_EXIT(p_vend->p_log); 784 return (status); 785} 786 787/********************************************************************** 788Send a MAD through. 789 790What is unclear to me is the need for the setting of all the MAD Wrapper 791fields. Seems like the OSM uses these values during it's processing... 792**********************************************************************/ 793ib_api_status_t 794osm_vendor_send(IN osm_bind_handle_t h_bind, 795 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) 796{ 797 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 798 osm_vendor_t *const p_vend = p_bind->p_vend; 799 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 800 ib_api_status_t status; 801 802 OSM_LOG_ENTER(p_vend->p_log); 803 804 /* 805 * If a response is expected to this MAD, then preallocate 806 * a mad wrapper to contain the wire MAD received in the 807 * response. Allocating a wrapper here allows for easier 808 * failure paths than after we already received the wire mad. 809 */ 810 if (resp_expected == TRUE) { 811 /* we track it in the vendor wrapper */ 812 p_vw->p_resp_madw = 813 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); 814 if (p_vw->p_resp_madw == NULL) { 815 osm_log(p_vend->p_log, OSM_LOG_ERROR, 816 "osm_vendor_send: ERR 5024: " 817 "Unable to allocate MAD wrapper.\n"); 818 status = IB_INSUFFICIENT_RESOURCES; 819 goto Exit; 820 } 821 822 /* put some minimal info on that wrapper */ 823 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind; 824 825 /* we also want to track it in the TID based map */ 826 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *) 827 p_bind, p_madw); 828 if (status != IB_SUCCESS) { 829 osm_log(p_vend->p_log, OSM_LOG_ERROR, 830 "osm_vendor_send: ERR 5025: " 831 "Error inserting request madw by TID (%d).\n", 832 status); 833 } 834 } else 835 p_vw->p_resp_madw = NULL; 836 837 /* do the actual send */ 838 /* HACK: to be replaced by call to RMPP Segmentation */ 839 status = osm_ts_send_mad(p_bind, p_madw); 840 841 /* we do not get an asycn callback so call it ourselves */ 842 /* this will handle all cleanup if neccessary */ 843 __osm_ts_send_callback(p_bind, !resp_expected, p_madw, status); 844 845Exit: 846 OSM_LOG_EXIT(p_vend->p_log); 847 return (status); 848} 849 850/********************************************************************** 851 * the idea here is to change the content of the bind such that it 852 * will hold the local address used for sending directed route by the SMA. 853 **********************************************************************/ 854ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind) 855{ 856 osm_vendor_t *p_vend = ((osm_ts_bind_info_t *) h_bind)->p_vend; 857 858 OSM_LOG_ENTER(p_vend->p_log); 859 860 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 861 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n"); 862 863 OSM_LOG_EXIT(p_vend->p_log); 864 865 return (IB_SUCCESS); 866} 867 868/********************************************************************** 869 **********************************************************************/ 870void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 871{ 872 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 873 osm_vendor_t *p_vend = p_bind->p_vend; 874 VAPI_ret_t status; 875 VAPI_hca_attr_t attr_mod; 876 VAPI_hca_attr_mask_t attr_mask; 877 878 OSM_LOG_ENTER(p_vend->p_log); 879 880 memset(&attr_mod, 0, sizeof(attr_mod)); 881 memset(&attr_mask, 0, sizeof(attr_mask)); 882 883 attr_mod.is_sm = is_sm_val; 884 attr_mask = HCA_ATTR_IS_SM; 885 886 status = 887 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 888 &attr_mask); 889 if (status != VAPI_OK) { 890 osm_log(p_vend->p_log, OSM_LOG_ERROR, 891 "osm_vendor_set_sm: ERR 5027: " 892 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n", 893 is_sm_val, status); 894 } 895 896 OSM_LOG_EXIT(p_vend->p_log); 897} 898 899/********************************************************************** 900 **********************************************************************/ 901void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level) 902{ 903 904} 905