1/* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36#if HAVE_CONFIG_H 37# include <config.h> 38#endif /* HAVE_CONFIG_H */ 39 40#ifdef OSM_VENDOR_INTF_MTL 41 42#include <stdlib.h> 43#include <string.h> 44#include <opensm/osm_helper.h> 45#include <opensm/osm_log.h> 46/* HACK - I do not know how to prevent complib from loading kernel H files */ 47#undef __init 48#include <vendor/osm_vendor_mtl.h> 49#include <vendor/osm_vendor_api.h> 50#include <opensm/osm_subnet.h> 51#include <opensm/osm_opensm.h> 52#include <vendor/osm_vendor_mtl_transaction_mgr.h> 53#include <vendor/osm_mtl_bind.h> 54 55/* 56 Since a race can accure on requests. Meaning - a response is received before 57 the send_callback is called - we will save both the madw_p and the fact 58 whether or not it is a response. A race can occure only on requests that did 59 not fail, and then the madw_p will be put back in the pool before the callback. 60*/ 61uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw) 62{ 63 uint64_t wrid = 0; 64 65 CL_ASSERT(p_madw->p_mad); 66 67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *)); 68 wrid = (wrid << 1) | 69 ib_mad_is_response(p_madw->p_mad); 70 return wrid; 71} 72 73void 74__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid, 75 OUT uint8_t * is_resp, 76 OUT osm_madw_t ** pp_madw) 77{ 78 *is_resp = wrid & 0x0000000000000001; 79 wrid = wrid >> 1; 80 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *)); 81} 82 83/********************************************************************** 84 * IB_MGT to OSM ADDRESS VECTOR 85 **********************************************************************/ 86void 87__osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend, 88 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc, 89 IN uint8_t is_smi, 90 OUT osm_mad_addr_t * p_mad_addr) 91{ 92 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */ 93 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid); 94 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */ 95 p_mad_addr->path_bits = p_rcv_desc->local_path_bits; 96 if (is_smi) { 97 /* SMI */ 98 p_mad_addr->addr_type.smi.source_lid = 99 cl_hton16(p_rcv_desc->remote_lid); 100 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */ 101 } else { 102 /* GSI */ 103 /* seems to me there is a IBMGT bug reversing the QPN ... */ 104 /* Does IBMGT supposed to provide the QPN is network or HOST ? */ 105 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp); 106 107 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 108 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */ 109 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */ 110 /* the full PKey table - than go by the index. */ 111 /* since this does not seem reasonable to me I simply use the default */ 112 /* There is a TAVOR limitation that only one P_KEY is supported per */ 113 /* QP - so QP1 must use IB_DEFAULT_PKEY */ 114 p_mad_addr->addr_type.gsi.pkey_ix = 0; 115 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl; 116 117 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag; 118 /* copy the GRH data if relevant */ 119 if (p_mad_addr->addr_type.gsi.global_route) { 120 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 121 ib_grh_set_ver_class_flow(p_rcv_desc->grh. 122 IP_version, 123 p_rcv_desc->grh. 124 traffic_class, 125 p_rcv_desc->grh. 126 flow_label); 127 p_mad_addr->addr_type.gsi.grh_info.hop_limit = 128 p_rcv_desc->grh.hop_limit; 129 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 130 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 131 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 132 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 133 } 134 } 135} 136 137/********************************************************************** 138 * OSM ADDR VECTOR TO IB_MGT 139 **********************************************************************/ 140void 141__osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr, 142 IN uint8_t is_smi, OUT IB_ud_av_t * p_av) 143{ 144 145 /* For global destination or Multicast address: */ 146 u_int8_t ver; 147 148 memset(p_av, 0, sizeof(IB_ud_av_t)); 149 150 p_av->src_path_bits = p_mad_addr->path_bits; 151 p_av->static_rate = p_mad_addr->static_rate; 152 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid); 153 154 if (is_smi) { 155 p_av->sl = 0; /* Just to note we use 0 here. */ 156 } else { 157 p_av->sl = p_mad_addr->addr_type.gsi.service_level; 158 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route; 159 160 if (p_mad_addr->addr_type.gsi.global_route) { 161 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi. 162 grh_info.ver_class_flow, &ver, 163 &p_av->traffic_class, 164 &p_av->flow_label); 165 p_av->hop_limit = 166 p_mad_addr->addr_type.gsi.grh_info.hop_limit; 167 p_av->sgid_index = 0; /* we always use source GID 0 */ 168 memcpy(&p_av->dgid, 169 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 170 sizeof(ib_net64_t)); 171 172 } 173 } 174} 175 176void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind) 177{ 178 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 179 osm_vendor_t *p_vend = p_bind->p_vend; 180 VAPI_ret_t status; 181 VAPI_hca_attr_t attr_mod; 182 VAPI_hca_attr_mask_t attr_mask; 183 184 OSM_LOG_ENTER(p_vend->p_log); 185 186 memset(&attr_mod, 0, sizeof(attr_mod)); 187 memset(&attr_mask, 0, sizeof(attr_mask)); 188 189 attr_mod.is_sm = FALSE; 190 attr_mask = HCA_ATTR_IS_SM; 191 192 status = 193 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 194 &attr_mask); 195 if (status != VAPI_OK) { 196 osm_log(p_vend->p_log, OSM_LOG_ERROR, 197 "__osm_vendor_clear_sm: ERR 3C21: " 198 "Unable set 'IS_SM' bit in port attributes (%d).\n", 199 status); 200 } 201 202 OSM_LOG_EXIT(p_vend->p_log); 203} 204 205/********************************************************************** 206 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT 207 **********************************************************************/ 208void osm_vendor_construct(IN osm_vendor_t * const p_vend) 209{ 210 memset(p_vend, 0, sizeof(*p_vend)); 211} 212 213/********************************************************************** 214 * DEALOCATE osm_vendor_t 215 **********************************************************************/ 216void osm_vendor_destroy(IN osm_vendor_t * const p_vend) 217{ 218 osm_vendor_mgt_bind_t *vendor_mgt_bind_p; 219 IB_MGT_ret_t mgt_ret; 220 OSM_LOG_ENTER(p_vend->p_log); 221 222 if (p_vend->h_al != NULL) { 223 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al; 224 if (vendor_mgt_bind_p->gsi_init) { 225 226 /* un register the class */ 227 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */ 228 mgt_ret = 229 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p-> 230 gsi_mads_hdl, 231 IB_MCLASS_SUBN_ADM); 232 if (mgt_ret != IB_MGT_OK) { 233 osm_log(p_vend->p_log, OSM_LOG_ERROR, 234 "osm_vendor_destroy: ERR 3C03: " 235 "Fail to unbind the SA class.\n"); 236 } 237 238 /* un bind the handle */ 239 if (IB_MGT_release_handle 240 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) { 241 osm_log(p_vend->p_log, OSM_LOG_ERROR, 242 "osm_vendor_destroy: ERR 3C02: " 243 "Fail to unbind the SA GSI handle.\n"); 244 } 245 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 246 "osm_vendor_destroy: DBG 1002: " 247 "Unbind the GSI handles.\n"); 248 } 249 if (vendor_mgt_bind_p->smi_init) { 250 /* first - clear the IS_SM in the capability mask */ 251 __osm_vendor_clear_sm((osm_bind_handle_t) 252 (vendor_mgt_bind_p->smi_p_bind)); 253 254 /* un register the class */ 255 mgt_ret = 256 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl); 257 if (mgt_ret != IB_MGT_OK) { 258 osm_log(p_vend->p_log, OSM_LOG_ERROR, 259 "osm_vendor_destroy: ERR 3C04: " 260 "Fail to unbind the SM class.\n"); 261 } 262 263 /* un bind the handle */ 264 if (IB_MGT_release_handle 265 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) { 266 osm_log(p_vend->p_log, OSM_LOG_ERROR, 267 "osm_vendor_destroy: ERR 3C05: " 268 "Fail to unbind the SMI handle.\n"); 269 } 270 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 271 "osm_vendor_destroy: DBG 1003: " 272 "Unbind the SMI handles.\n"); 273 274 } 275 } 276 osm_transaction_mgr_destroy(p_vend); 277 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */ 278 OSM_LOG_EXIT(p_vend->p_log); 279} 280 281/********************************************************************** 282DEALLOCATE A POINTER TO osm_vendor_t 283**********************************************************************/ 284void osm_vendor_delete(IN osm_vendor_t ** const pp_vend) 285{ 286 CL_ASSERT(pp_vend); 287 288 osm_vendor_destroy(*pp_vend); 289 free(*pp_vend); 290 *pp_vend = NULL; 291} 292 293/********************************************************************** 294 * This proc actuall binds the handle to the lower level. 295 * 296 * We might have here as a result a casting of our struct to the ib_al_handle_t 297 * 298 * Q: Do we need 2 of those - one for MSI and one for GSI ? 299 * A: Yes! We should be able to do the SA too. So we need a struct! 300 * 301 **********************************************************************/ 302 303ib_api_status_t 304osm_vendor_init(IN osm_vendor_t * const p_vend, 305 IN osm_log_t * const p_log, IN const uint32_t timeout) 306{ 307 osm_vendor_mgt_bind_t *ib_mgt_hdl_p; 308 ib_api_status_t status = IB_SUCCESS; 309 310 OSM_LOG_ENTER(p_log); 311 312 p_vend->p_log = p_log; 313 314 /* 315 * HACK: We need no handle. Assuming the driver is up. 316 */ 317 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) 318 malloc(sizeof(osm_vendor_mgt_bind_t)); 319 if (ib_mgt_hdl_p == NULL) { 320 osm_log(p_vend->p_log, OSM_LOG_ERROR, 321 "osm_vendor_init: ERR 3C06: " 322 "Fail to allocate vendor mgt handle.\n"); 323 goto Exit; 324 } 325 326 ib_mgt_hdl_p->smi_init = FALSE; 327 ib_mgt_hdl_p->gsi_init = FALSE; 328 /* cast it into the ib_al_handle_t h_al */ 329 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p; 330 p_vend->p_transaction_mgr = NULL; 331 osm_transaction_mgr_init(p_vend); 332 /* p_vend->madw_by_tid_map_p = NULL; */ 333 /* __osm_mtl_init_tid_mad_map( p_vend ); */ 334 p_vend->timeout = timeout; 335 336Exit: 337 OSM_LOG_EXIT(p_log); 338 return (status); 339} 340 341/********************************************************************** 342 * Create and Initialize osm_vendor_t Object 343 **********************************************************************/ 344osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log, 345 IN const uint32_t timeout) 346{ 347 ib_api_status_t status; 348 osm_vendor_t *p_vend; 349 350 OSM_LOG_ENTER(p_log); 351 352 CL_ASSERT(p_log); 353 354 p_vend = malloc(sizeof(*p_vend)); 355 if (p_vend != NULL) { 356 memset(p_vend, 0, sizeof(*p_vend)); 357 status = osm_vendor_init(p_vend, p_log, timeout); 358 if (status != IB_SUCCESS) { 359 osm_vendor_delete(&p_vend); 360 } 361 } else { 362 osm_log(p_vend->p_log, OSM_LOG_ERROR, 363 "osm_vendor_new: ERR 3C07: " 364 "Fail to allocate vendor object.\n"); 365 } 366 367 OSM_LOG_EXIT(p_log); 368 return (p_vend); 369} 370 371/********************************************************************** 372 * IB_MGT RCV callback 373 * 374 **********************************************************************/ 375void 376__osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl, 377 IN void *private_ctx_p, 378 IN void *payload_p, 379 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p) 380{ 381 IB_MGT_ret_t status; 382 osm_mtl_bind_info_t *bind_info_p = private_ctx_p; 383 osm_madw_t *req_madw_p = NULL; 384 osm_madw_t *madw_p; 385 osm_vend_wrap_t *p_new_vw; 386 osm_mad_addr_t mad_addr; 387 ib_mad_t *mad_buf_p; 388 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 389 390 OSM_LOG_ENTER(p_log); 391 392 /* if it is a response MAD we mustbe able to get the request */ 393 if (ib_mad_is_response((ib_mad_t *) payload_p)) { 394 /* can we find a matching madw by this payload TID */ 395 status = 396 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend, 397 (ib_mad_t *) payload_p, 398 &req_madw_p); 399 if (status != IB_MGT_OK) { 400 osm_log(p_log, OSM_LOG_ERROR, 401 "__osm_mtl_rcv_callback: ERR 3C08: " 402 "Error obtaining request madw by TID (%d).\n", 403 status); 404 req_madw_p = NULL; 405 } 406 407 if (req_madw_p == NULL) { 408 osm_log(p_log, OSM_LOG_ERROR, 409 "__osm_mtl_rcv_callback: ERR 3C09: " 410 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n", 411 ((ib_mad_t *) payload_p)->method, 412 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id) 413 414 ); 415 goto Exit; 416 } 417 } 418 419 /* do we have a request ??? */ 420 if (req_madw_p == NULL) { 421 422 /* first arrange an address */ 423 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend, 424 rcv_remote_info_p, 425 (((ib_mad_t *) 426 payload_p)-> 427 mgmt_class == 428 IB_MCLASS_SUBN_LID) 429 || (((ib_mad_t *) 430 payload_p)-> 431 mgmt_class == 432 IB_MCLASS_SUBN_DIR), 433 &mad_addr); 434 435 osm_log(p_log, OSM_LOG_ERROR, 436 "__osm_mtl_rcv_callback: : " 437 "Received MAD from QP:%X.\n", 438 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp) 439 ); 440 441 /* if not - get new osm_madw and arrange it. */ 442 /* create the new madw in the pool */ 443 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool, 444 (osm_bind_handle_t) bind_info_p, 445 MAD_BLOCK_SIZE, &mad_addr); 446 if (madw_p == NULL) { 447 osm_log(p_log, OSM_LOG_ERROR, 448 "__osm_mtl_rcv_callback: ERR 3C10: " 449 "Error request for a new madw.\n"); 450 goto Exit; 451 } 452 /* HACK: we cust to avoid the const ??? */ 453 mad_buf_p = (void *)madw_p->p_mad; 454 } else { 455 /* we have the madw defined during the send and stored in the vend_wrap */ 456 /* we need to make sure the wrapper is correctly init there */ 457 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0); 458 madw_p = req_madw_p->vend_wrap.p_resp_madw; 459 460 /* HACK: we do not Support RMPP */ 461 CL_ASSERT(madw_p->h_bind); 462 mad_buf_p = 463 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE, 464 &madw_p->vend_wrap); 465 466 if (mad_buf_p == NULL) { 467 osm_log(p_log, OSM_LOG_ERROR, 468 "__osm_mtl_rcv_callback: ERR 3C11: " 469 "Unable to acquire wire MAD.\n"); 470 471 goto Exit; 472 } 473 474 /* 475 Finally, attach the wire MAD to this wrapper. 476 */ 477 osm_madw_set_mad(madw_p, mad_buf_p); 478 479 /* also we need to handle the size of the mad since we did not init ... */ 480 madw_p->mad_size = MAD_BLOCK_SIZE; 481 } 482 483 /* init some fields of the vendor wrapper */ 484 p_new_vw = osm_madw_get_vend_ptr(madw_p); 485 p_new_vw->h_bind = bind_info_p; 486 p_new_vw->size = MAD_BLOCK_SIZE; 487 p_new_vw->p_resp_madw = NULL; 488 p_new_vw->mad_buf_p = mad_buf_p; 489 490 /* HACK: We do not support RMPP in receiving MADS */ 491 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE); 492 493 /* attach the buffer to the wrapper */ 494 madw_p->p_mad = mad_buf_p; 495 496 /* we can also make sure we marked the size and bind on the returned madw */ 497 madw_p->h_bind = p_new_vw->h_bind; 498 499 /* call the CB */ 500 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context, 501 req_madw_p); 502 503Exit: 504 OSM_LOG_EXIT(p_log); 505} 506 507/********************************************************************** 508 * IB_MGT Send callback : invoked after each send 509 * 510 **********************************************************************/ 511void 512__osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl, 513 IN u_int64_t wrid, 514 IN IB_comp_status_t status, IN void *private_ctx_p) 515{ 516 osm_madw_t *madw_p; 517 osm_mtl_bind_info_t *bind_info_p = 518 (osm_mtl_bind_info_t *) private_ctx_p; 519 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 520 osm_vend_wrap_t *p_vw; 521 uint8_t is_resp; 522 523 OSM_LOG_ENTER(p_log); 524 525 /* obtain the madp from the wrid */ 526 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p); 527 528 osm_log(p_log, OSM_LOG_DEBUG, 529 "__osm_mtl_send_callback: INFO 1008: " 530 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp); 531 532 /* we need to handle requests and responses differently */ 533 if (is_resp) { 534 if (status != IB_COMP_SUCCESS) { 535 osm_log(p_log, OSM_LOG_ERROR, 536 "__osm_mtl_send_callback: ERR 3C12: " 537 "Error Sending Response MADW:%p.\n", madw_p); 538 } else { 539 osm_log(p_log, OSM_LOG_DEBUG, 540 "__osm_mtl_send_callback: DBG 1008: " 541 "Completed Sending Response MADW:%p.\n", 542 madw_p); 543 } 544 545 /* if we are a response - we need to clean it up */ 546 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p); 547 } else { 548 549 /* this call back is invoked on completion of send - error or not */ 550 if (status != IB_COMP_SUCCESS) { 551 552 osm_log(p_log, OSM_LOG_ERROR, 553 "__osm_mtl_send_callback: ERR 3C13: " 554 "Received an Error from IB_MGT Send (%d).\n", 555 status); 556 557 p_vw = osm_madw_get_vend_ptr(madw_p); 558 CL_ASSERT(p_vw); 559 560 /* 561 Return any wrappers to the pool that may have been 562 pre-emptively allocated to handle a receive. 563 */ 564 if (p_vw->p_resp_madw) { 565 osm_mad_pool_put(bind_info_p->p_osm_pool, 566 p_vw->p_resp_madw); 567 p_vw->p_resp_madw = NULL; 568 } 569 570 /* invoke the CB */ 571 (*bind_info_p->send_err_callback) (bind_info_p-> 572 client_context, 573 madw_p); 574 } else { 575 /* successful request send - do nothing - the response will need the 576 out mad */ 577 osm_log(p_log, OSM_LOG_DEBUG, 578 "__osm_mtl_send_callback: DBG 1008: " 579 "Completed Sending Request MADW:%p.\n", madw_p); 580 } 581 } 582 583 OSM_LOG_EXIT(p_log); 584} 585 586/********************************************************************** 587 * BINDs a callback (rcv and send error) for a given class and method 588 * defined by the given: osm_bind_info_t 589 **********************************************************************/ 590osm_bind_handle_t 591osm_vendor_bind(IN osm_vendor_t * const p_vend, 592 IN osm_bind_info_t * const p_user_bind, 593 IN osm_mad_pool_t * const p_mad_pool, 594 IN osm_vend_mad_recv_callback_t mad_recv_callback, 595 IN osm_vend_mad_send_err_callback_t send_err_callback, 596 IN void *context) 597{ 598 ib_net64_t port_guid; 599 osm_mtl_bind_info_t *p_bind = NULL; 600 VAPI_hca_hndl_t hca_hndl; 601 VAPI_hca_id_t hca_id; 602 IB_MGT_mad_type_t mad_type; 603 uint32_t port_num; 604 osm_vendor_mgt_bind_t *ib_mgt_hdl_p; 605 IB_MGT_ret_t mgt_ret; 606 607 OSM_LOG_ENTER(p_vend->p_log); 608 609 CL_ASSERT(p_user_bind); 610 CL_ASSERT(p_mad_pool); 611 CL_ASSERT(mad_recv_callback); 612 CL_ASSERT(send_err_callback); 613 614 /* cast back the AL handle to vendor mgt bind */ 615 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al; 616 617 port_guid = p_user_bind->port_guid; 618 619 osm_log(p_vend->p_log, OSM_LOG_INFO, 620 "osm_vendor_bind: " 621 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid)); 622 623 /* obtain the hca name and port num from the guid */ 624 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 625 "osm_vendor_bind: " 626 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n", 627 port_guid); 628 629 mgt_ret = 630 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl, 631 &hca_id, &port_num); 632 if (mgt_ret != IB_MGT_OK) { 633 osm_log(p_vend->p_log, OSM_LOG_ERROR, 634 "osm_vendor_bind: ERR 3C14: " 635 "Unable to obtain CA and port (%d).\n"); 636 goto Exit; 637 } 638 639 /* create the bind object tracking this binding */ 640 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t)); 641 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t)); 642 if (p_bind == NULL) { 643 osm_log(p_vend->p_log, OSM_LOG_ERROR, 644 "osm_vendor_bind: ERR 3C15: " 645 "Unable to allocate internal bind object.\n"); 646 goto Exit; 647 } 648 649 /* track this bind request info */ 650 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t)); 651 p_bind->port_num = port_num; 652 p_bind->p_vend = p_vend; 653 p_bind->client_context = context; 654 p_bind->rcv_callback = mad_recv_callback; 655 p_bind->send_err_callback = send_err_callback; 656 p_bind->p_osm_pool = p_mad_pool; 657 658 CL_ASSERT(p_bind->port_num); 659 660 /* 661 * Get the proper CLASS 662 */ 663 664 switch (p_user_bind->mad_class) { 665 case IB_MCLASS_SUBN_LID: 666 case IB_MCLASS_SUBN_DIR: 667 mad_type = IB_MGT_SMI; 668 break; 669 670 case IB_MCLASS_SUBN_ADM: 671 default: 672 mad_type = IB_MGT_GSI; 673 break; 674 } 675 676 /* we split here - based on the type of MADS GSI / SMI */ 677 /* HACK: we only support one class registration per SMI/GSI !!! */ 678 if (mad_type == IB_MGT_SMI) { 679 /* 680 * SMI CASE 681 */ 682 683 /* we do not need to bind the handle if already available */ 684 if (ib_mgt_hdl_p->smi_init == FALSE) { 685 686 /* First we have to reg and get the handle for the mad */ 687 osm_log(p_vend->p_log, OSM_LOG_ERROR, 688 "osm_vendor_bind: " 689 "Binding to IB_MGT SMI of %s port %u\n", hca_id, 690 port_num); 691 692 mgt_ret = 693 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI, 694 &(ib_mgt_hdl_p->smi_mads_hdl)); 695 if (IB_MGT_OK != mgt_ret) { 696 free(p_bind); 697 p_bind = NULL; 698 osm_log(p_vend->p_log, OSM_LOG_ERROR, 699 "osm_vendor_bind: ERR 3C16: " 700 "Error obtaining IB_MGT handle to SMI.\n"); 701 goto Exit; 702 } 703 704 /* bind it */ 705 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl); 706 if (IB_MGT_OK != mgt_ret) { 707 free(p_bind); 708 p_bind = NULL; 709 osm_log(p_vend->p_log, OSM_LOG_ERROR, 710 "osm_vendor_bind: ERR 3C17: " 711 "Error binding IB_MGT handle to SM.\n"); 712 goto Exit; 713 } 714 715 ib_mgt_hdl_p->smi_init = TRUE; 716 717 } 718 719 /* attach to this bind info */ 720 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl; 721 ib_mgt_hdl_p->smi_p_bind = p_bind; 722 723 /* now register the callback */ 724 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl, 725 &__osm_mtl_rcv_callback, 726 p_bind, 727 &__osm_mtl_send_callback, 728 p_bind, 729 IB_MGT_RCV_CB_MASK | 730 IB_MGT_SEND_CB_MASK); 731 732 } else { 733 /* 734 * GSI CASE 735 */ 736 737 if (ib_mgt_hdl_p->gsi_init == FALSE) { 738 osm_log(p_vend->p_log, OSM_LOG_ERROR, 739 "osm_vendor_bind: " "Binding to IB_MGT GSI\n"); 740 741 /* First we have to reg and get the handle for the mad */ 742 mgt_ret = 743 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI, 744 &(ib_mgt_hdl_p->gsi_mads_hdl)); 745 if (IB_MGT_OK != mgt_ret) { 746 free(p_bind); 747 p_bind = NULL; 748 osm_log(p_vend->p_log, OSM_LOG_ERROR, 749 "osm_vendor_bind: ERR 3C20: " 750 "Error obtaining IB_MGT handle to GSI.\n"); 751 goto Exit; 752 } 753 754 /* bind it */ 755 mgt_ret = 756 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl, 757 p_user_bind->mad_class); 758 if (IB_MGT_OK != mgt_ret) { 759 free(p_bind); 760 p_bind = NULL; 761 osm_log(p_vend->p_log, OSM_LOG_ERROR, 762 "osm_vendor_bind: ERR 3C22: " 763 "Error binding IB_MGT handle to GSI.\n"); 764 goto Exit; 765 } 766 767 ib_mgt_hdl_p->gsi_init = TRUE; 768 769 /* attach to this bind info */ 770 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl; 771 772 /* now register the callback */ 773 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl, 774 &__osm_mtl_rcv_callback, 775 p_bind, 776 &__osm_mtl_send_callback, 777 p_bind, 778 IB_MGT_RCV_CB_MASK | 779 IB_MGT_SEND_CB_MASK); 780 781 } else { 782 /* we can use the existing handle */ 783 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl; 784 mgt_ret = IB_MGT_OK; 785 } 786 787 } 788 789 if (IB_MGT_OK != mgt_ret) { 790 free(p_bind); 791 p_bind = NULL; 792 osm_log(p_vend->p_log, OSM_LOG_ERROR, 793 "osm_vendor_bind: ERR 3C23: " 794 "Error binding IB_MGT CB (%d).\n", mgt_ret); 795 goto Exit; 796 } 797 798 /* HACK: Do we need to initialize an address vector ???? */ 799 800Exit: 801 OSM_LOG_EXIT(p_vend->p_log); 802 return ((osm_bind_handle_t) p_bind); 803} 804 805/********************************************************************** 806Get a mad from the lower level. 807The osm_vend_wrap_t is a wrapper used to connect the mad to the response. 808**********************************************************************/ 809ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind, 810 IN const uint32_t mad_size, 811 IN osm_vend_wrap_t * const p_vw) 812{ 813 ib_mad_t *mad_p; 814 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 815 osm_vendor_t *p_vend = p_bind->p_vend; 816 817 OSM_LOG_ENTER(p_vend->p_log); 818 819 CL_ASSERT(p_vw); 820 /* HACK: We know we can not send through IB_MGT */ 821 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE); 822 823 /* IB_MGT assumes it is 256 - we must follow */ 824 p_vw->size = MAD_BLOCK_SIZE; 825 826 /* allocate it */ 827 mad_p = (ib_mad_t *) malloc(p_vw->size); 828 if (mad_p == NULL) { 829 osm_log(p_vend->p_log, OSM_LOG_ERROR, 830 "osm_vendor_get: ERR 3C24: " 831 "Error Obtaining MAD buffer.\n"); 832 goto Exit; 833 } 834 835 memset(mad_p, 0, p_vw->size); 836 837 /* track locally */ 838 p_vw->mad_buf_p = mad_p; 839 p_vw->h_bind = h_bind; 840 p_vw->p_resp_madw = NULL; 841 842 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 843 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 844 "osm_vendor_get: " 845 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size); 846 } 847 848Exit: 849 OSM_LOG_EXIT(p_vend->p_log); 850 return (mad_p); 851} 852 853/********************************************************************** 854 * Return a MAD by providing it's wrapper object. 855 **********************************************************************/ 856void 857osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw) 858{ 859 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 860 osm_vendor_t *p_vend = p_bind->p_vend; 861 osm_madw_t *p_madw; 862 863 OSM_LOG_ENTER(p_vend->p_log); 864 865 CL_ASSERT(p_vw); 866 CL_ASSERT(p_vw->mad_buf_p); 867 868 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 869 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 870 "osm_vendor_put: " "Retiring MAD %p.\n", 871 p_vw->mad_buf_p); 872 } 873 874 /* 875 * We moved the removal of the transaction to immediatly after 876 * it was looked up. 877 */ 878 879 /* free the mad but the wrapper is part of the madw object */ 880 free(p_vw->mad_buf_p); 881 p_vw->mad_buf_p = NULL; 882 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap); 883 p_madw->p_mad = NULL; 884 885 OSM_LOG_EXIT(p_vend->p_log); 886} 887 888/********************************************************************** 889Actually Send a MAD 890 891This is for internal use by osm_vendor_send and the transaction mgr 892retry too. 893**********************************************************************/ 894ib_api_status_t 895osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw) 896{ 897 osm_vendor_t *const p_vend = p_bind->p_vend; 898 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 899 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); 900 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); 901 ib_api_status_t status; 902 IB_MGT_ret_t mgt_res; 903 IB_ud_av_t av; 904 uint64_t wrid; 905 uint32_t qpn; 906 907 OSM_LOG_ENTER(p_vend->p_log); 908 909 /* 910 * For all sends other than directed route SM MADs, 911 * acquire an address vector for the destination. 912 */ 913 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 914 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr, 915 p_mad->mgmt_class == 916 IB_MCLASS_SUBN_LID, &av); 917 } else { 918 /* is a directed route - we need to construct a permissive address */ 919 memset(&av, 0, sizeof(av)); 920 /* we do not need port number since it is part of the mad_hndl */ 921 av.dlid = IB_LID_PERMISSIVE; 922 } 923 924 wrid = __osm_set_wrid_by_p_madw(p_madw); 925 926 /* send it */ 927 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 928 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 929 930 /* SMI CASE */ 931 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 932 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 933 "osm_mtl_send_mad: " 934 "av.dlid 0x%X, " 935 "av.static_rate %d, " 936 "av.path_bits %d.\n", 937 cl_ntoh16(av.dlid), av.static_rate, 938 av.src_path_bits); 939 } 940 941 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */ 942 &av, /* address vector */ 943 wrid, /* casting the mad wrapper pointer for err cb */ 944 p_vend->timeout); 945 946 } else { 947 /* GSI CASE - Support Remote QP */ 948 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 949 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 950 "osm_mtl_send_mad: " 951 "av.dlid 0x%X, av.static_rate %d, " 952 "av.path_bits %d, remote qp: 0x%06X \n", 953 av.dlid, 954 av.static_rate, 955 av.src_path_bits, 956 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp) 957 ); 958 } 959 960 /* IBMGT have a bug sending to a QP not 1 - 961 the QPN must be in network order except when it qpn 1 ... */ 962 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp); 963 964 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */ 965 &av, /* address vector */ 966 wrid, /* casting the mad wrapper pointer for err cb */ 967 p_vend->timeout, qpn); 968 } 969 970 if (mgt_res != IB_MGT_OK) { 971 osm_log(p_vend->p_log, OSM_LOG_ERROR, 972 "osm_mtl_send_mad: ERR 3C26: " 973 "Error sending mad (%d).\n", mgt_res); 974 if (p_vw->p_resp_madw) 975 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); 976 status = IB_ERROR; 977 goto Exit; 978 } 979 980 status = IB_SUCCESS; 981 982Exit: 983 OSM_LOG_EXIT(p_vend->p_log); 984 return (status); 985} 986 987/********************************************************************** 988Send a MAD through. 989 990What is unclear to me is the need for the setting of all the MAD Wrapper 991fields. Seems like the OSM uses these values during it's processing... 992**********************************************************************/ 993ib_api_status_t 994osm_vendor_send(IN osm_bind_handle_t h_bind, 995 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) 996{ 997 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind; 998 osm_vendor_t *const p_vend = p_bind->p_vend; 999 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 1000 ib_api_status_t status; 1001 1002 OSM_LOG_ENTER(p_vend->p_log); 1003 1004 /* 1005 * If a response is expected to this MAD, then preallocate 1006 * a mad wrapper to contain the wire MAD received in the 1007 * response. Allocating a wrapper here allows for easier 1008 * failure paths than after we already received the wire mad. 1009 */ 1010 if (resp_expected == TRUE) { 1011 /* we track it in the vendor wrapper */ 1012 p_vw->p_resp_madw = 1013 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); 1014 if (p_vw->p_resp_madw == NULL) { 1015 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1016 "osm_vendor_send: ERR 3C27: " 1017 "Unable to allocate MAD wrapper.\n"); 1018 status = IB_INSUFFICIENT_RESOURCES; 1019 goto Exit; 1020 } 1021 1022 /* put some minimal info on that wrapper */ 1023 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind; 1024 1025 /* we also want to track it in the TID based map */ 1026 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t) 1027 p_bind, p_madw); 1028 if (status != IB_SUCCESS) { 1029 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1030 "osm_vendor_send: ERR 3C25: " 1031 "Error inserting request madw by TID (%d).\n", 1032 status); 1033 } 1034 1035 } else 1036 p_vw->p_resp_madw = NULL; 1037 1038 /* do the actual send */ 1039 status = osm_mtl_send_mad(p_bind, p_madw); 1040 1041Exit: 1042 OSM_LOG_EXIT(p_vend->p_log); 1043 return (status); 1044} 1045 1046/********************************************************************** 1047 * the idea here is to change the content of the bind such that it 1048 * will hold the local address used for sending directed route by the SMA. 1049 **********************************************************************/ 1050ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind) 1051{ 1052 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend; 1053 1054 OSM_LOG_ENTER(p_vend->p_log); 1055 1056 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1057 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n"); 1058 1059 OSM_LOG_EXIT(p_vend->p_log); 1060 1061 return (IB_SUCCESS); 1062} 1063 1064void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 1065{ 1066 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 1067 osm_vendor_t *p_vend = p_bind->p_vend; 1068 VAPI_ret_t status; 1069 VAPI_hca_attr_t attr_mod; 1070 VAPI_hca_attr_mask_t attr_mask; 1071 1072 OSM_LOG_ENTER(p_vend->p_log); 1073 1074 memset(&attr_mod, 0, sizeof(attr_mod)); 1075 memset(&attr_mask, 0, sizeof(attr_mask)); 1076 1077 attr_mod.is_sm = is_sm_val; 1078 attr_mask = HCA_ATTR_IS_SM; 1079 1080 status = 1081 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 1082 &attr_mask); 1083 if (status != VAPI_OK) { 1084 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1085 "osm_vendor_set_sm: ERR 3C28: " 1086 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n", 1087 is_sm_val, status); 1088 } 1089 1090 OSM_LOG_EXIT(p_vend->p_log); 1091} 1092 1093void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level) 1094{ 1095 1096} 1097 1098#endif /* OSM_VENDOR_INTF_TEST */ 1099