1/* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36#if HAVE_CONFIG_H 37# include <config.h> 38#endif /* HAVE_CONFIG_H */ 39 40#ifdef OSM_VENDOR_INTF_MTL 41 42#include <stdlib.h> 43#include <string.h> 44#include <opensm/osm_helper.h> 45#include <opensm/osm_log.h> 46/* HACK - I do not know how to prevent complib from loading kernel H files */ 47#undef __init 48#include <vendor/osm_vendor_mtl.h> 49#include <vendor/osm_vendor_api.h> 50#include <opensm/osm_subnet.h> 51#include <opensm/osm_opensm.h> 52#include <vendor/osm_vendor_mtl_transaction_mgr.h> 53#include <vendor/osm_mtl_bind.h> 54 55/* 56 Since a race can accure on requests. Meaning - a response is received before 57 the send_callback is called - we will save both the madw_p and the fact 58 whether or not it is a response. A race can occure only on requests that did 59 not fail, and then the madw_p will be put back in the pool before the callback. 60*/ 61uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw) 62{ 63 uint64_t wrid = 0; 64 65 CL_ASSERT(p_madw->p_mad); 66 67 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *)); 68 wrid = (wrid << 1) | 69 ib_mad_is_response(p_madw->p_mad) | 70 (p_madw->p_mad->method == IB_MAD_METHOD_TRAP_REPRESS); 71 return wrid; 72} 73 74void 75__osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid, 76 OUT uint8_t * is_resp, 77 OUT osm_madw_t ** pp_madw) 78{ 79 *is_resp = wrid & 0x0000000000000001; 80 wrid = wrid >> 1; 81 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *)); 82} 83 84/********************************************************************** 85 * IB_MGT to OSM ADDRESS VECTOR 86 **********************************************************************/ 87void 88__osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend, 89 IN IB_MGT_mad_rcv_desc_t * p_rcv_desc, 90 IN uint8_t is_smi, 91 OUT osm_mad_addr_t * p_mad_addr) 92{ 93 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */ 94 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid); 95 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */ 96 p_mad_addr->path_bits = p_rcv_desc->local_path_bits; 97 if (is_smi) { 98 /* SMI */ 99 p_mad_addr->addr_type.smi.source_lid = 100 cl_hton16(p_rcv_desc->remote_lid); 101 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */ 102 } else { 103 /* GSI */ 104 /* seems to me there is a IBMGT bug reversing the QPN ... */ 105 /* Does IBMGT supposed to provide the QPN is network or HOST ? */ 106 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp); 107 108 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 109 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */ 110 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */ 111 /* the full PKey table - than go by the index. */ 112 /* since this does not seem reasonable to me I simply use the default */ 113 /* There is a TAVOR limitation that only one P_KEY is supported per */ 114 /* QP - so QP1 must use IB_DEFAULT_PKEY */ 115 p_mad_addr->addr_type.gsi.pkey_ix = 0; 116 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl; 117 118 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag; 119 /* copy the GRH data if relevant */ 120 if (p_mad_addr->addr_type.gsi.global_route) { 121 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 122 ib_grh_set_ver_class_flow(p_rcv_desc->grh. 123 IP_version, 124 p_rcv_desc->grh. 125 traffic_class, 126 p_rcv_desc->grh. 127 flow_label); 128 p_mad_addr->addr_type.gsi.grh_info.hop_limit = 129 p_rcv_desc->grh.hop_limit; 130 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 131 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 132 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 133 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 134 } 135 } 136} 137 138/********************************************************************** 139 * OSM ADDR VECTOR TO IB_MGT 140 **********************************************************************/ 141void 142__osm_mtl_conv_osm_addr_to_ibmgt_addr(IN osm_mad_addr_t * p_mad_addr, 143 IN uint8_t is_smi, OUT IB_ud_av_t * p_av) 144{ 145 146 /* For global destination or Multicast address: */ 147 u_int8_t ver; 148 149 memset(p_av, 0, sizeof(IB_ud_av_t)); 150 151 p_av->src_path_bits = p_mad_addr->path_bits; 152 p_av->static_rate = p_mad_addr->static_rate; 153 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid); 154 155 if (is_smi) { 156 p_av->sl = 0; /* Just to note we use 0 here. */ 157 } else { 158 p_av->sl = p_mad_addr->addr_type.gsi.service_level; 159 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route; 160 161 if (p_mad_addr->addr_type.gsi.global_route) { 162 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi. 163 grh_info.ver_class_flow, &ver, 164 &p_av->traffic_class, 165 &p_av->flow_label); 166 p_av->hop_limit = 167 p_mad_addr->addr_type.gsi.grh_info.hop_limit; 168 p_av->sgid_index = 0; /* we always use source GID 0 */ 169 memcpy(&p_av->dgid, 170 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 171 sizeof(ib_net64_t)); 172 173 } 174 } 175} 176 177/********************************************************************** 178 **********************************************************************/ 179void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind) 180{ 181 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 182 osm_vendor_t *p_vend = p_bind->p_vend; 183 VAPI_ret_t status; 184 VAPI_hca_attr_t attr_mod; 185 VAPI_hca_attr_mask_t attr_mask; 186 187 OSM_LOG_ENTER(p_vend->p_log); 188 189 memset(&attr_mod, 0, sizeof(attr_mod)); 190 memset(&attr_mask, 0, sizeof(attr_mask)); 191 192 attr_mod.is_sm = FALSE; 193 attr_mask = HCA_ATTR_IS_SM; 194 195 status = 196 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 197 &attr_mask); 198 if (status != VAPI_OK) { 199 osm_log(p_vend->p_log, OSM_LOG_ERROR, 200 "__osm_vendor_clear_sm: ERR 3C21: " 201 "Unable set 'IS_SM' bit in port attributes (%d).\n", 202 status); 203 } 204 205 OSM_LOG_EXIT(p_vend->p_log); 206} 207 208/********************************************************************** 209 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT 210 **********************************************************************/ 211void osm_vendor_construct(IN osm_vendor_t * const p_vend) 212{ 213 memset(p_vend, 0, sizeof(*p_vend)); 214} 215 216/********************************************************************** 217 * DEALOCATE osm_vendor_t 218 **********************************************************************/ 219void osm_vendor_destroy(IN osm_vendor_t * const p_vend) 220{ 221 osm_vendor_mgt_bind_t *vendor_mgt_bind_p; 222 IB_MGT_ret_t mgt_ret; 223 OSM_LOG_ENTER(p_vend->p_log); 224 225 if (p_vend->h_al != NULL) { 226 vendor_mgt_bind_p = (osm_vendor_mgt_bind_t *) p_vend->h_al; 227 if (vendor_mgt_bind_p->gsi_init) { 228 229 /* un register the class */ 230 /* HACK WE ASSUME WE ONLY GOT SA CLASS REGISTERD ON GSI !!! */ 231 mgt_ret = 232 IB_MGT_unbind_gsi_class(vendor_mgt_bind_p-> 233 gsi_mads_hdl, 234 IB_MCLASS_SUBN_ADM); 235 if (mgt_ret != IB_MGT_OK) { 236 osm_log(p_vend->p_log, OSM_LOG_ERROR, 237 "osm_vendor_destroy: ERR 3C03: " 238 "Fail to unbind the SA class.\n"); 239 } 240 241 /* un bind the handle */ 242 if (IB_MGT_release_handle 243 (vendor_mgt_bind_p->gsi_mads_hdl) != IB_MGT_OK) { 244 osm_log(p_vend->p_log, OSM_LOG_ERROR, 245 "osm_vendor_destroy: ERR 3C02: " 246 "Fail to unbind the SA GSI handle.\n"); 247 } 248 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 249 "osm_vendor_destroy: DBG 1002: " 250 "Unbind the GSI handles.\n"); 251 } 252 if (vendor_mgt_bind_p->smi_init) { 253 /* first - clear the IS_SM in the capability mask */ 254 __osm_vendor_clear_sm((osm_bind_handle_t) 255 (vendor_mgt_bind_p->smi_p_bind)); 256 257 /* un register the class */ 258 mgt_ret = 259 IB_MGT_unbind_sm(vendor_mgt_bind_p->smi_mads_hdl); 260 if (mgt_ret != IB_MGT_OK) { 261 osm_log(p_vend->p_log, OSM_LOG_ERROR, 262 "osm_vendor_destroy: ERR 3C04: " 263 "Fail to unbind the SM class.\n"); 264 } 265 266 /* un bind the handle */ 267 if (IB_MGT_release_handle 268 (vendor_mgt_bind_p->smi_mads_hdl) != IB_MGT_OK) { 269 osm_log(p_vend->p_log, OSM_LOG_ERROR, 270 "osm_vendor_destroy: ERR 3C05: " 271 "Fail to unbind the SMI handle.\n"); 272 } 273 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 274 "osm_vendor_destroy: DBG 1003: " 275 "Unbind the SMI handles.\n"); 276 277 } 278 } 279 osm_transaction_mgr_destroy(p_vend); 280 /* __osm_mtl_destroy_tid_mad_map( p_vend ); */ 281 OSM_LOG_EXIT(p_vend->p_log); 282} 283 284/********************************************************************** 285DEALLOCATE A POINTER TO osm_vendor_t 286**********************************************************************/ 287void osm_vendor_delete(IN osm_vendor_t ** const pp_vend) 288{ 289 CL_ASSERT(pp_vend); 290 291 osm_vendor_destroy(*pp_vend); 292 free(*pp_vend); 293 *pp_vend = NULL; 294} 295 296/********************************************************************** 297 * This proc actuall binds the handle to the lower level. 298 * 299 * We might have here as a result a casting of our struct to the ib_al_handle_t 300 * 301 * Q: Do we need 2 of those - one for MSI and one for GSI ? 302 * A: Yes! We should be able to do the SA too. So we need a struct! 303 * 304 **********************************************************************/ 305 306ib_api_status_t 307osm_vendor_init(IN osm_vendor_t * const p_vend, 308 IN osm_log_t * const p_log, IN const uint32_t timeout) 309{ 310 osm_vendor_mgt_bind_t *ib_mgt_hdl_p; 311 ib_api_status_t status = IB_SUCCESS; 312 313 OSM_LOG_ENTER(p_log); 314 315 p_vend->p_log = p_log; 316 317 /* 318 * HACK: We need no handle. Assuming the driver is up. 319 */ 320 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) 321 malloc(sizeof(osm_vendor_mgt_bind_t)); 322 if (ib_mgt_hdl_p == NULL) { 323 osm_log(p_vend->p_log, OSM_LOG_ERROR, 324 "osm_vendor_init: ERR 3C06: " 325 "Fail to allocate vendor mgt handle.\n"); 326 goto Exit; 327 } 328 329 ib_mgt_hdl_p->smi_init = FALSE; 330 ib_mgt_hdl_p->gsi_init = FALSE; 331 /* cast it into the ib_al_handle_t h_al */ 332 p_vend->h_al = (ib_al_handle_t) ib_mgt_hdl_p; 333 p_vend->p_transaction_mgr = NULL; 334 osm_transaction_mgr_init(p_vend); 335 /* p_vend->madw_by_tid_map_p = NULL; */ 336 /* __osm_mtl_init_tid_mad_map( p_vend ); */ 337 p_vend->timeout = timeout; 338 339Exit: 340 OSM_LOG_EXIT(p_log); 341 return (status); 342} 343 344/********************************************************************** 345 * Create and Initialize osm_vendor_t Object 346 **********************************************************************/ 347osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log, 348 IN const uint32_t timeout) 349{ 350 ib_api_status_t status; 351 osm_vendor_t *p_vend; 352 353 OSM_LOG_ENTER(p_log); 354 355 CL_ASSERT(p_log); 356 357 p_vend = malloc(sizeof(*p_vend)); 358 if (p_vend != NULL) { 359 memset(p_vend, 0, sizeof(*p_vend)); 360 status = osm_vendor_init(p_vend, p_log, timeout); 361 if (status != IB_SUCCESS) { 362 osm_vendor_delete(&p_vend); 363 } 364 } else { 365 osm_log(p_vend->p_log, OSM_LOG_ERROR, 366 "osm_vendor_new: ERR 3C07: " 367 "Fail to allocate vendor object.\n"); 368 } 369 370 OSM_LOG_EXIT(p_log); 371 return (p_vend); 372} 373 374/********************************************************************** 375 * IB_MGT RCV callback 376 * 377 **********************************************************************/ 378void 379__osm_mtl_rcv_callback(IN IB_MGT_mad_hndl_t mad_hndl, 380 IN void *private_ctx_p, 381 IN void *payload_p, 382 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p) 383{ 384 IB_MGT_ret_t status; 385 osm_mtl_bind_info_t *bind_info_p = private_ctx_p; 386 osm_madw_t *req_madw_p = NULL; 387 osm_madw_t *madw_p; 388 osm_vend_wrap_t *p_new_vw; 389 osm_mad_addr_t mad_addr; 390 ib_mad_t *mad_buf_p; 391 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 392 393 OSM_LOG_ENTER(p_log); 394 395 /* if it is a response MAD we mustbe able to get the request */ 396 if (ib_mad_is_response((ib_mad_t *) payload_p)) { 397 /* can we find a matching madw by this payload TID */ 398 status = 399 osm_transaction_mgr_get_madw_for_tid(bind_info_p->p_vend, 400 (ib_mad_t *) payload_p, 401 &req_madw_p); 402 if (status != IB_MGT_OK) { 403 osm_log(p_log, OSM_LOG_ERROR, 404 "__osm_mtl_rcv_callback: ERR 3C08: " 405 "Error obtaining request madw by TID (%d).\n", 406 status); 407 req_madw_p = NULL; 408 } 409 410 if (req_madw_p == NULL) { 411 osm_log(p_log, OSM_LOG_ERROR, 412 "__osm_mtl_rcv_callback: ERR 3C09: " 413 "Fail to obtain request madw for received MAD.(method=%X attr=%X) Aborting CB.\n", 414 ((ib_mad_t *) payload_p)->method, 415 cl_ntoh16(((ib_mad_t *) payload_p)->attr_id) 416 417 ); 418 goto Exit; 419 } 420 } 421 422 /* do we have a request ??? */ 423 if (req_madw_p == NULL) { 424 425 /* first arrange an address */ 426 __osm_mtl_conv_ibmgt_rcv_desc_to_osm_addr(bind_info_p->p_vend, 427 rcv_remote_info_p, 428 (((ib_mad_t *) 429 payload_p)-> 430 mgmt_class == 431 IB_MCLASS_SUBN_LID) 432 || (((ib_mad_t *) 433 payload_p)-> 434 mgmt_class == 435 IB_MCLASS_SUBN_DIR), 436 &mad_addr); 437 438 osm_log(p_log, OSM_LOG_ERROR, 439 "__osm_mtl_rcv_callback: : " 440 "Received MAD from QP:%X.\n", 441 cl_ntoh32(mad_addr.addr_type.gsi.remote_qp) 442 ); 443 444 /* if not - get new osm_madw and arrange it. */ 445 /* create the new madw in the pool */ 446 madw_p = osm_mad_pool_get(bind_info_p->p_osm_pool, 447 (osm_bind_handle_t) bind_info_p, 448 MAD_BLOCK_SIZE, &mad_addr); 449 if (madw_p == NULL) { 450 osm_log(p_log, OSM_LOG_ERROR, 451 "__osm_mtl_rcv_callback: ERR 3C10: " 452 "Error request for a new madw.\n"); 453 goto Exit; 454 } 455 /* HACK: we cust to avoid the const ??? */ 456 mad_buf_p = (void *)madw_p->p_mad; 457 } else { 458 /* we have the madw defined during the send and stored in the vend_wrap */ 459 /* we need to make sure the wrapper is correctly init there */ 460 CL_ASSERT(req_madw_p->vend_wrap.p_resp_madw != 0); 461 madw_p = req_madw_p->vend_wrap.p_resp_madw; 462 463 /* HACK: we do not Support RMPP */ 464 CL_ASSERT(madw_p->h_bind); 465 mad_buf_p = 466 osm_vendor_get(madw_p->h_bind, MAD_BLOCK_SIZE, 467 &madw_p->vend_wrap); 468 469 if (mad_buf_p == NULL) { 470 osm_log(p_log, OSM_LOG_ERROR, 471 "__osm_mtl_rcv_callback: ERR 3C11: " 472 "Unable to acquire wire MAD.\n"); 473 474 goto Exit; 475 } 476 477 /* 478 Finally, attach the wire MAD to this wrapper. 479 */ 480 osm_madw_set_mad(madw_p, mad_buf_p); 481 482 /* also we need to handle the size of the mad since we did not init ... */ 483 madw_p->mad_size = MAD_BLOCK_SIZE; 484 } 485 486 /* init some fields of the vendor wrapper */ 487 p_new_vw = osm_madw_get_vend_ptr(madw_p); 488 p_new_vw->h_bind = bind_info_p; 489 p_new_vw->size = MAD_BLOCK_SIZE; 490 p_new_vw->p_resp_madw = NULL; 491 p_new_vw->mad_buf_p = mad_buf_p; 492 493 /* HACK: We do not support RMPP in receiving MADS */ 494 memcpy(p_new_vw->mad_buf_p, payload_p, MAD_BLOCK_SIZE); 495 496 /* attach the buffer to the wrapper */ 497 madw_p->p_mad = mad_buf_p; 498 499 /* we can also make sure we marked the size and bind on the returned madw */ 500 madw_p->h_bind = p_new_vw->h_bind; 501 502 /* call the CB */ 503 (*bind_info_p->rcv_callback) (madw_p, bind_info_p->client_context, 504 req_madw_p); 505 506Exit: 507 OSM_LOG_EXIT(p_log); 508} 509 510/********************************************************************** 511 * IB_MGT Send callback : invoked after each send 512 * 513 **********************************************************************/ 514void 515__osm_mtl_send_callback(IN IB_MGT_mad_hndl_t mad_hndl, 516 IN u_int64_t wrid, 517 IN IB_comp_status_t status, IN void *private_ctx_p) 518{ 519 osm_madw_t *madw_p; 520 osm_mtl_bind_info_t *bind_info_p = 521 (osm_mtl_bind_info_t *) private_ctx_p; 522 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 523 osm_vend_wrap_t *p_vw; 524 uint8_t is_resp; 525 526 OSM_LOG_ENTER(p_log); 527 528 /* obtain the madp from the wrid */ 529 __osm_set_p_madw_and_resp_by_wrid(wrid, &is_resp, &madw_p); 530 531 osm_log(p_log, OSM_LOG_DEBUG, 532 "__osm_mtl_send_callback: INFO 1008: " 533 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp); 534 535 /* we need to handle requests and responses differently */ 536 if (is_resp) { 537 if (status != IB_COMP_SUCCESS) { 538 osm_log(p_log, OSM_LOG_ERROR, 539 "__osm_mtl_send_callback: ERR 3C12: " 540 "Error Sending Response MADW:%p.\n", madw_p); 541 } else { 542 osm_log(p_log, OSM_LOG_DEBUG, 543 "__osm_mtl_send_callback: DBG 1008: " 544 "Completed Sending Response MADW:%p.\n", 545 madw_p); 546 } 547 548 /* if we are a response - we need to clean it up */ 549 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p); 550 } else { 551 552 /* this call back is invoked on completion of send - error or not */ 553 if (status != IB_COMP_SUCCESS) { 554 555 osm_log(p_log, OSM_LOG_ERROR, 556 "__osm_mtl_send_callback: ERR 3C13: " 557 "Received an Error from IB_MGT Send (%d).\n", 558 status); 559 560 p_vw = osm_madw_get_vend_ptr(madw_p); 561 CL_ASSERT(p_vw); 562 563 /* 564 Return any wrappers to the pool that may have been 565 pre-emptively allocated to handle a receive. 566 */ 567 if (p_vw->p_resp_madw) { 568 osm_mad_pool_put(bind_info_p->p_osm_pool, 569 p_vw->p_resp_madw); 570 p_vw->p_resp_madw = NULL; 571 } 572 573 /* invoke the CB */ 574 (*bind_info_p->send_err_callback) (bind_info_p-> 575 client_context, 576 madw_p); 577 } else { 578 /* successful request send - do nothing - the response will need the 579 out mad */ 580 osm_log(p_log, OSM_LOG_DEBUG, 581 "__osm_mtl_send_callback: DBG 1008: " 582 "Completed Sending Request MADW:%p.\n", madw_p); 583 } 584 } 585 586 OSM_LOG_EXIT(p_log); 587} 588 589/********************************************************************** 590 * BINDs a callback (rcv and send error) for a given class and method 591 * defined by the given: osm_bind_info_t 592 **********************************************************************/ 593osm_bind_handle_t 594osm_vendor_bind(IN osm_vendor_t * const p_vend, 595 IN osm_bind_info_t * const p_user_bind, 596 IN osm_mad_pool_t * const p_mad_pool, 597 IN osm_vend_mad_recv_callback_t mad_recv_callback, 598 IN osm_vend_mad_send_err_callback_t send_err_callback, 599 IN void *context) 600{ 601 ib_net64_t port_guid; 602 osm_mtl_bind_info_t *p_bind = NULL; 603 VAPI_hca_hndl_t hca_hndl; 604 VAPI_hca_id_t hca_id; 605 IB_MGT_mad_type_t mad_type; 606 uint32_t port_num; 607 osm_vendor_mgt_bind_t *ib_mgt_hdl_p; 608 IB_MGT_ret_t mgt_ret; 609 610 OSM_LOG_ENTER(p_vend->p_log); 611 612 CL_ASSERT(p_user_bind); 613 CL_ASSERT(p_mad_pool); 614 CL_ASSERT(mad_recv_callback); 615 CL_ASSERT(send_err_callback); 616 617 /* cast back the AL handle to vendor mgt bind */ 618 ib_mgt_hdl_p = (osm_vendor_mgt_bind_t *) p_vend->h_al; 619 620 port_guid = p_user_bind->port_guid; 621 622 osm_log(p_vend->p_log, OSM_LOG_INFO, 623 "osm_vendor_bind: " 624 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid)); 625 626 /* obtain the hca name and port num from the guid */ 627 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 628 "osm_vendor_bind: " 629 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n", 630 port_guid); 631 632 mgt_ret = 633 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl, 634 &hca_id, &port_num); 635 if (mgt_ret != IB_MGT_OK) { 636 osm_log(p_vend->p_log, OSM_LOG_ERROR, 637 "osm_vendor_bind: ERR 3C14: " 638 "Unable to obtain CA and port (%d).\n"); 639 goto Exit; 640 } 641 642 /* create the bind object tracking this binding */ 643 p_bind = (osm_mtl_bind_info_t *) malloc(sizeof(osm_mtl_bind_info_t)); 644 memset(p_bind, 0, sizeof(osm_mtl_bind_info_t)); 645 if (p_bind == NULL) { 646 osm_log(p_vend->p_log, OSM_LOG_ERROR, 647 "osm_vendor_bind: ERR 3C15: " 648 "Unable to allocate internal bind object.\n"); 649 goto Exit; 650 } 651 652 /* track this bind request info */ 653 memcpy(p_bind->hca_id, hca_id, sizeof(VAPI_hca_id_t)); 654 p_bind->port_num = port_num; 655 p_bind->p_vend = p_vend; 656 p_bind->client_context = context; 657 p_bind->rcv_callback = mad_recv_callback; 658 p_bind->send_err_callback = send_err_callback; 659 p_bind->p_osm_pool = p_mad_pool; 660 661 CL_ASSERT(p_bind->port_num); 662 663 /* 664 * Get the proper CLASS 665 */ 666 667 switch (p_user_bind->mad_class) { 668 case IB_MCLASS_SUBN_LID: 669 case IB_MCLASS_SUBN_DIR: 670 mad_type = IB_MGT_SMI; 671 break; 672 673 case IB_MCLASS_SUBN_ADM: 674 default: 675 mad_type = IB_MGT_GSI; 676 break; 677 } 678 679 /* we split here - based on the type of MADS GSI / SMI */ 680 /* HACK: we only support one class registration per SMI/GSI !!! */ 681 if (mad_type == IB_MGT_SMI) { 682 /* 683 * SMI CASE 684 */ 685 686 /* we do not need to bind the handle if already available */ 687 if (ib_mgt_hdl_p->smi_init == FALSE) { 688 689 /* First we have to reg and get the handle for the mad */ 690 osm_log(p_vend->p_log, OSM_LOG_ERROR, 691 "osm_vendor_bind: " 692 "Binding to IB_MGT SMI of %s port %u\n", hca_id, 693 port_num); 694 695 mgt_ret = 696 IB_MGT_get_handle(hca_id, port_num, IB_MGT_SMI, 697 &(ib_mgt_hdl_p->smi_mads_hdl)); 698 if (IB_MGT_OK != mgt_ret) { 699 free(p_bind); 700 p_bind = NULL; 701 osm_log(p_vend->p_log, OSM_LOG_ERROR, 702 "osm_vendor_bind: ERR 3C16: " 703 "Error obtaining IB_MGT handle to SMI.\n"); 704 goto Exit; 705 } 706 707 /* bind it */ 708 mgt_ret = IB_MGT_bind_sm(ib_mgt_hdl_p->smi_mads_hdl); 709 if (IB_MGT_OK != mgt_ret) { 710 free(p_bind); 711 p_bind = NULL; 712 osm_log(p_vend->p_log, OSM_LOG_ERROR, 713 "osm_vendor_bind: ERR 3C17: " 714 "Error binding IB_MGT handle to SM.\n"); 715 goto Exit; 716 } 717 718 ib_mgt_hdl_p->smi_init = TRUE; 719 720 } 721 722 /* attach to this bind info */ 723 p_bind->mad_hndl = ib_mgt_hdl_p->smi_mads_hdl; 724 ib_mgt_hdl_p->smi_p_bind = p_bind; 725 726 /* now register the callback */ 727 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl, 728 &__osm_mtl_rcv_callback, 729 p_bind, 730 &__osm_mtl_send_callback, 731 p_bind, 732 IB_MGT_RCV_CB_MASK | 733 IB_MGT_SEND_CB_MASK); 734 735 } else { 736 /* 737 * GSI CASE 738 */ 739 740 if (ib_mgt_hdl_p->gsi_init == FALSE) { 741 osm_log(p_vend->p_log, OSM_LOG_ERROR, 742 "osm_vendor_bind: " "Binding to IB_MGT GSI\n"); 743 744 /* First we have to reg and get the handle for the mad */ 745 mgt_ret = 746 IB_MGT_get_handle(hca_id, port_num, IB_MGT_GSI, 747 &(ib_mgt_hdl_p->gsi_mads_hdl)); 748 if (IB_MGT_OK != mgt_ret) { 749 free(p_bind); 750 p_bind = NULL; 751 osm_log(p_vend->p_log, OSM_LOG_ERROR, 752 "osm_vendor_bind: ERR 3C20: " 753 "Error obtaining IB_MGT handle to GSI.\n"); 754 goto Exit; 755 } 756 757 /* bind it */ 758 mgt_ret = 759 IB_MGT_bind_gsi_class(ib_mgt_hdl_p->gsi_mads_hdl, 760 p_user_bind->mad_class); 761 if (IB_MGT_OK != mgt_ret) { 762 free(p_bind); 763 p_bind = NULL; 764 osm_log(p_vend->p_log, OSM_LOG_ERROR, 765 "osm_vendor_bind: ERR 3C22: " 766 "Error binding IB_MGT handle to GSI.\n"); 767 goto Exit; 768 } 769 770 ib_mgt_hdl_p->gsi_init = TRUE; 771 772 /* attach to this bind info */ 773 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl; 774 775 /* now register the callback */ 776 mgt_ret = IB_MGT_reg_cb(p_bind->mad_hndl, 777 &__osm_mtl_rcv_callback, 778 p_bind, 779 &__osm_mtl_send_callback, 780 p_bind, 781 IB_MGT_RCV_CB_MASK | 782 IB_MGT_SEND_CB_MASK); 783 784 } else { 785 /* we can use the existing handle */ 786 p_bind->mad_hndl = ib_mgt_hdl_p->gsi_mads_hdl; 787 mgt_ret = IB_MGT_OK; 788 } 789 790 } 791 792 if (IB_MGT_OK != mgt_ret) { 793 free(p_bind); 794 p_bind = NULL; 795 osm_log(p_vend->p_log, OSM_LOG_ERROR, 796 "osm_vendor_bind: ERR 3C23: " 797 "Error binding IB_MGT CB (%d).\n", mgt_ret); 798 goto Exit; 799 } 800 801 /* HACK: Do we need to initialize an address vector ???? */ 802 803Exit: 804 OSM_LOG_EXIT(p_vend->p_log); 805 return ((osm_bind_handle_t) p_bind); 806} 807 808/********************************************************************** 809Get a mad from the lower level. 810The osm_vend_wrap_t is a wrapper used to connect the mad to the response. 811**********************************************************************/ 812ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind, 813 IN const uint32_t mad_size, 814 IN osm_vend_wrap_t * const p_vw) 815{ 816 ib_mad_t *mad_p; 817 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 818 osm_vendor_t *p_vend = p_bind->p_vend; 819 820 OSM_LOG_ENTER(p_vend->p_log); 821 822 CL_ASSERT(p_vw); 823 /* HACK: We know we can not send through IB_MGT */ 824 CL_ASSERT(mad_size <= MAD_BLOCK_SIZE); 825 826 /* IB_MGT assumes it is 256 - we must follow */ 827 p_vw->size = MAD_BLOCK_SIZE; 828 829 /* allocate it */ 830 mad_p = (ib_mad_t *) malloc(p_vw->size); 831 if (mad_p == NULL) { 832 osm_log(p_vend->p_log, OSM_LOG_ERROR, 833 "osm_vendor_get: ERR 3C24: " 834 "Error Obtaining MAD buffer.\n"); 835 goto Exit; 836 } 837 838 memset(mad_p, 0, p_vw->size); 839 840 /* track locally */ 841 p_vw->mad_buf_p = mad_p; 842 p_vw->h_bind = h_bind; 843 p_vw->p_resp_madw = NULL; 844 845 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 846 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 847 "osm_vendor_get: " 848 "Acquired MAD %p, size = %u.\n", mad_p, p_vw->size); 849 } 850 851Exit: 852 OSM_LOG_EXIT(p_vend->p_log); 853 return (mad_p); 854} 855 856/********************************************************************** 857 * Return a MAD by providing it's wrapper object. 858 **********************************************************************/ 859void 860osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw) 861{ 862 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 863 osm_vendor_t *p_vend = p_bind->p_vend; 864 osm_madw_t *p_madw; 865 866 OSM_LOG_ENTER(p_vend->p_log); 867 868 CL_ASSERT(p_vw); 869 CL_ASSERT(p_vw->mad_buf_p); 870 871 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 872 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 873 "osm_vendor_put: " "Retiring MAD %p.\n", 874 p_vw->mad_buf_p); 875 } 876 877 /* 878 * We moved the removal of the transaction to immediatly after 879 * it was looked up. 880 */ 881 882 /* free the mad but the wrapper is part of the madw object */ 883 free(p_vw->mad_buf_p); 884 p_vw->mad_buf_p = NULL; 885 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap); 886 p_madw->p_mad = NULL; 887 888 OSM_LOG_EXIT(p_vend->p_log); 889} 890 891/********************************************************************** 892Actually Send a MAD 893 894This is for internal use by osm_vendor_send and the transaction mgr 895retry too. 896**********************************************************************/ 897ib_api_status_t 898osm_mtl_send_mad(IN osm_mtl_bind_info_t * p_bind, IN osm_madw_t * const p_madw) 899{ 900 osm_vendor_t *const p_vend = p_bind->p_vend; 901 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 902 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); 903 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); 904 ib_api_status_t status; 905 IB_MGT_ret_t mgt_res; 906 IB_ud_av_t av; 907 uint64_t wrid; 908 uint32_t qpn; 909 910 OSM_LOG_ENTER(p_vend->p_log); 911 912 /* 913 * For all sends other than directed route SM MADs, 914 * acquire an address vector for the destination. 915 */ 916 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 917 __osm_mtl_conv_osm_addr_to_ibmgt_addr(p_mad_addr, 918 p_mad->mgmt_class == 919 IB_MCLASS_SUBN_LID, &av); 920 } else { 921 /* is a directed route - we need to construct a permissive address */ 922 memset(&av, 0, sizeof(av)); 923 /* we do not need port number since it is part of the mad_hndl */ 924 av.dlid = IB_LID_PERMISSIVE; 925 } 926 927 wrid = __osm_set_wrid_by_p_madw(p_madw); 928 929 /* send it */ 930 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 931 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 932 933 /* SMI CASE */ 934 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 935 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 936 "osm_mtl_send_mad: " 937 "av.dlid 0x%X, " 938 "av.static_rate %d, " 939 "av.path_bits %d.\n", 940 cl_ntoh16(av.dlid), av.static_rate, 941 av.src_path_bits); 942 } 943 944 mgt_res = IB_MGT_send_mad(p_bind->mad_hndl, p_mad, /* actual payload */ 945 &av, /* address vector */ 946 wrid, /* casting the mad wrapper pointer for err cb */ 947 p_vend->timeout); 948 949 } else { 950 /* GSI CASE - Support Remote QP */ 951 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 952 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 953 "osm_mtl_send_mad: " 954 "av.dlid 0x%X, av.static_rate %d, " 955 "av.path_bits %d, remote qp: 0x%06X \n", 956 av.dlid, 957 av.static_rate, 958 av.src_path_bits, 959 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp) 960 ); 961 } 962 963 /* IBMGT have a bug sending to a QP not 1 - 964 the QPN must be in network order except when it qpn 1 ... */ 965 qpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp); 966 967 mgt_res = IB_MGT_send_mad_to_qp(p_bind->mad_hndl, p_mad, /* actual payload */ 968 &av, /* address vector */ 969 wrid, /* casting the mad wrapper pointer for err cb */ 970 p_vend->timeout, qpn); 971 } 972 973 if (mgt_res != IB_MGT_OK) { 974 osm_log(p_vend->p_log, OSM_LOG_ERROR, 975 "osm_mtl_send_mad: ERR 3C26: " 976 "Error sending mad (%d).\n", mgt_res); 977 if (p_vw->p_resp_madw) 978 osm_mad_pool_put(p_bind->p_osm_pool, p_vw->p_resp_madw); 979 status = IB_ERROR; 980 goto Exit; 981 } 982 983 status = IB_SUCCESS; 984 985Exit: 986 OSM_LOG_EXIT(p_vend->p_log); 987 return (status); 988} 989 990/********************************************************************** 991Send a MAD through. 992 993What is unclear to me is the need for the setting of all the MAD Wrapper 994fields. Seems like the OSM uses these values during it's processing... 995**********************************************************************/ 996ib_api_status_t 997osm_vendor_send(IN osm_bind_handle_t h_bind, 998 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) 999{ 1000 osm_mtl_bind_info_t *const p_bind = (osm_mtl_bind_info_t *) h_bind; 1001 osm_vendor_t *const p_vend = p_bind->p_vend; 1002 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 1003 ib_api_status_t status; 1004 1005 OSM_LOG_ENTER(p_vend->p_log); 1006 1007 /* 1008 * If a response is expected to this MAD, then preallocate 1009 * a mad wrapper to contain the wire MAD received in the 1010 * response. Allocating a wrapper here allows for easier 1011 * failure paths than after we already received the wire mad. 1012 */ 1013 if (resp_expected == TRUE) { 1014 /* we track it in the vendor wrapper */ 1015 p_vw->p_resp_madw = 1016 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); 1017 if (p_vw->p_resp_madw == NULL) { 1018 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1019 "osm_vendor_send: ERR 3C27: " 1020 "Unable to allocate MAD wrapper.\n"); 1021 status = IB_INSUFFICIENT_RESOURCES; 1022 goto Exit; 1023 } 1024 1025 /* put some minimal info on that wrapper */ 1026 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind; 1027 1028 /* we also want to track it in the TID based map */ 1029 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t) 1030 p_bind, p_madw); 1031 if (status != IB_SUCCESS) { 1032 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1033 "osm_vendor_send: ERR 3C25: " 1034 "Error inserting request madw by TID (%d).\n", 1035 status); 1036 } 1037 1038 } else 1039 p_vw->p_resp_madw = NULL; 1040 1041 /* do the actual send */ 1042 status = osm_mtl_send_mad(p_bind, p_madw); 1043 1044Exit: 1045 OSM_LOG_EXIT(p_vend->p_log); 1046 return (status); 1047} 1048 1049/********************************************************************** 1050 * the idea here is to change the content of the bind such that it 1051 * will hold the local address used for sending directed route by the SMA. 1052 **********************************************************************/ 1053ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind) 1054{ 1055 osm_vendor_t *p_vend = ((osm_mtl_bind_info_t *) h_bind)->p_vend; 1056 1057 OSM_LOG_ENTER(p_vend->p_log); 1058 1059 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 1060 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n"); 1061 1062 OSM_LOG_EXIT(p_vend->p_log); 1063 1064 return (IB_SUCCESS); 1065} 1066 1067/********************************************************************** 1068 **********************************************************************/ 1069void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 1070{ 1071 osm_mtl_bind_info_t *p_bind = (osm_mtl_bind_info_t *) h_bind; 1072 osm_vendor_t *p_vend = p_bind->p_vend; 1073 VAPI_ret_t status; 1074 VAPI_hca_attr_t attr_mod; 1075 VAPI_hca_attr_mask_t attr_mask; 1076 1077 OSM_LOG_ENTER(p_vend->p_log); 1078 1079 memset(&attr_mod, 0, sizeof(attr_mod)); 1080 memset(&attr_mask, 0, sizeof(attr_mask)); 1081 1082 attr_mod.is_sm = is_sm_val; 1083 attr_mask = HCA_ATTR_IS_SM; 1084 1085 status = 1086 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 1087 &attr_mask); 1088 if (status != VAPI_OK) { 1089 osm_log(p_vend->p_log, OSM_LOG_ERROR, 1090 "osm_vendor_set_sm: ERR 3C28: " 1091 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n", 1092 is_sm_val, status); 1093 } 1094 1095 OSM_LOG_EXIT(p_vend->p_log); 1096} 1097 1098/********************************************************************** 1099 **********************************************************************/ 1100void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level) 1101{ 1102 1103} 1104 1105#endif /* OSM_VENDOR_INTF_TEST */ 1106