1/* 2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * Copyright (c) 2009-2011 ZIH, TU Dresden, Federal Republic of Germany. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 * 35 */ 36 37/* 38 * Abstract: 39 * Implementation of osm_opensm_t. 40 * This object represents the opensm super object. 41 * This object is part of the opensm family of objects. 42 */ 43 44#if HAVE_CONFIG_H 45# include <config.h> 46#endif /* HAVE_CONFIG_H */ 47 48#include <stdio.h> 49#include <stdlib.h> 50#include <string.h> 51#include <complib/cl_dispatcher.h> 52#include <complib/cl_passivelock.h> 53#include <opensm/osm_file_ids.h> 54#define FILE_ID OSM_FILE_OPENSM_C 55#include <vendor/osm_vendor_api.h> 56#include <opensm/osm_version.h> 57#include <opensm/osm_base.h> 58#include <opensm/osm_opensm.h> 59#include <opensm/osm_log.h> 60#include <opensm/osm_subnet.h> 61#include <opensm/osm_sm.h> 62#include <opensm/osm_vl15intf.h> 63#include <opensm/osm_event_plugin.h> 64#include <opensm/osm_congestion_control.h> 65 66struct routing_engine_module { 67 const char *name; 68 int (*setup) (struct osm_routing_engine *, osm_opensm_t *); 69}; 70 71extern int osm_ucast_minhop_setup(struct osm_routing_engine *, osm_opensm_t *); 72extern int osm_ucast_updn_setup(struct osm_routing_engine *, osm_opensm_t *); 73extern int osm_ucast_dnup_setup(struct osm_routing_engine *, osm_opensm_t *); 74extern int osm_ucast_file_setup(struct osm_routing_engine *, osm_opensm_t *); 75extern int osm_ucast_ftree_setup(struct osm_routing_engine *, osm_opensm_t *); 76extern int osm_ucast_lash_setup(struct osm_routing_engine *, osm_opensm_t *); 77extern int osm_ucast_dor_setup(struct osm_routing_engine *, osm_opensm_t *); 78extern int osm_ucast_torus2QoS_setup(struct osm_routing_engine *, osm_opensm_t *); 79extern int osm_ucast_sssp_setup(struct osm_routing_engine *, osm_opensm_t *); 80extern int osm_ucast_dfsssp_setup(struct osm_routing_engine *, osm_opensm_t *); 81 82const static struct routing_engine_module routing_modules[] = { 83 {"minhop", osm_ucast_minhop_setup}, 84 {"updn", osm_ucast_updn_setup}, 85 {"dnup", osm_ucast_dnup_setup}, 86 {"file", osm_ucast_file_setup}, 87 {"ftree", osm_ucast_ftree_setup}, 88 {"lash", osm_ucast_lash_setup}, 89 {"dor", osm_ucast_dor_setup}, 90 {"torus-2QoS", osm_ucast_torus2QoS_setup}, 91 {"dfsssp", osm_ucast_dfsssp_setup}, 92 {"sssp", osm_ucast_sssp_setup}, 93 {NULL, NULL} 94}; 95 96const char *osm_routing_engine_type_str(IN osm_routing_engine_type_t type) 97{ 98 switch (type) { 99 case OSM_ROUTING_ENGINE_TYPE_NONE: 100 return "none"; 101 case OSM_ROUTING_ENGINE_TYPE_MINHOP: 102 return "minhop"; 103 case OSM_ROUTING_ENGINE_TYPE_UPDN: 104 return "updn"; 105 case OSM_ROUTING_ENGINE_TYPE_DNUP: 106 return "dnup"; 107 case OSM_ROUTING_ENGINE_TYPE_FILE: 108 return "file"; 109 case OSM_ROUTING_ENGINE_TYPE_FTREE: 110 return "ftree"; 111 case OSM_ROUTING_ENGINE_TYPE_LASH: 112 return "lash"; 113 case OSM_ROUTING_ENGINE_TYPE_DOR: 114 return "dor"; 115 case OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS: 116 return "torus-2QoS"; 117 case OSM_ROUTING_ENGINE_TYPE_DFSSSP: 118 return "dfsssp"; 119 case OSM_ROUTING_ENGINE_TYPE_SSSP: 120 return "sssp"; 121 default: 122 break; 123 } 124 return "unknown"; 125} 126 127osm_routing_engine_type_t osm_routing_engine_type(IN const char *str) 128{ 129 /* For legacy reasons, consider a NULL pointer and the string 130 * "null" as the minhop routing engine. 131 */ 132 if (!str || !strcasecmp(str, "null") 133 || !strcasecmp(str, "minhop")) 134 return OSM_ROUTING_ENGINE_TYPE_MINHOP; 135 else if (!strcasecmp(str, "none")) 136 return OSM_ROUTING_ENGINE_TYPE_NONE; 137 else if (!strcasecmp(str, "updn")) 138 return OSM_ROUTING_ENGINE_TYPE_UPDN; 139 else if (!strcasecmp(str, "dnup")) 140 return OSM_ROUTING_ENGINE_TYPE_DNUP; 141 else if (!strcasecmp(str, "file")) 142 return OSM_ROUTING_ENGINE_TYPE_FILE; 143 else if (!strcasecmp(str, "ftree")) 144 return OSM_ROUTING_ENGINE_TYPE_FTREE; 145 else if (!strcasecmp(str, "lash")) 146 return OSM_ROUTING_ENGINE_TYPE_LASH; 147 else if (!strcasecmp(str, "dor")) 148 return OSM_ROUTING_ENGINE_TYPE_DOR; 149 else if (!strcasecmp(str, "torus-2QoS")) 150 return OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS; 151 else if (!strcasecmp(str, "sssp")) 152 return OSM_ROUTING_ENGINE_TYPE_SSSP; 153 else if (!strcasecmp(str, "dfsssp")) 154 return OSM_ROUTING_ENGINE_TYPE_DFSSSP; 155 else 156 return OSM_ROUTING_ENGINE_TYPE_UNKNOWN; 157} 158 159static void append_routing_engine(osm_opensm_t *osm, 160 struct osm_routing_engine *routing_engine) 161{ 162 struct osm_routing_engine *r; 163 164 routing_engine->next = NULL; 165 166 if (!osm->routing_engine_list) { 167 osm->routing_engine_list = routing_engine; 168 return; 169 } 170 171 r = osm->routing_engine_list; 172 while (r->next) 173 r = r->next; 174 175 r->next = routing_engine; 176} 177 178static struct osm_routing_engine *setup_routing_engine(osm_opensm_t *osm, 179 const char *name) 180{ 181 struct osm_routing_engine *re; 182 const struct routing_engine_module *m; 183 184 if (!strcmp(name, "no_fallback")) { 185 osm->no_fallback_routing_engine = TRUE; 186 return NULL; 187 } 188 189 for (m = routing_modules; m->name && *m->name; m++) { 190 if (!strcmp(m->name, name)) { 191 re = malloc(sizeof(struct osm_routing_engine)); 192 if (!re) { 193 OSM_LOG(&osm->log, OSM_LOG_VERBOSE, 194 "memory allocation failed\n"); 195 return NULL; 196 } 197 memset(re, 0, sizeof(struct osm_routing_engine)); 198 199 re->name = m->name; 200 re->type = osm_routing_engine_type(m->name); 201 if (m->setup(re, osm)) { 202 OSM_LOG(&osm->log, OSM_LOG_VERBOSE, 203 "setup of routing" 204 " engine \'%s\' failed\n", name); 205 free(re); 206 return NULL; 207 } 208 OSM_LOG(&osm->log, OSM_LOG_DEBUG, 209 "\'%s\' routing engine set up\n", re->name); 210 if (re->type == OSM_ROUTING_ENGINE_TYPE_MINHOP) 211 osm->default_routing_engine = re; 212 return re; 213 } 214 } 215 216 OSM_LOG(&osm->log, OSM_LOG_ERROR, 217 "cannot find or setup routing engine \'%s\'\n", name); 218 return NULL; 219} 220 221static void setup_routing_engines(osm_opensm_t *osm, const char *engine_names) 222{ 223 char *name, *str, *p; 224 struct osm_routing_engine *re; 225 226 if (engine_names && *engine_names) { 227 str = strdup(engine_names); 228 name = strtok_r(str, ", \t\n", &p); 229 while (name && *name) { 230 re = setup_routing_engine(osm, name); 231 if (re) 232 append_routing_engine(osm, re); 233 name = strtok_r(NULL, ", \t\n", &p); 234 } 235 free(str); 236 } 237 if (!osm->default_routing_engine) 238 setup_routing_engine(osm, "minhop"); 239} 240 241void osm_opensm_construct(IN osm_opensm_t * p_osm) 242{ 243 memset(p_osm, 0, sizeof(*p_osm)); 244 p_osm->osm_version = OSM_VERSION; 245 osm_subn_construct(&p_osm->subn); 246 osm_db_construct(&p_osm->db); 247 osm_log_construct(&p_osm->log); 248} 249 250void osm_opensm_construct_finish(IN osm_opensm_t * p_osm) 251{ 252 osm_sm_construct(&p_osm->sm); 253 osm_sa_construct(&p_osm->sa); 254 osm_mad_pool_construct(&p_osm->mad_pool); 255 p_osm->mad_pool_constructed = TRUE; 256 osm_vl15_construct(&p_osm->vl15); 257 p_osm->vl15_constructed = TRUE; 258} 259 260static void destroy_routing_engines(osm_opensm_t *osm) 261{ 262 struct osm_routing_engine *r, *next; 263 264 next = osm->routing_engine_list; 265 while (next) { 266 r = next; 267 next = r->next; 268 if (r->destroy) 269 r->destroy(r->context); 270 free(r); 271 } 272} 273 274static void destroy_plugins(osm_opensm_t *osm) 275{ 276 osm_epi_plugin_t *p; 277 /* remove from the list, and destroy it */ 278 while (!cl_is_qlist_empty(&osm->plugin_list)){ 279 p = (osm_epi_plugin_t *)cl_qlist_remove_head(&osm->plugin_list); 280 /* plugin is responsible for freeing its own resources */ 281 osm_epi_destroy(p); 282 } 283} 284 285void osm_opensm_destroy(IN osm_opensm_t * p_osm) 286{ 287 /* in case of shutdown through exit proc - no ^C */ 288 osm_exit_flag = TRUE; 289 290 /* 291 * First of all, clear the is_sm bit. 292 */ 293 if (p_osm->sm.mad_ctrl.h_bind) 294 osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, FALSE); 295 296#ifdef ENABLE_OSM_PERF_MGR 297 /* Shutdown the PerfMgr */ 298 osm_perfmgr_shutdown(&p_osm->perfmgr); 299#endif /* ENABLE_OSM_PERF_MGR */ 300 301 osm_congestion_control_shutdown(&p_osm->cc); 302 303 /* shut down the SM 304 * - make sure the SM sweeper thread exited 305 * - unbind from QP0 messages 306 */ 307 osm_sm_shutdown(&p_osm->sm); 308 309 /* shut down the SA 310 * - unbind from QP1 messages 311 */ 312 osm_sa_shutdown(&p_osm->sa); 313 314 /* cleanup all messages on VL15 fifo that were not sent yet */ 315 osm_vl15_shutdown(&p_osm->vl15, &p_osm->mad_pool); 316 317 /* shut down the dispatcher - so no new messages cross */ 318 cl_disp_shutdown(&p_osm->disp); 319 if (p_osm->sa_set_disp_initialized) 320 cl_disp_shutdown(&p_osm->sa_set_disp); 321 322 /* dump SA DB */ 323 if ((p_osm->sm.p_subn->sm_state == IB_SMINFO_STATE_MASTER) && 324 p_osm->subn.opt.sa_db_dump) 325 osm_sa_db_file_dump(p_osm); 326 327 /* do the destruction in reverse order as init */ 328 destroy_plugins(p_osm); 329 destroy_routing_engines(p_osm); 330 osm_sa_destroy(&p_osm->sa); 331 osm_sm_destroy(&p_osm->sm); 332#ifdef ENABLE_OSM_PERF_MGR 333 osm_perfmgr_destroy(&p_osm->perfmgr); 334#endif /* ENABLE_OSM_PERF_MGR */ 335 osm_congestion_control_destroy(&p_osm->cc); 336} 337 338void osm_opensm_destroy_finish(IN osm_opensm_t * p_osm) 339{ 340 osm_db_destroy(&p_osm->db); 341 if (p_osm->vl15_constructed && p_osm->mad_pool_constructed) 342 osm_vl15_destroy(&p_osm->vl15, &p_osm->mad_pool); 343 if (p_osm->mad_pool_constructed) 344 osm_mad_pool_destroy(&p_osm->mad_pool); 345 p_osm->vl15_constructed = FALSE; 346 p_osm->mad_pool_constructed = FALSE; 347 osm_vendor_delete(&p_osm->p_vendor); 348 osm_subn_destroy(&p_osm->subn); 349 cl_disp_destroy(&p_osm->disp); 350 if (p_osm->sa_set_disp_initialized) 351 cl_disp_destroy(&p_osm->sa_set_disp); 352#ifdef HAVE_LIBPTHREAD 353 pthread_cond_destroy(&p_osm->stats.cond); 354 pthread_mutex_destroy(&p_osm->stats.mutex); 355#else 356 cl_event_destroy(&p_osm->stats.event); 357#endif 358 if (p_osm->node_name_map) 359 close_node_name_map(p_osm->node_name_map); 360 cl_plock_destroy(&p_osm->lock); 361 362 osm_log_destroy(&p_osm->log); 363} 364 365static void load_plugins(osm_opensm_t *osm, const char *plugin_names) 366{ 367 osm_epi_plugin_t *epi; 368 char *p_names, *name, *p; 369 370 p_names = strdup(plugin_names); 371 name = strtok_r(p_names, ", \t\n", &p); 372 while (name && *name) { 373 epi = osm_epi_construct(osm, name); 374 if (!epi) 375 osm_log_v2(&osm->log, OSM_LOG_ERROR, FILE_ID, 376 "ERR 1000: cannot load plugin \'%s\'\n", 377 name); 378 else 379 cl_qlist_insert_tail(&osm->plugin_list, &epi->list); 380 name = strtok_r(NULL, " \t\n", &p); 381 } 382 free(p_names); 383} 384 385ib_api_status_t osm_opensm_init(IN osm_opensm_t * p_osm, 386 IN const osm_subn_opt_t * p_opt) 387{ 388 ib_api_status_t status; 389 390 /* Can't use log macros here, since we're initializing the log */ 391 osm_opensm_construct(p_osm); 392 393 if (p_opt->daemon) 394 p_osm->log.daemon = 1; 395 396 status = osm_log_init_v2(&p_osm->log, p_opt->force_log_flush, 397 p_opt->log_flags, p_opt->log_file, 398 p_opt->log_max_size, p_opt->accum_log_file); 399 if (status != IB_SUCCESS) 400 return status; 401 p_osm->log.log_prefix = p_opt->log_prefix; 402 403 /* If there is a log level defined - add the OSM_VERSION to it */ 404 osm_log_v2(&p_osm->log, 405 osm_log_get_level(&p_osm->log) & (OSM_LOG_SYS ^ 0xFF), 406 FILE_ID, "%s\n", p_osm->osm_version); 407 /* Write the OSM_VERSION to the SYS_LOG */ 408 osm_log_v2(&p_osm->log, OSM_LOG_SYS, FILE_ID, "%s\n", p_osm->osm_version); /* Format Waived */ 409 410 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "[\n"); /* Format Waived */ 411 412 status = cl_plock_init(&p_osm->lock); 413 if (status != IB_SUCCESS) 414 goto Exit; 415 416#ifdef HAVE_LIBPTHREAD 417 pthread_mutex_init(&p_osm->stats.mutex, NULL); 418 pthread_cond_init(&p_osm->stats.cond, NULL); 419#else 420 status = cl_event_init(&p_osm->stats.event, FALSE); 421 if (status != IB_SUCCESS) 422 goto Exit; 423#endif 424 425 if (p_opt->single_thread) { 426 OSM_LOG(&p_osm->log, OSM_LOG_INFO, 427 "Forcing single threaded dispatcher\n"); 428 status = cl_disp_init(&p_osm->disp, 1, "opensm"); 429 } else { 430 /* 431 * Normal behavior is to initialize the dispatcher with 432 * one thread per CPU, as specified by a thread count of '0'. 433 */ 434 status = cl_disp_init(&p_osm->disp, 0, "opensm"); 435 } 436 if (status != IB_SUCCESS) 437 goto Exit; 438 439 /* Unless OpenSM runs in single threaded mode, we create new single 440 * threaded dispatcher for SA Set and Delete requets. 441 */ 442 p_osm->sa_set_disp_initialized = FALSE; 443 if (!p_opt->single_thread) { 444 status = cl_disp_init(&p_osm->sa_set_disp, 1, "subnadmin_set"); 445 if (status != IB_SUCCESS) 446 goto Exit; 447 p_osm->sa_set_disp_initialized = TRUE; 448 } 449 450 /* the DB is in use by subn so init before */ 451 status = osm_db_init(&p_osm->db, &p_osm->log); 452 if (status != IB_SUCCESS) 453 goto Exit; 454 455 status = osm_subn_init(&p_osm->subn, p_osm, p_opt); 456 if (status != IB_SUCCESS) 457 goto Exit; 458 459 p_osm->p_vendor = 460 osm_vendor_new(&p_osm->log, p_opt->transaction_timeout); 461 if (p_osm->p_vendor == NULL) 462 status = IB_INSUFFICIENT_RESOURCES; 463 464Exit: 465 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */ 466 return status; 467} 468 469ib_api_status_t osm_opensm_init_finish(IN osm_opensm_t * p_osm, 470 IN const osm_subn_opt_t * p_opt) 471{ 472 ib_api_status_t status; 473 474 osm_opensm_construct_finish(p_osm); 475 476 p_osm->subn.sm_port_guid = p_opt->guid; 477 478 status = osm_mad_pool_init(&p_osm->mad_pool); 479 if (status != IB_SUCCESS) 480 goto Exit; 481 482 status = osm_vl15_init(&p_osm->vl15, p_osm->p_vendor, 483 &p_osm->log, &p_osm->stats, &p_osm->subn, 484 p_opt->max_wire_smps, p_opt->max_wire_smps2, 485 p_opt->max_smps_timeout); 486 if (status != IB_SUCCESS) 487 goto Exit; 488 489 status = osm_sm_init(&p_osm->sm, &p_osm->subn, &p_osm->db, 490 p_osm->p_vendor, &p_osm->mad_pool, &p_osm->vl15, 491 &p_osm->log, &p_osm->stats, &p_osm->disp, 492 &p_osm->lock); 493 if (status != IB_SUCCESS) 494 goto Exit; 495 496 status = osm_sa_init(&p_osm->sm, &p_osm->sa, &p_osm->subn, 497 p_osm->p_vendor, &p_osm->mad_pool, &p_osm->log, 498 &p_osm->stats, &p_osm->disp, 499 p_opt->single_thread ? NULL : &p_osm->sa_set_disp, 500 &p_osm->lock); 501 if (status != IB_SUCCESS) 502 goto Exit; 503 504 cl_qlist_init(&p_osm->plugin_list); 505 506 if (p_opt->event_plugin_name) 507 load_plugins(p_osm, p_opt->event_plugin_name); 508 509#ifdef ENABLE_OSM_PERF_MGR 510 status = osm_perfmgr_init(&p_osm->perfmgr, p_osm, p_opt); 511 if (status != IB_SUCCESS) 512 goto Exit; 513#endif /* ENABLE_OSM_PERF_MGR */ 514 515 status = osm_congestion_control_init(&p_osm->cc, 516 p_osm, p_opt); 517 if (status != IB_SUCCESS) 518 goto Exit; 519 520 p_osm->no_fallback_routing_engine = FALSE; 521 522 setup_routing_engines(p_osm, p_opt->routing_engine_names); 523 524 p_osm->routing_engine_used = NULL /* OSM_ROUTING_ENGINE_TYPE_NONE */; 525 526 p_osm->node_name_map = open_node_name_map(p_opt->node_name_map_name); 527 528Exit: 529 OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */ 530 return status; 531} 532 533ib_api_status_t osm_opensm_bind(IN osm_opensm_t * p_osm, IN ib_net64_t guid) 534{ 535 ib_api_status_t status; 536 537 OSM_LOG_ENTER(&p_osm->log); 538 539 status = osm_sm_bind(&p_osm->sm, guid); 540 if (status != IB_SUCCESS) 541 goto Exit; 542 543 status = osm_sa_bind(&p_osm->sa, guid); 544 if (status != IB_SUCCESS) 545 goto Exit; 546 547#ifdef ENABLE_OSM_PERF_MGR 548 status = osm_perfmgr_bind(&p_osm->perfmgr, guid); 549 if (status != IB_SUCCESS) 550 goto Exit; 551#endif /* ENABLE_OSM_PERF_MGR */ 552 553 status = osm_congestion_control_bind(&p_osm->cc, guid); 554 if (status != IB_SUCCESS) 555 goto Exit; 556 557 /* setting IS_SM in capability mask */ 558 OSM_LOG(&p_osm->log, OSM_LOG_INFO, "Setting IS_SM on port 0x%016" PRIx64 "\n", 559 cl_ntoh64(guid)); 560 osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, TRUE); 561 562Exit: 563 OSM_LOG_EXIT(&p_osm->log); 564 return status; 565} 566 567void osm_opensm_report_event(osm_opensm_t *osm, osm_epi_event_id_t event_id, 568 void *event_data) 569{ 570 cl_list_item_t *item; 571 572 for (item = cl_qlist_head(&osm->plugin_list); 573 !osm_exit_flag && item != cl_qlist_end(&osm->plugin_list); 574 item = cl_qlist_next(item)) { 575 osm_epi_plugin_t *p = (osm_epi_plugin_t *)item; 576 if (p->impl->report) 577 p->impl->report(p->plugin_data, event_id, event_data); 578 } 579} 580