/* * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved. * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. * Copyright (c) 2009-2011 ZIH, TU Dresden, Federal Republic of Germany. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ /* * Abstract: * Implementation of osm_opensm_t. * This object represents the opensm super object. * This object is part of the opensm family of objects. */ #if HAVE_CONFIG_H # include #endif /* HAVE_CONFIG_H */ #include #include #include #include #include #include #define FILE_ID OSM_FILE_OPENSM_C #include #include #include #include #include #include #include #include #include #include struct routing_engine_module { const char *name; int (*setup) (struct osm_routing_engine *, osm_opensm_t *); }; extern int osm_ucast_minhop_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_updn_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_dnup_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_file_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_ftree_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_lash_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_dor_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_torus2QoS_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_sssp_setup(struct osm_routing_engine *, osm_opensm_t *); extern int osm_ucast_dfsssp_setup(struct osm_routing_engine *, osm_opensm_t *); const static struct routing_engine_module routing_modules[] = { {"minhop", osm_ucast_minhop_setup}, {"updn", osm_ucast_updn_setup}, {"dnup", osm_ucast_dnup_setup}, {"file", osm_ucast_file_setup}, {"ftree", osm_ucast_ftree_setup}, {"lash", osm_ucast_lash_setup}, {"dor", osm_ucast_dor_setup}, {"torus-2QoS", osm_ucast_torus2QoS_setup}, {"dfsssp", osm_ucast_dfsssp_setup}, {"sssp", osm_ucast_sssp_setup}, {NULL, NULL} }; const char *osm_routing_engine_type_str(IN osm_routing_engine_type_t type) { switch (type) { case OSM_ROUTING_ENGINE_TYPE_NONE: return "none"; case OSM_ROUTING_ENGINE_TYPE_MINHOP: return "minhop"; case OSM_ROUTING_ENGINE_TYPE_UPDN: return "updn"; case OSM_ROUTING_ENGINE_TYPE_DNUP: return "dnup"; case OSM_ROUTING_ENGINE_TYPE_FILE: return "file"; case OSM_ROUTING_ENGINE_TYPE_FTREE: return "ftree"; case OSM_ROUTING_ENGINE_TYPE_LASH: return "lash"; case OSM_ROUTING_ENGINE_TYPE_DOR: return "dor"; case OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS: return "torus-2QoS"; case OSM_ROUTING_ENGINE_TYPE_DFSSSP: return "dfsssp"; case OSM_ROUTING_ENGINE_TYPE_SSSP: return "sssp"; default: break; } return "unknown"; } osm_routing_engine_type_t osm_routing_engine_type(IN const char *str) { /* For legacy reasons, consider a NULL pointer and the string * "null" as the minhop routing engine. */ if (!str || !strcasecmp(str, "null") || !strcasecmp(str, "minhop")) return OSM_ROUTING_ENGINE_TYPE_MINHOP; else if (!strcasecmp(str, "none")) return OSM_ROUTING_ENGINE_TYPE_NONE; else if (!strcasecmp(str, "updn")) return OSM_ROUTING_ENGINE_TYPE_UPDN; else if (!strcasecmp(str, "dnup")) return OSM_ROUTING_ENGINE_TYPE_DNUP; else if (!strcasecmp(str, "file")) return OSM_ROUTING_ENGINE_TYPE_FILE; else if (!strcasecmp(str, "ftree")) return OSM_ROUTING_ENGINE_TYPE_FTREE; else if (!strcasecmp(str, "lash")) return OSM_ROUTING_ENGINE_TYPE_LASH; else if (!strcasecmp(str, "dor")) return OSM_ROUTING_ENGINE_TYPE_DOR; else if (!strcasecmp(str, "torus-2QoS")) return OSM_ROUTING_ENGINE_TYPE_TORUS_2QOS; else if (!strcasecmp(str, "sssp")) return OSM_ROUTING_ENGINE_TYPE_SSSP; else if (!strcasecmp(str, "dfsssp")) return OSM_ROUTING_ENGINE_TYPE_DFSSSP; else return OSM_ROUTING_ENGINE_TYPE_UNKNOWN; } static void append_routing_engine(osm_opensm_t *osm, struct osm_routing_engine *routing_engine) { struct osm_routing_engine *r; routing_engine->next = NULL; if (!osm->routing_engine_list) { osm->routing_engine_list = routing_engine; return; } r = osm->routing_engine_list; while (r->next) r = r->next; r->next = routing_engine; } static struct osm_routing_engine *setup_routing_engine(osm_opensm_t *osm, const char *name) { struct osm_routing_engine *re; const struct routing_engine_module *m; if (!strcmp(name, "no_fallback")) { osm->no_fallback_routing_engine = TRUE; return NULL; } for (m = routing_modules; m->name && *m->name; m++) { if (!strcmp(m->name, name)) { re = malloc(sizeof(struct osm_routing_engine)); if (!re) { OSM_LOG(&osm->log, OSM_LOG_VERBOSE, "memory allocation failed\n"); return NULL; } memset(re, 0, sizeof(struct osm_routing_engine)); re->name = m->name; re->type = osm_routing_engine_type(m->name); if (m->setup(re, osm)) { OSM_LOG(&osm->log, OSM_LOG_VERBOSE, "setup of routing" " engine \'%s\' failed\n", name); free(re); return NULL; } OSM_LOG(&osm->log, OSM_LOG_DEBUG, "\'%s\' routing engine set up\n", re->name); if (re->type == OSM_ROUTING_ENGINE_TYPE_MINHOP) osm->default_routing_engine = re; return re; } } OSM_LOG(&osm->log, OSM_LOG_ERROR, "cannot find or setup routing engine \'%s\'\n", name); return NULL; } static void setup_routing_engines(osm_opensm_t *osm, const char *engine_names) { char *name, *str, *p; struct osm_routing_engine *re; if (engine_names && *engine_names) { str = strdup(engine_names); name = strtok_r(str, ", \t\n", &p); while (name && *name) { re = setup_routing_engine(osm, name); if (re) append_routing_engine(osm, re); name = strtok_r(NULL, ", \t\n", &p); } free(str); } if (!osm->default_routing_engine) setup_routing_engine(osm, "minhop"); } void osm_opensm_construct(IN osm_opensm_t * p_osm) { memset(p_osm, 0, sizeof(*p_osm)); p_osm->osm_version = OSM_VERSION; osm_subn_construct(&p_osm->subn); osm_db_construct(&p_osm->db); osm_log_construct(&p_osm->log); } void osm_opensm_construct_finish(IN osm_opensm_t * p_osm) { osm_sm_construct(&p_osm->sm); osm_sa_construct(&p_osm->sa); osm_mad_pool_construct(&p_osm->mad_pool); p_osm->mad_pool_constructed = TRUE; osm_vl15_construct(&p_osm->vl15); p_osm->vl15_constructed = TRUE; } static void destroy_routing_engines(osm_opensm_t *osm) { struct osm_routing_engine *r, *next; next = osm->routing_engine_list; while (next) { r = next; next = r->next; if (r->destroy) r->destroy(r->context); free(r); } } static void destroy_plugins(osm_opensm_t *osm) { osm_epi_plugin_t *p; /* remove from the list, and destroy it */ while (!cl_is_qlist_empty(&osm->plugin_list)){ p = (osm_epi_plugin_t *)cl_qlist_remove_head(&osm->plugin_list); /* plugin is responsible for freeing its own resources */ osm_epi_destroy(p); } } void osm_opensm_destroy(IN osm_opensm_t * p_osm) { /* in case of shutdown through exit proc - no ^C */ osm_exit_flag = TRUE; /* * First of all, clear the is_sm bit. */ if (p_osm->sm.mad_ctrl.h_bind) osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, FALSE); #ifdef ENABLE_OSM_PERF_MGR /* Shutdown the PerfMgr */ osm_perfmgr_shutdown(&p_osm->perfmgr); #endif /* ENABLE_OSM_PERF_MGR */ osm_congestion_control_shutdown(&p_osm->cc); /* shut down the SM * - make sure the SM sweeper thread exited * - unbind from QP0 messages */ osm_sm_shutdown(&p_osm->sm); /* shut down the SA * - unbind from QP1 messages */ osm_sa_shutdown(&p_osm->sa); /* cleanup all messages on VL15 fifo that were not sent yet */ osm_vl15_shutdown(&p_osm->vl15, &p_osm->mad_pool); /* shut down the dispatcher - so no new messages cross */ cl_disp_shutdown(&p_osm->disp); if (p_osm->sa_set_disp_initialized) cl_disp_shutdown(&p_osm->sa_set_disp); /* dump SA DB */ if ((p_osm->sm.p_subn->sm_state == IB_SMINFO_STATE_MASTER) && p_osm->subn.opt.sa_db_dump) osm_sa_db_file_dump(p_osm); /* do the destruction in reverse order as init */ destroy_plugins(p_osm); destroy_routing_engines(p_osm); osm_sa_destroy(&p_osm->sa); osm_sm_destroy(&p_osm->sm); #ifdef ENABLE_OSM_PERF_MGR osm_perfmgr_destroy(&p_osm->perfmgr); #endif /* ENABLE_OSM_PERF_MGR */ osm_congestion_control_destroy(&p_osm->cc); } void osm_opensm_destroy_finish(IN osm_opensm_t * p_osm) { osm_db_destroy(&p_osm->db); if (p_osm->vl15_constructed && p_osm->mad_pool_constructed) osm_vl15_destroy(&p_osm->vl15, &p_osm->mad_pool); if (p_osm->mad_pool_constructed) osm_mad_pool_destroy(&p_osm->mad_pool); p_osm->vl15_constructed = FALSE; p_osm->mad_pool_constructed = FALSE; osm_vendor_delete(&p_osm->p_vendor); osm_subn_destroy(&p_osm->subn); cl_disp_destroy(&p_osm->disp); if (p_osm->sa_set_disp_initialized) cl_disp_destroy(&p_osm->sa_set_disp); #ifdef HAVE_LIBPTHREAD pthread_cond_destroy(&p_osm->stats.cond); pthread_mutex_destroy(&p_osm->stats.mutex); #else cl_event_destroy(&p_osm->stats.event); #endif if (p_osm->node_name_map) close_node_name_map(p_osm->node_name_map); cl_plock_destroy(&p_osm->lock); osm_log_destroy(&p_osm->log); } static void load_plugins(osm_opensm_t *osm, const char *plugin_names) { osm_epi_plugin_t *epi; char *p_names, *name, *p; p_names = strdup(plugin_names); name = strtok_r(p_names, ", \t\n", &p); while (name && *name) { epi = osm_epi_construct(osm, name); if (!epi) osm_log_v2(&osm->log, OSM_LOG_ERROR, FILE_ID, "ERR 1000: cannot load plugin \'%s\'\n", name); else cl_qlist_insert_tail(&osm->plugin_list, &epi->list); name = strtok_r(NULL, " \t\n", &p); } free(p_names); } ib_api_status_t osm_opensm_init(IN osm_opensm_t * p_osm, IN const osm_subn_opt_t * p_opt) { ib_api_status_t status; /* Can't use log macros here, since we're initializing the log */ osm_opensm_construct(p_osm); if (p_opt->daemon) p_osm->log.daemon = 1; status = osm_log_init_v2(&p_osm->log, p_opt->force_log_flush, p_opt->log_flags, p_opt->log_file, p_opt->log_max_size, p_opt->accum_log_file); if (status != IB_SUCCESS) return status; p_osm->log.log_prefix = p_opt->log_prefix; /* If there is a log level defined - add the OSM_VERSION to it */ osm_log_v2(&p_osm->log, osm_log_get_level(&p_osm->log) & (OSM_LOG_SYS ^ 0xFF), FILE_ID, "%s\n", p_osm->osm_version); /* Write the OSM_VERSION to the SYS_LOG */ osm_log_v2(&p_osm->log, OSM_LOG_SYS, FILE_ID, "%s\n", p_osm->osm_version); /* Format Waived */ OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "[\n"); /* Format Waived */ status = cl_plock_init(&p_osm->lock); if (status != IB_SUCCESS) goto Exit; #ifdef HAVE_LIBPTHREAD pthread_mutex_init(&p_osm->stats.mutex, NULL); pthread_cond_init(&p_osm->stats.cond, NULL); #else status = cl_event_init(&p_osm->stats.event, FALSE); if (status != IB_SUCCESS) goto Exit; #endif if (p_opt->single_thread) { OSM_LOG(&p_osm->log, OSM_LOG_INFO, "Forcing single threaded dispatcher\n"); status = cl_disp_init(&p_osm->disp, 1, "opensm"); } else { /* * Normal behavior is to initialize the dispatcher with * one thread per CPU, as specified by a thread count of '0'. */ status = cl_disp_init(&p_osm->disp, 0, "opensm"); } if (status != IB_SUCCESS) goto Exit; /* Unless OpenSM runs in single threaded mode, we create new single * threaded dispatcher for SA Set and Delete requets. */ p_osm->sa_set_disp_initialized = FALSE; if (!p_opt->single_thread) { status = cl_disp_init(&p_osm->sa_set_disp, 1, "subnadmin_set"); if (status != IB_SUCCESS) goto Exit; p_osm->sa_set_disp_initialized = TRUE; } /* the DB is in use by subn so init before */ status = osm_db_init(&p_osm->db, &p_osm->log); if (status != IB_SUCCESS) goto Exit; status = osm_subn_init(&p_osm->subn, p_osm, p_opt); if (status != IB_SUCCESS) goto Exit; p_osm->p_vendor = osm_vendor_new(&p_osm->log, p_opt->transaction_timeout); if (p_osm->p_vendor == NULL) status = IB_INSUFFICIENT_RESOURCES; Exit: OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */ return status; } ib_api_status_t osm_opensm_init_finish(IN osm_opensm_t * p_osm, IN const osm_subn_opt_t * p_opt) { ib_api_status_t status; osm_opensm_construct_finish(p_osm); p_osm->subn.sm_port_guid = p_opt->guid; status = osm_mad_pool_init(&p_osm->mad_pool); if (status != IB_SUCCESS) goto Exit; status = osm_vl15_init(&p_osm->vl15, p_osm->p_vendor, &p_osm->log, &p_osm->stats, &p_osm->subn, p_opt->max_wire_smps, p_opt->max_wire_smps2, p_opt->max_smps_timeout); if (status != IB_SUCCESS) goto Exit; status = osm_sm_init(&p_osm->sm, &p_osm->subn, &p_osm->db, p_osm->p_vendor, &p_osm->mad_pool, &p_osm->vl15, &p_osm->log, &p_osm->stats, &p_osm->disp, &p_osm->lock); if (status != IB_SUCCESS) goto Exit; status = osm_sa_init(&p_osm->sm, &p_osm->sa, &p_osm->subn, p_osm->p_vendor, &p_osm->mad_pool, &p_osm->log, &p_osm->stats, &p_osm->disp, p_opt->single_thread ? NULL : &p_osm->sa_set_disp, &p_osm->lock); if (status != IB_SUCCESS) goto Exit; cl_qlist_init(&p_osm->plugin_list); if (p_opt->event_plugin_name) load_plugins(p_osm, p_opt->event_plugin_name); #ifdef ENABLE_OSM_PERF_MGR status = osm_perfmgr_init(&p_osm->perfmgr, p_osm, p_opt); if (status != IB_SUCCESS) goto Exit; #endif /* ENABLE_OSM_PERF_MGR */ status = osm_congestion_control_init(&p_osm->cc, p_osm, p_opt); if (status != IB_SUCCESS) goto Exit; p_osm->no_fallback_routing_engine = FALSE; setup_routing_engines(p_osm, p_opt->routing_engine_names); p_osm->routing_engine_used = NULL /* OSM_ROUTING_ENGINE_TYPE_NONE */; p_osm->node_name_map = open_node_name_map(p_opt->node_name_map_name); Exit: OSM_LOG(&p_osm->log, OSM_LOG_FUNCS, "]\n"); /* Format Waived */ return status; } ib_api_status_t osm_opensm_bind(IN osm_opensm_t * p_osm, IN ib_net64_t guid) { ib_api_status_t status; OSM_LOG_ENTER(&p_osm->log); status = osm_sm_bind(&p_osm->sm, guid); if (status != IB_SUCCESS) goto Exit; status = osm_sa_bind(&p_osm->sa, guid); if (status != IB_SUCCESS) goto Exit; #ifdef ENABLE_OSM_PERF_MGR status = osm_perfmgr_bind(&p_osm->perfmgr, guid); if (status != IB_SUCCESS) goto Exit; #endif /* ENABLE_OSM_PERF_MGR */ status = osm_congestion_control_bind(&p_osm->cc, guid); if (status != IB_SUCCESS) goto Exit; /* setting IS_SM in capability mask */ OSM_LOG(&p_osm->log, OSM_LOG_INFO, "Setting IS_SM on port 0x%016" PRIx64 "\n", cl_ntoh64(guid)); osm_vendor_set_sm(p_osm->sm.mad_ctrl.h_bind, TRUE); Exit: OSM_LOG_EXIT(&p_osm->log); return status; } void osm_opensm_report_event(osm_opensm_t *osm, osm_epi_event_id_t event_id, void *event_data) { cl_list_item_t *item; for (item = cl_qlist_head(&osm->plugin_list); !osm_exit_flag && item != cl_qlist_end(&osm->plugin_list); item = cl_qlist_next(item)) { osm_epi_plugin_t *p = (osm_epi_plugin_t *)item; if (p->impl->report) p->impl->report(p->plugin_data, event_id, event_data); } }