1219820Sjeff/* 2219820Sjeff * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved. 3219820Sjeff * 4219820Sjeff * This software is available to you under a choice of one of two 5219820Sjeff * licenses. You may choose to be licensed under the terms of the GNU 6219820Sjeff * General Public License (GPL) Version 2, available from the file 7219820Sjeff * COPYING in the main directory of this source tree, or the 8219820Sjeff * OpenIB.org BSD license below: 9219820Sjeff * 10219820Sjeff * Redistribution and use in source and binary forms, with or 11219820Sjeff * without modification, are permitted provided that the following 12219820Sjeff * conditions are met: 13219820Sjeff * 14219820Sjeff * - Redistributions of source code must retain the above 15219820Sjeff * copyright notice, this list of conditions and the following 16219820Sjeff * disclaimer. 17219820Sjeff * 18219820Sjeff * - Redistributions in binary form must reproduce the above 19219820Sjeff * copyright notice, this list of conditions and the following 20219820Sjeff * disclaimer in the documentation and/or other materials 21219820Sjeff * provided with the distribution. 22219820Sjeff * 23219820Sjeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24219820Sjeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25219820Sjeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26219820Sjeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27219820Sjeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28219820Sjeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29219820Sjeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30219820Sjeff * SOFTWARE. 31219820Sjeff * 32219820Sjeff */ 33219820Sjeff 34219820Sjeff/* 35219820Sjeff * Abstract: 36219820Sjeff * Implementation of OpenSM QoS infrastructure primitives 37219820Sjeff */ 38219820Sjeff 39219820Sjeff#if HAVE_CONFIG_H 40219820Sjeff# include <config.h> 41219820Sjeff#endif /* HAVE_CONFIG_H */ 42219820Sjeff 43219820Sjeff#include <stdlib.h> 44219820Sjeff#include <string.h> 45219820Sjeff 46219820Sjeff#include <iba/ib_types.h> 47219820Sjeff#include <complib/cl_qmap.h> 48219820Sjeff#include <complib/cl_debug.h> 49219820Sjeff#include <opensm/osm_opensm.h> 50219820Sjeff#include <opensm/osm_subnet.h> 51219820Sjeff#include <opensm/osm_qos_policy.h> 52219820Sjeff 53219820Sjeffstruct qos_config { 54219820Sjeff uint8_t max_vls; 55219820Sjeff uint8_t vl_high_limit; 56219820Sjeff ib_vl_arb_table_t vlarb_high[2]; 57219820Sjeff ib_vl_arb_table_t vlarb_low[2]; 58219820Sjeff ib_slvl_table_t sl2vl; 59219820Sjeff}; 60219820Sjeff 61219820Sjeffstatic void qos_build_config(struct qos_config *cfg, 62219820Sjeff osm_qos_options_t * opt, osm_qos_options_t * dflt); 63219820Sjeff 64219820Sjeff/* 65219820Sjeff * QoS primitives 66219820Sjeff */ 67219820Sjeffstatic ib_api_status_t vlarb_update_table_block(osm_sm_t * sm, 68219820Sjeff osm_physp_t * p, 69219820Sjeff uint8_t port_num, 70219820Sjeff unsigned force_update, 71219820Sjeff const ib_vl_arb_table_t * 72219820Sjeff table_block, 73219820Sjeff unsigned block_length, 74219820Sjeff unsigned block_num) 75219820Sjeff{ 76219820Sjeff ib_vl_arb_table_t block; 77219820Sjeff osm_madw_context_t context; 78219820Sjeff uint32_t attr_mod; 79219820Sjeff unsigned vl_mask, i; 80219820Sjeff 81219820Sjeff vl_mask = (1 << (ib_port_info_get_op_vls(&p->port_info) - 1)) - 1; 82219820Sjeff 83219820Sjeff memset(&block, 0, sizeof(block)); 84219820Sjeff memcpy(&block, table_block, block_length * sizeof(block.vl_entry[0])); 85219820Sjeff for (i = 0; i < block_length; i++) 86219820Sjeff block.vl_entry[i].vl &= vl_mask; 87219820Sjeff 88219820Sjeff if (!force_update && 89219820Sjeff !memcmp(&p->vl_arb[block_num], &block, 90219820Sjeff block_length * sizeof(block.vl_entry[0]))) 91219820Sjeff return IB_SUCCESS; 92219820Sjeff 93219820Sjeff context.vla_context.node_guid = 94219820Sjeff osm_node_get_node_guid(osm_physp_get_node_ptr(p)); 95219820Sjeff context.vla_context.port_guid = osm_physp_get_port_guid(p); 96219820Sjeff context.vla_context.set_method = TRUE; 97219820Sjeff attr_mod = ((block_num + 1) << 16) | port_num; 98219820Sjeff 99219820Sjeff return osm_req_set(sm, osm_physp_get_dr_path_ptr(p), 100219820Sjeff (uint8_t *) & block, sizeof(block), 101219820Sjeff IB_MAD_ATTR_VL_ARBITRATION, 102219820Sjeff cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); 103219820Sjeff} 104219820Sjeff 105219820Sjeffstatic ib_api_status_t vlarb_update(osm_sm_t * sm, 106219820Sjeff osm_physp_t * p, uint8_t port_num, 107219820Sjeff unsigned force_update, 108219820Sjeff const struct qos_config *qcfg) 109219820Sjeff{ 110219820Sjeff ib_api_status_t status = IB_SUCCESS; 111219820Sjeff ib_port_info_t *p_pi = &p->port_info; 112219820Sjeff unsigned len; 113219820Sjeff 114219820Sjeff if (p_pi->vl_arb_low_cap > 0) { 115219820Sjeff len = p_pi->vl_arb_low_cap < IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK ? 116219820Sjeff p_pi->vl_arb_low_cap : IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 117219820Sjeff if ((status = vlarb_update_table_block(sm, p, port_num, 118219820Sjeff force_update, 119219820Sjeff &qcfg->vlarb_low[0], 120219820Sjeff len, 0)) != IB_SUCCESS) 121219820Sjeff return status; 122219820Sjeff } 123219820Sjeff if (p_pi->vl_arb_low_cap > IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK) { 124219820Sjeff len = p_pi->vl_arb_low_cap % IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 125219820Sjeff if ((status = vlarb_update_table_block(sm, p, port_num, 126219820Sjeff force_update, 127219820Sjeff &qcfg->vlarb_low[1], 128219820Sjeff len, 1)) != IB_SUCCESS) 129219820Sjeff return status; 130219820Sjeff } 131219820Sjeff if (p_pi->vl_arb_high_cap > 0) { 132219820Sjeff len = p_pi->vl_arb_high_cap < IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK ? 133219820Sjeff p_pi->vl_arb_high_cap : IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 134219820Sjeff if ((status = vlarb_update_table_block(sm, p, port_num, 135219820Sjeff force_update, 136219820Sjeff &qcfg->vlarb_high[0], 137219820Sjeff len, 2)) != IB_SUCCESS) 138219820Sjeff return status; 139219820Sjeff } 140219820Sjeff if (p_pi->vl_arb_high_cap > IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK) { 141219820Sjeff len = p_pi->vl_arb_high_cap % IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 142219820Sjeff if ((status = vlarb_update_table_block(sm, p, port_num, 143219820Sjeff force_update, 144219820Sjeff &qcfg->vlarb_high[1], 145219820Sjeff len, 3)) != IB_SUCCESS) 146219820Sjeff return status; 147219820Sjeff } 148219820Sjeff 149219820Sjeff return status; 150219820Sjeff} 151219820Sjeff 152219820Sjeffstatic ib_api_status_t sl2vl_update_table(osm_sm_t * sm, 153219820Sjeff osm_physp_t * p, uint8_t in_port, 154219820Sjeff uint8_t out_port, 155219820Sjeff unsigned force_update, 156219820Sjeff const ib_slvl_table_t * sl2vl_table) 157219820Sjeff{ 158219820Sjeff osm_madw_context_t context; 159219820Sjeff ib_slvl_table_t tbl, *p_tbl; 160219820Sjeff osm_node_t *p_node = osm_physp_get_node_ptr(p); 161219820Sjeff uint32_t attr_mod; 162219820Sjeff unsigned vl_mask; 163219820Sjeff uint8_t vl1, vl2; 164219820Sjeff int i; 165219820Sjeff 166219820Sjeff vl_mask = (1 << (ib_port_info_get_op_vls(&p->port_info) - 1)) - 1; 167219820Sjeff 168219820Sjeff for (i = 0; i < IB_MAX_NUM_VLS / 2; i++) { 169219820Sjeff vl1 = sl2vl_table->raw_vl_by_sl[i] >> 4; 170219820Sjeff vl2 = sl2vl_table->raw_vl_by_sl[i] & 0xf; 171219820Sjeff if (vl1 != 15) 172219820Sjeff vl1 &= vl_mask; 173219820Sjeff if (vl2 != 15) 174219820Sjeff vl2 &= vl_mask; 175219820Sjeff tbl.raw_vl_by_sl[i] = (vl1 << 4) | vl2; 176219820Sjeff } 177219820Sjeff 178219820Sjeff if (!force_update && (p_tbl = osm_physp_get_slvl_tbl(p, in_port)) && 179219820Sjeff !memcmp(p_tbl, &tbl, sizeof(tbl))) 180219820Sjeff return IB_SUCCESS; 181219820Sjeff 182219820Sjeff context.slvl_context.node_guid = osm_node_get_node_guid(p_node); 183219820Sjeff context.slvl_context.port_guid = osm_physp_get_port_guid(p); 184219820Sjeff context.slvl_context.set_method = TRUE; 185219820Sjeff attr_mod = in_port << 8 | out_port; 186219820Sjeff return osm_req_set(sm, osm_physp_get_dr_path_ptr(p), 187219820Sjeff (uint8_t *) & tbl, sizeof(tbl), 188219820Sjeff IB_MAD_ATTR_SLVL_TABLE, 189219820Sjeff cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); 190219820Sjeff} 191219820Sjeff 192219820Sjeffstatic ib_api_status_t sl2vl_update(osm_sm_t * sm, osm_port_t * p_port, 193219820Sjeff osm_physp_t * p, uint8_t port_num, 194219820Sjeff unsigned force_update, 195219820Sjeff const struct qos_config *qcfg) 196219820Sjeff{ 197219820Sjeff ib_api_status_t status; 198219820Sjeff uint8_t i, num_ports; 199219820Sjeff osm_physp_t *p_physp; 200219820Sjeff 201219820Sjeff if (osm_node_get_type(osm_physp_get_node_ptr(p)) == IB_NODE_TYPE_SWITCH) { 202219820Sjeff if (ib_port_info_get_vl_cap(&p->port_info) == 1) { 203219820Sjeff /* Check port 0's capability mask */ 204219820Sjeff p_physp = p_port->p_physp; 205219820Sjeff if (! 206219820Sjeff (p_physp->port_info. 207219820Sjeff capability_mask & IB_PORT_CAP_HAS_SL_MAP)) 208219820Sjeff return IB_SUCCESS; 209219820Sjeff } 210219820Sjeff num_ports = osm_node_get_num_physp(osm_physp_get_node_ptr(p)); 211219820Sjeff } else { 212219820Sjeff if (!(p->port_info.capability_mask & IB_PORT_CAP_HAS_SL_MAP)) 213219820Sjeff return IB_SUCCESS; 214219820Sjeff num_ports = 1; 215219820Sjeff } 216219820Sjeff 217219820Sjeff for (i = 0; i < num_ports; i++) { 218219820Sjeff status = 219219820Sjeff sl2vl_update_table(sm, p, i, port_num, 220219820Sjeff force_update, &qcfg->sl2vl); 221219820Sjeff if (status != IB_SUCCESS) 222219820Sjeff return status; 223219820Sjeff } 224219820Sjeff 225219820Sjeff return IB_SUCCESS; 226219820Sjeff} 227219820Sjeff 228219820Sjeffstatic ib_api_status_t qos_physp_setup(osm_log_t * p_log, osm_sm_t * sm, 229219820Sjeff osm_port_t * p_port, osm_physp_t * p, 230219820Sjeff uint8_t port_num, 231219820Sjeff unsigned force_update, 232219820Sjeff const struct qos_config *qcfg) 233219820Sjeff{ 234219820Sjeff ib_api_status_t status; 235219820Sjeff 236219820Sjeff /* OpVLs should be ok at this moment - just use it */ 237219820Sjeff 238219820Sjeff /* setup VL high limit on the physp later to be updated by link mgr */ 239219820Sjeff p->vl_high_limit = qcfg->vl_high_limit; 240219820Sjeff 241219820Sjeff /* setup VLArbitration */ 242219820Sjeff status = vlarb_update(sm, p, port_num, force_update, qcfg); 243219820Sjeff if (status != IB_SUCCESS) { 244219820Sjeff OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6202 : " 245219820Sjeff "failed to update VLArbitration tables " 246219820Sjeff "for port %" PRIx64 " #%d\n", 247219820Sjeff cl_ntoh64(p->port_guid), port_num); 248219820Sjeff return status; 249219820Sjeff } 250219820Sjeff 251219820Sjeff /* setup SL2VL tables */ 252219820Sjeff status = sl2vl_update(sm, p_port, p, port_num, force_update, qcfg); 253219820Sjeff if (status != IB_SUCCESS) { 254219820Sjeff OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6203 : " 255219820Sjeff "failed to update SL2VLMapping tables " 256219820Sjeff "for port %" PRIx64 " #%d\n", 257219820Sjeff cl_ntoh64(p->port_guid), port_num); 258219820Sjeff return status; 259219820Sjeff } 260219820Sjeff 261219820Sjeff return IB_SUCCESS; 262219820Sjeff} 263219820Sjeff 264219820Sjeffosm_signal_t osm_qos_setup(osm_opensm_t * p_osm) 265219820Sjeff{ 266219820Sjeff struct qos_config ca_config, sw0_config, swe_config, rtr_config; 267219820Sjeff struct qos_config *cfg; 268219820Sjeff cl_qmap_t *p_tbl; 269219820Sjeff cl_map_item_t *p_next; 270219820Sjeff osm_port_t *p_port; 271219820Sjeff uint32_t num_physp; 272219820Sjeff osm_physp_t *p_physp; 273219820Sjeff osm_node_t *p_node; 274219820Sjeff ib_api_status_t status; 275219820Sjeff unsigned force_update; 276219820Sjeff uint8_t i; 277219820Sjeff 278219820Sjeff if (!p_osm->subn.opt.qos) 279219820Sjeff return OSM_SIGNAL_DONE; 280219820Sjeff 281219820Sjeff OSM_LOG_ENTER(&p_osm->log); 282219820Sjeff 283219820Sjeff qos_build_config(&ca_config, &p_osm->subn.opt.qos_ca_options, 284219820Sjeff &p_osm->subn.opt.qos_options); 285219820Sjeff qos_build_config(&sw0_config, &p_osm->subn.opt.qos_sw0_options, 286219820Sjeff &p_osm->subn.opt.qos_options); 287219820Sjeff qos_build_config(&swe_config, &p_osm->subn.opt.qos_swe_options, 288219820Sjeff &p_osm->subn.opt.qos_options); 289219820Sjeff qos_build_config(&rtr_config, &p_osm->subn.opt.qos_rtr_options, 290219820Sjeff &p_osm->subn.opt.qos_options); 291219820Sjeff 292219820Sjeff cl_plock_excl_acquire(&p_osm->lock); 293219820Sjeff 294219820Sjeff /* read QoS policy config file */ 295219820Sjeff osm_qos_parse_policy_file(&p_osm->subn); 296219820Sjeff 297219820Sjeff p_tbl = &p_osm->subn.port_guid_tbl; 298219820Sjeff p_next = cl_qmap_head(p_tbl); 299219820Sjeff while (p_next != cl_qmap_end(p_tbl)) { 300219820Sjeff p_port = (osm_port_t *) p_next; 301219820Sjeff p_next = cl_qmap_next(p_next); 302219820Sjeff 303219820Sjeff p_node = p_port->p_node; 304219820Sjeff if (p_node->sw) { 305219820Sjeff num_physp = osm_node_get_num_physp(p_node); 306219820Sjeff for (i = 1; i < num_physp; i++) { 307219820Sjeff p_physp = osm_node_get_physp_ptr(p_node, i); 308219820Sjeff if (!p_physp) 309219820Sjeff continue; 310219820Sjeff force_update = p_physp->need_update || 311219820Sjeff p_osm->subn.need_update; 312219820Sjeff status = 313219820Sjeff qos_physp_setup(&p_osm->log, &p_osm->sm, 314219820Sjeff p_port, p_physp, i, 315219820Sjeff force_update, &swe_config); 316219820Sjeff } 317219820Sjeff /* skip base port 0 */ 318219820Sjeff if (!ib_switch_info_is_enhanced_port0 319219820Sjeff (&p_node->sw->switch_info)) 320219820Sjeff continue; 321219820Sjeff 322219820Sjeff cfg = &sw0_config; 323219820Sjeff } else if (osm_node_get_type(p_node) == IB_NODE_TYPE_ROUTER) 324219820Sjeff cfg = &rtr_config; 325219820Sjeff else 326219820Sjeff cfg = &ca_config; 327219820Sjeff 328219820Sjeff p_physp = p_port->p_physp; 329219820Sjeff if (!p_physp) 330219820Sjeff continue; 331219820Sjeff 332219820Sjeff force_update = p_physp->need_update || p_osm->subn.need_update; 333219820Sjeff status = qos_physp_setup(&p_osm->log, &p_osm->sm, 334219820Sjeff p_port, p_physp, 0, force_update, cfg); 335219820Sjeff } 336219820Sjeff 337219820Sjeff cl_plock_release(&p_osm->lock); 338219820Sjeff OSM_LOG_EXIT(&p_osm->log); 339219820Sjeff 340219820Sjeff return OSM_SIGNAL_DONE; 341219820Sjeff} 342219820Sjeff 343219820Sjeff/* 344219820Sjeff * QoS config stuff 345219820Sjeff */ 346219820Sjeffstatic int parse_one_unsigned(char *str, char delim, unsigned *val) 347219820Sjeff{ 348219820Sjeff char *end; 349219820Sjeff *val = strtoul(str, &end, 0); 350219820Sjeff if (*end) 351219820Sjeff end++; 352219820Sjeff return (int)(end - str); 353219820Sjeff} 354219820Sjeff 355219820Sjeffstatic int parse_vlarb_entry(char *str, ib_vl_arb_element_t * e) 356219820Sjeff{ 357219820Sjeff unsigned val; 358219820Sjeff char *p = str; 359219820Sjeff p += parse_one_unsigned(p, ':', &val); 360219820Sjeff e->vl = val % 15; 361219820Sjeff p += parse_one_unsigned(p, ',', &val); 362219820Sjeff e->weight = (uint8_t) val; 363219820Sjeff return (int)(p - str); 364219820Sjeff} 365219820Sjeff 366219820Sjeffstatic int parse_sl2vl_entry(char *str, uint8_t * raw) 367219820Sjeff{ 368219820Sjeff unsigned val1, val2; 369219820Sjeff char *p = str; 370219820Sjeff p += parse_one_unsigned(p, ',', &val1); 371219820Sjeff p += parse_one_unsigned(p, ',', &val2); 372219820Sjeff *raw = (val1 << 4) | (val2 & 0xf); 373219820Sjeff return (int)(p - str); 374219820Sjeff} 375219820Sjeff 376219820Sjeffstatic void qos_build_config(struct qos_config *cfg, 377219820Sjeff osm_qos_options_t * opt, osm_qos_options_t * dflt) 378219820Sjeff{ 379219820Sjeff int i; 380219820Sjeff char *p; 381219820Sjeff 382219820Sjeff memset(cfg, 0, sizeof(*cfg)); 383219820Sjeff 384219820Sjeff cfg->max_vls = opt->max_vls > 0 ? opt->max_vls : dflt->max_vls; 385219820Sjeff 386219820Sjeff if (opt->high_limit >= 0) 387219820Sjeff cfg->vl_high_limit = (uint8_t) opt->high_limit; 388219820Sjeff else 389219820Sjeff cfg->vl_high_limit = (uint8_t) dflt->high_limit; 390219820Sjeff 391219820Sjeff p = opt->vlarb_high ? opt->vlarb_high : dflt->vlarb_high; 392219820Sjeff for (i = 0; i < 2 * IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; i++) { 393219820Sjeff p += parse_vlarb_entry(p, 394219820Sjeff &cfg->vlarb_high[i / 395219820Sjeff IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]. 396219820Sjeff vl_entry[i % 397219820Sjeff IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]); 398219820Sjeff } 399219820Sjeff 400219820Sjeff p = opt->vlarb_low ? opt->vlarb_low : dflt->vlarb_low; 401219820Sjeff for (i = 0; i < 2 * IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; i++) { 402219820Sjeff p += parse_vlarb_entry(p, 403219820Sjeff &cfg->vlarb_low[i / 404219820Sjeff IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]. 405219820Sjeff vl_entry[i % 406219820Sjeff IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]); 407219820Sjeff } 408219820Sjeff 409219820Sjeff p = opt->sl2vl ? opt->sl2vl : dflt->sl2vl; 410219820Sjeff for (i = 0; i < IB_MAX_NUM_VLS / 2; i++) 411219820Sjeff p += parse_sl2vl_entry(p, &cfg->sl2vl.raw_vl_by_sl[i]); 412219820Sjeff 413219820Sjeff} 414