1/* 2 * Copyright (c) 2006-2008 Voltaire, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34/* 35 * Abstract: 36 * Implementation of OpenSM QoS infrastructure primitives 37 */ 38 39#if HAVE_CONFIG_H 40# include <config.h> 41#endif /* HAVE_CONFIG_H */ 42 43#include <stdlib.h> 44#include <string.h> 45 46#include <iba/ib_types.h> 47#include <complib/cl_qmap.h> 48#include <complib/cl_debug.h> 49#include <opensm/osm_opensm.h> 50#include <opensm/osm_subnet.h> 51#include <opensm/osm_qos_policy.h> 52 53struct qos_config { 54 uint8_t max_vls; 55 uint8_t vl_high_limit; 56 ib_vl_arb_table_t vlarb_high[2]; 57 ib_vl_arb_table_t vlarb_low[2]; 58 ib_slvl_table_t sl2vl; 59}; 60 61static void qos_build_config(struct qos_config *cfg, 62 osm_qos_options_t * opt, osm_qos_options_t * dflt); 63 64/* 65 * QoS primitives 66 */ 67static ib_api_status_t vlarb_update_table_block(osm_sm_t * sm, 68 osm_physp_t * p, 69 uint8_t port_num, 70 unsigned force_update, 71 const ib_vl_arb_table_t * 72 table_block, 73 unsigned block_length, 74 unsigned block_num) 75{ 76 ib_vl_arb_table_t block; 77 osm_madw_context_t context; 78 uint32_t attr_mod; 79 unsigned vl_mask, i; 80 81 vl_mask = (1 << (ib_port_info_get_op_vls(&p->port_info) - 1)) - 1; 82 83 memset(&block, 0, sizeof(block)); 84 memcpy(&block, table_block, block_length * sizeof(block.vl_entry[0])); 85 for (i = 0; i < block_length; i++) 86 block.vl_entry[i].vl &= vl_mask; 87 88 if (!force_update && 89 !memcmp(&p->vl_arb[block_num], &block, 90 block_length * sizeof(block.vl_entry[0]))) 91 return IB_SUCCESS; 92 93 context.vla_context.node_guid = 94 osm_node_get_node_guid(osm_physp_get_node_ptr(p)); 95 context.vla_context.port_guid = osm_physp_get_port_guid(p); 96 context.vla_context.set_method = TRUE; 97 attr_mod = ((block_num + 1) << 16) | port_num; 98 99 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p), 100 (uint8_t *) & block, sizeof(block), 101 IB_MAD_ATTR_VL_ARBITRATION, 102 cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); 103} 104 105static ib_api_status_t vlarb_update(osm_sm_t * sm, 106 osm_physp_t * p, uint8_t port_num, 107 unsigned force_update, 108 const struct qos_config *qcfg) 109{ 110 ib_api_status_t status = IB_SUCCESS; 111 ib_port_info_t *p_pi = &p->port_info; 112 unsigned len; 113 114 if (p_pi->vl_arb_low_cap > 0) { 115 len = p_pi->vl_arb_low_cap < IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK ? 116 p_pi->vl_arb_low_cap : IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 117 if ((status = vlarb_update_table_block(sm, p, port_num, 118 force_update, 119 &qcfg->vlarb_low[0], 120 len, 0)) != IB_SUCCESS) 121 return status; 122 } 123 if (p_pi->vl_arb_low_cap > IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK) { 124 len = p_pi->vl_arb_low_cap % IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 125 if ((status = vlarb_update_table_block(sm, p, port_num, 126 force_update, 127 &qcfg->vlarb_low[1], 128 len, 1)) != IB_SUCCESS) 129 return status; 130 } 131 if (p_pi->vl_arb_high_cap > 0) { 132 len = p_pi->vl_arb_high_cap < IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK ? 133 p_pi->vl_arb_high_cap : IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 134 if ((status = vlarb_update_table_block(sm, p, port_num, 135 force_update, 136 &qcfg->vlarb_high[0], 137 len, 2)) != IB_SUCCESS) 138 return status; 139 } 140 if (p_pi->vl_arb_high_cap > IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK) { 141 len = p_pi->vl_arb_high_cap % IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; 142 if ((status = vlarb_update_table_block(sm, p, port_num, 143 force_update, 144 &qcfg->vlarb_high[1], 145 len, 3)) != IB_SUCCESS) 146 return status; 147 } 148 149 return status; 150} 151 152static ib_api_status_t sl2vl_update_table(osm_sm_t * sm, 153 osm_physp_t * p, uint8_t in_port, 154 uint8_t out_port, 155 unsigned force_update, 156 const ib_slvl_table_t * sl2vl_table) 157{ 158 osm_madw_context_t context; 159 ib_slvl_table_t tbl, *p_tbl; 160 osm_node_t *p_node = osm_physp_get_node_ptr(p); 161 uint32_t attr_mod; 162 unsigned vl_mask; 163 uint8_t vl1, vl2; 164 int i; 165 166 vl_mask = (1 << (ib_port_info_get_op_vls(&p->port_info) - 1)) - 1; 167 168 for (i = 0; i < IB_MAX_NUM_VLS / 2; i++) { 169 vl1 = sl2vl_table->raw_vl_by_sl[i] >> 4; 170 vl2 = sl2vl_table->raw_vl_by_sl[i] & 0xf; 171 if (vl1 != 15) 172 vl1 &= vl_mask; 173 if (vl2 != 15) 174 vl2 &= vl_mask; 175 tbl.raw_vl_by_sl[i] = (vl1 << 4) | vl2; 176 } 177 178 if (!force_update && (p_tbl = osm_physp_get_slvl_tbl(p, in_port)) && 179 !memcmp(p_tbl, &tbl, sizeof(tbl))) 180 return IB_SUCCESS; 181 182 context.slvl_context.node_guid = osm_node_get_node_guid(p_node); 183 context.slvl_context.port_guid = osm_physp_get_port_guid(p); 184 context.slvl_context.set_method = TRUE; 185 attr_mod = in_port << 8 | out_port; 186 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p), 187 (uint8_t *) & tbl, sizeof(tbl), 188 IB_MAD_ATTR_SLVL_TABLE, 189 cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); 190} 191 192static ib_api_status_t sl2vl_update(osm_sm_t * sm, osm_port_t * p_port, 193 osm_physp_t * p, uint8_t port_num, 194 unsigned force_update, 195 const struct qos_config *qcfg) 196{ 197 ib_api_status_t status; 198 uint8_t i, num_ports; 199 osm_physp_t *p_physp; 200 201 if (osm_node_get_type(osm_physp_get_node_ptr(p)) == IB_NODE_TYPE_SWITCH) { 202 if (ib_port_info_get_vl_cap(&p->port_info) == 1) { 203 /* Check port 0's capability mask */ 204 p_physp = p_port->p_physp; 205 if (! 206 (p_physp->port_info. 207 capability_mask & IB_PORT_CAP_HAS_SL_MAP)) 208 return IB_SUCCESS; 209 } 210 num_ports = osm_node_get_num_physp(osm_physp_get_node_ptr(p)); 211 } else { 212 if (!(p->port_info.capability_mask & IB_PORT_CAP_HAS_SL_MAP)) 213 return IB_SUCCESS; 214 num_ports = 1; 215 } 216 217 for (i = 0; i < num_ports; i++) { 218 status = 219 sl2vl_update_table(sm, p, i, port_num, 220 force_update, &qcfg->sl2vl); 221 if (status != IB_SUCCESS) 222 return status; 223 } 224 225 return IB_SUCCESS; 226} 227 228static ib_api_status_t qos_physp_setup(osm_log_t * p_log, osm_sm_t * sm, 229 osm_port_t * p_port, osm_physp_t * p, 230 uint8_t port_num, 231 unsigned force_update, 232 const struct qos_config *qcfg) 233{ 234 ib_api_status_t status; 235 236 /* OpVLs should be ok at this moment - just use it */ 237 238 /* setup VL high limit on the physp later to be updated by link mgr */ 239 p->vl_high_limit = qcfg->vl_high_limit; 240 241 /* setup VLArbitration */ 242 status = vlarb_update(sm, p, port_num, force_update, qcfg); 243 if (status != IB_SUCCESS) { 244 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6202 : " 245 "failed to update VLArbitration tables " 246 "for port %" PRIx64 " #%d\n", 247 cl_ntoh64(p->port_guid), port_num); 248 return status; 249 } 250 251 /* setup SL2VL tables */ 252 status = sl2vl_update(sm, p_port, p, port_num, force_update, qcfg); 253 if (status != IB_SUCCESS) { 254 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 6203 : " 255 "failed to update SL2VLMapping tables " 256 "for port %" PRIx64 " #%d\n", 257 cl_ntoh64(p->port_guid), port_num); 258 return status; 259 } 260 261 return IB_SUCCESS; 262} 263 264osm_signal_t osm_qos_setup(osm_opensm_t * p_osm) 265{ 266 struct qos_config ca_config, sw0_config, swe_config, rtr_config; 267 struct qos_config *cfg; 268 cl_qmap_t *p_tbl; 269 cl_map_item_t *p_next; 270 osm_port_t *p_port; 271 uint32_t num_physp; 272 osm_physp_t *p_physp; 273 osm_node_t *p_node; 274 ib_api_status_t status; 275 unsigned force_update; 276 uint8_t i; 277 278 if (!p_osm->subn.opt.qos) 279 return OSM_SIGNAL_DONE; 280 281 OSM_LOG_ENTER(&p_osm->log); 282 283 qos_build_config(&ca_config, &p_osm->subn.opt.qos_ca_options, 284 &p_osm->subn.opt.qos_options); 285 qos_build_config(&sw0_config, &p_osm->subn.opt.qos_sw0_options, 286 &p_osm->subn.opt.qos_options); 287 qos_build_config(&swe_config, &p_osm->subn.opt.qos_swe_options, 288 &p_osm->subn.opt.qos_options); 289 qos_build_config(&rtr_config, &p_osm->subn.opt.qos_rtr_options, 290 &p_osm->subn.opt.qos_options); 291 292 cl_plock_excl_acquire(&p_osm->lock); 293 294 /* read QoS policy config file */ 295 osm_qos_parse_policy_file(&p_osm->subn); 296 297 p_tbl = &p_osm->subn.port_guid_tbl; 298 p_next = cl_qmap_head(p_tbl); 299 while (p_next != cl_qmap_end(p_tbl)) { 300 p_port = (osm_port_t *) p_next; 301 p_next = cl_qmap_next(p_next); 302 303 p_node = p_port->p_node; 304 if (p_node->sw) { 305 num_physp = osm_node_get_num_physp(p_node); 306 for (i = 1; i < num_physp; i++) { 307 p_physp = osm_node_get_physp_ptr(p_node, i); 308 if (!p_physp) 309 continue; 310 force_update = p_physp->need_update || 311 p_osm->subn.need_update; 312 status = 313 qos_physp_setup(&p_osm->log, &p_osm->sm, 314 p_port, p_physp, i, 315 force_update, &swe_config); 316 } 317 /* skip base port 0 */ 318 if (!ib_switch_info_is_enhanced_port0 319 (&p_node->sw->switch_info)) 320 continue; 321 322 cfg = &sw0_config; 323 } else if (osm_node_get_type(p_node) == IB_NODE_TYPE_ROUTER) 324 cfg = &rtr_config; 325 else 326 cfg = &ca_config; 327 328 p_physp = p_port->p_physp; 329 if (!p_physp) 330 continue; 331 332 force_update = p_physp->need_update || p_osm->subn.need_update; 333 status = qos_physp_setup(&p_osm->log, &p_osm->sm, 334 p_port, p_physp, 0, force_update, cfg); 335 } 336 337 cl_plock_release(&p_osm->lock); 338 OSM_LOG_EXIT(&p_osm->log); 339 340 return OSM_SIGNAL_DONE; 341} 342 343/* 344 * QoS config stuff 345 */ 346static int parse_one_unsigned(char *str, char delim, unsigned *val) 347{ 348 char *end; 349 *val = strtoul(str, &end, 0); 350 if (*end) 351 end++; 352 return (int)(end - str); 353} 354 355static int parse_vlarb_entry(char *str, ib_vl_arb_element_t * e) 356{ 357 unsigned val; 358 char *p = str; 359 p += parse_one_unsigned(p, ':', &val); 360 e->vl = val % 15; 361 p += parse_one_unsigned(p, ',', &val); 362 e->weight = (uint8_t) val; 363 return (int)(p - str); 364} 365 366static int parse_sl2vl_entry(char *str, uint8_t * raw) 367{ 368 unsigned val1, val2; 369 char *p = str; 370 p += parse_one_unsigned(p, ',', &val1); 371 p += parse_one_unsigned(p, ',', &val2); 372 *raw = (val1 << 4) | (val2 & 0xf); 373 return (int)(p - str); 374} 375 376static void qos_build_config(struct qos_config *cfg, 377 osm_qos_options_t * opt, osm_qos_options_t * dflt) 378{ 379 int i; 380 char *p; 381 382 memset(cfg, 0, sizeof(*cfg)); 383 384 cfg->max_vls = opt->max_vls > 0 ? opt->max_vls : dflt->max_vls; 385 386 if (opt->high_limit >= 0) 387 cfg->vl_high_limit = (uint8_t) opt->high_limit; 388 else 389 cfg->vl_high_limit = (uint8_t) dflt->high_limit; 390 391 p = opt->vlarb_high ? opt->vlarb_high : dflt->vlarb_high; 392 for (i = 0; i < 2 * IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; i++) { 393 p += parse_vlarb_entry(p, 394 &cfg->vlarb_high[i / 395 IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]. 396 vl_entry[i % 397 IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]); 398 } 399 400 p = opt->vlarb_low ? opt->vlarb_low : dflt->vlarb_low; 401 for (i = 0; i < 2 * IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK; i++) { 402 p += parse_vlarb_entry(p, 403 &cfg->vlarb_low[i / 404 IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]. 405 vl_entry[i % 406 IB_NUM_VL_ARB_ELEMENTS_IN_BLOCK]); 407 } 408 409 p = opt->sl2vl ? opt->sl2vl : dflt->sl2vl; 410 for (i = 0; i < IB_MAX_NUM_VLS / 2; i++) 411 p += parse_sl2vl_entry(p, &cfg->sl2vl.raw_vl_by_sl[i]); 412 413} 414