1/* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2006 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36/* 37 * Abstract: 38 * Implementation of the P_Key Manager (Partititon Manager). 39 * This is part of the OpenSM. 40 */ 41 42#if HAVE_CONFIG_H 43# include <config.h> 44#endif /* HAVE_CONFIG_H */ 45 46#include <string.h> 47#include <iba/ib_types.h> 48#include <complib/cl_qmap.h> 49#include <complib/cl_debug.h> 50#include <opensm/osm_node.h> 51#include <opensm/osm_switch.h> 52#include <opensm/osm_partition.h> 53#include <opensm/osm_opensm.h> 54 55/********************************************************************** 56 **********************************************************************/ 57/* 58 The max number of pkey blocks for a physical port is located in 59 a different place for switch external ports (SwitchInfo) and the 60 rest of the ports (NodeInfo). 61*/ 62static uint16_t 63pkey_mgr_get_physp_max_blocks(IN const osm_subn_t * p_subn, 64 IN const osm_physp_t * p_physp) 65{ 66 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 67 uint16_t num_pkeys = 0; 68 69 if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0)) 70 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap); 71 else 72 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap); 73 return ((num_pkeys + 31) / 32); 74} 75 76/********************************************************************** 77 **********************************************************************/ 78/* 79 * Insert new pending pkey entry to the specific port pkey table 80 * pending pkeys. New entries are inserted at the back. 81 */ 82static void 83pkey_mgr_process_physical_port(IN osm_log_t * p_log, 84 IN osm_sm_t * sm, 85 IN const ib_net16_t pkey, 86 IN osm_physp_t * p_physp) 87{ 88 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 89 osm_pkey_tbl_t *p_pkey_tbl; 90 ib_net16_t *p_orig_pkey; 91 char *stat = NULL; 92 osm_pending_pkey_t *p_pending; 93 94 p_pkey_tbl = &p_physp->pkeys; 95 p_pending = (osm_pending_pkey_t *) malloc(sizeof(osm_pending_pkey_t)); 96 if (!p_pending) { 97 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: " 98 "Failed to allocate new pending pkey entry for node " 99 "0x%016" PRIx64 " port %u\n", 100 cl_ntoh64(osm_node_get_node_guid(p_node)), 101 osm_physp_get_port_num(p_physp)); 102 return; 103 } 104 p_pending->pkey = pkey; 105 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, ib_pkey_get_base(pkey)); 106 if (!p_orig_pkey) { 107 p_pending->is_new = TRUE; 108 cl_qlist_insert_tail(&p_pkey_tbl->pending, 109 (cl_list_item_t *) p_pending); 110 stat = "inserted"; 111 } else { 112 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) == 113 ib_pkey_get_base(pkey)); 114 p_pending->is_new = FALSE; 115 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey, 116 &p_pending->block, 117 &p_pending->index) != 118 IB_SUCCESS) { 119 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: " 120 "Failed to obtain P_Key 0x%04x block and index for node " 121 "0x%016" PRIx64 " port %u\n", 122 ib_pkey_get_base(pkey), 123 cl_ntoh64(osm_node_get_node_guid(p_node)), 124 osm_physp_get_port_num(p_physp)); 125 return; 126 } 127 cl_qlist_insert_head(&p_pkey_tbl->pending, 128 (cl_list_item_t *) p_pending); 129 stat = "updated"; 130 } 131 132 OSM_LOG(p_log, OSM_LOG_DEBUG, 133 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n", 134 cl_ntoh16(pkey), stat, 135 cl_ntoh64(osm_node_get_node_guid(p_node)), 136 osm_physp_get_port_num(p_physp)); 137} 138 139/********************************************************************** 140 **********************************************************************/ 141static void 142pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm, 143 const osm_prtn_t * p_prtn, 144 const boolean_t full) 145{ 146 const cl_map_t *p_tbl = 147 full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl; 148 cl_map_iterator_t i, i_next; 149 ib_net16_t pkey = p_prtn->pkey; 150 osm_physp_t *p_physp; 151 152 if (full) 153 pkey |= cl_hton16(0x8000); 154 155 i_next = cl_map_head(p_tbl); 156 while (i_next != cl_map_end(p_tbl)) { 157 i = i_next; 158 i_next = cl_map_next(i); 159 p_physp = cl_map_obj(i); 160 if (p_physp) 161 pkey_mgr_process_physical_port(p_log, sm, pkey, 162 p_physp); 163 } 164} 165 166/********************************************************************** 167 **********************************************************************/ 168static ib_api_status_t 169pkey_mgr_update_pkey_entry(IN osm_sm_t * sm, 170 IN const osm_physp_t * p_physp, 171 IN const ib_pkey_table_t * block, 172 IN const uint16_t block_index) 173{ 174 osm_madw_context_t context; 175 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 176 uint32_t attr_mod; 177 178 context.pkey_context.node_guid = osm_node_get_node_guid(p_node); 179 context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp); 180 context.pkey_context.set_method = TRUE; 181 attr_mod = block_index; 182 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH) 183 attr_mod |= osm_physp_get_port_num(p_physp) << 16; 184 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp), 185 (uint8_t *) block, sizeof(*block), 186 IB_MAD_ATTR_P_KEY_TABLE, 187 cl_hton32(attr_mod), CL_DISP_MSGID_NONE, &context); 188} 189 190/********************************************************************** 191 **********************************************************************/ 192static boolean_t 193pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm, 194 IN osm_physp_t * p_physp, IN const boolean_t enforce) 195{ 196 osm_madw_context_t context; 197 uint8_t payload[IB_SMP_DATA_SIZE]; 198 ib_port_info_t *p_pi; 199 ib_api_status_t status; 200 201 p_pi = &p_physp->port_info; 202 203 if ((p_pi->vl_enforce & 0xc) == (0xc) * (enforce == TRUE)) { 204 OSM_LOG(p_log, OSM_LOG_DEBUG, 205 "No need to update PortInfo for " 206 "node 0x%016" PRIx64 " port %u\n", 207 cl_ntoh64(osm_node_get_node_guid 208 (osm_physp_get_node_ptr(p_physp))), 209 osm_physp_get_port_num(p_physp)); 210 return FALSE; 211 } 212 213 memset(payload, 0, IB_SMP_DATA_SIZE); 214 memcpy(payload, p_pi, sizeof(ib_port_info_t)); 215 216 p_pi = (ib_port_info_t *) payload; 217 if (enforce == TRUE) 218 p_pi->vl_enforce |= 0xc; 219 else 220 p_pi->vl_enforce &= ~0xc; 221 p_pi->state_info2 = 0; 222 ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE); 223 224 context.pi_context.node_guid = 225 osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp)); 226 context.pi_context.port_guid = osm_physp_get_port_guid(p_physp); 227 context.pi_context.set_method = TRUE; 228 context.pi_context.light_sweep = FALSE; 229 context.pi_context.active_transition = FALSE; 230 231 status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp), 232 payload, sizeof(payload), 233 IB_MAD_ATTR_PORT_INFO, 234 cl_hton32(osm_physp_get_port_num(p_physp)), 235 CL_DISP_MSGID_NONE, &context); 236 if (status != IB_SUCCESS) { 237 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: " 238 "Failed to set PortInfo for " 239 "node 0x%016" PRIx64 " port %u\n", 240 cl_ntoh64(osm_node_get_node_guid 241 (osm_physp_get_node_ptr(p_physp))), 242 osm_physp_get_port_num(p_physp)); 243 return FALSE; 244 } else { 245 OSM_LOG(p_log, OSM_LOG_DEBUG, 246 "Set PortInfo for node 0x%016" PRIx64 " port %u\n", 247 cl_ntoh64(osm_node_get_node_guid 248 (osm_physp_get_node_ptr(p_physp))), 249 osm_physp_get_port_num(p_physp)); 250 return TRUE; 251 } 252} 253 254/********************************************************************** 255 **********************************************************************/ 256static boolean_t pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, 257 const osm_port_t * const p_port) 258{ 259 osm_physp_t *p_physp; 260 osm_node_t *p_node; 261 ib_pkey_table_t *block, *new_block; 262 osm_pkey_tbl_t *p_pkey_tbl; 263 uint16_t block_index; 264 uint8_t pkey_index; 265 uint16_t last_free_block_index = 0; 266 uint8_t last_free_pkey_index = 0; 267 uint16_t num_of_blocks; 268 uint16_t max_num_of_blocks; 269 ib_api_status_t status; 270 boolean_t ret_val = FALSE; 271 osm_pending_pkey_t *p_pending; 272 boolean_t found; 273 ib_pkey_table_t empty_block; 274 275 memset(&empty_block, 0, sizeof(ib_pkey_table_t)); 276 277 p_physp = p_port->p_physp; 278 if (!p_physp) 279 return FALSE; 280 281 p_node = osm_physp_get_node_ptr(p_physp); 282 p_pkey_tbl = &p_physp->pkeys; 283 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl); 284 max_num_of_blocks = 285 pkey_mgr_get_physp_max_blocks(sm->p_subn, p_physp); 286 if (p_pkey_tbl->max_blocks > max_num_of_blocks) { 287 OSM_LOG(p_log, OSM_LOG_INFO, 288 "Max number of blocks reduced from %u to %u " 289 "for node 0x%016" PRIx64 " port %u\n", 290 p_pkey_tbl->max_blocks, max_num_of_blocks, 291 cl_ntoh64(osm_node_get_node_guid(p_node)), 292 osm_physp_get_port_num(p_physp)); 293 } 294 p_pkey_tbl->max_blocks = max_num_of_blocks; 295 296 osm_pkey_tbl_init_new_blocks(p_pkey_tbl); 297 p_pkey_tbl->used_blocks = 0; 298 299 /* 300 process every pending pkey in order - 301 first must be "updated" last are "new" 302 */ 303 p_pending = 304 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending); 305 while (p_pending != 306 (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) { 307 if (p_pending->is_new == FALSE) { 308 block_index = p_pending->block; 309 pkey_index = p_pending->index; 310 found = TRUE; 311 } else { 312 found = osm_pkey_find_next_free_entry(p_pkey_tbl, 313 &last_free_block_index, 314 &last_free_pkey_index); 315 if (!found) { 316 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0504: " 317 "Failed to find empty space for new pkey 0x%04x " 318 "for node 0x%016" PRIx64 " port %u\n", 319 cl_ntoh16(p_pending->pkey), 320 cl_ntoh64(osm_node_get_node_guid 321 (p_node)), 322 osm_physp_get_port_num(p_physp)); 323 } else { 324 block_index = last_free_block_index; 325 pkey_index = last_free_pkey_index++; 326 } 327 } 328 329 if (found) { 330 if (IB_SUCCESS != 331 osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index, 332 pkey_index, 333 p_pending->pkey)) { 334 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: " 335 "Failed to set PKey 0x%04x in block %u idx %u " 336 "for node 0x%016" PRIx64 " port %u\n", 337 cl_ntoh16(p_pending->pkey), block_index, 338 pkey_index, 339 cl_ntoh64(osm_node_get_node_guid 340 (p_node)), 341 osm_physp_get_port_num(p_physp)); 342 } 343 } 344 345 free(p_pending); 346 p_pending = 347 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl-> 348 pending); 349 } 350 351 /* now look for changes and store */ 352 for (block_index = 0; block_index < num_of_blocks; block_index++) { 353 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index); 354 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); 355 if (!new_block) 356 new_block = &empty_block; 357 if (block && !memcmp(new_block, block, sizeof(*block))) 358 continue; 359 360 status = 361 pkey_mgr_update_pkey_entry(sm, p_physp, new_block, 362 block_index); 363 if (status == IB_SUCCESS) { 364 OSM_LOG(p_log, OSM_LOG_DEBUG, 365 "Updated pkey table block %d for node 0x%016" 366 PRIx64 " port %u\n", block_index, 367 cl_ntoh64(osm_node_get_node_guid(p_node)), 368 osm_physp_get_port_num(p_physp)); 369 ret_val = TRUE; 370 } else { 371 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: " 372 "pkey_mgr_update_pkey_entry() failed to update " 373 "pkey table block %d for node 0x%016" PRIx64 374 " port %u\n", block_index, 375 cl_ntoh64(osm_node_get_node_guid(p_node)), 376 osm_physp_get_port_num(p_physp)); 377 } 378 } 379 380 return ret_val; 381} 382 383/********************************************************************** 384 **********************************************************************/ 385static boolean_t 386pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm, 387 const osm_subn_t * p_subn, 388 const osm_port_t * const p_port, boolean_t enforce) 389{ 390 osm_physp_t *p_physp, *peer; 391 osm_node_t *p_node; 392 ib_pkey_table_t *block, *peer_block; 393 const osm_pkey_tbl_t *p_pkey_tbl; 394 osm_pkey_tbl_t *p_peer_pkey_tbl; 395 uint16_t block_index; 396 uint16_t num_of_blocks; 397 uint16_t peer_max_blocks; 398 ib_api_status_t status = IB_SUCCESS; 399 boolean_t ret_val = FALSE; 400 boolean_t port_info_set = FALSE; 401 ib_pkey_table_t empty_block; 402 403 memset(&empty_block, 0, sizeof(ib_pkey_table_t)); 404 405 p_physp = p_port->p_physp; 406 if (!p_physp) 407 return FALSE; 408 peer = osm_physp_get_remote(p_physp); 409 if (!peer) 410 return FALSE; 411 p_node = osm_physp_get_node_ptr(peer); 412 if (!p_node->sw || !p_node->sw->switch_info.enforce_cap) 413 return FALSE; 414 415 p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp); 416 p_peer_pkey_tbl = &peer->pkeys; 417 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl); 418 peer_max_blocks = pkey_mgr_get_physp_max_blocks(p_subn, peer); 419 if (peer_max_blocks < p_pkey_tbl->used_blocks) { 420 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: " 421 "Not enough pkey entries (%u < %u) on switch 0x%016" 422 PRIx64 " port %u. Clearing Enforcement bit\n", 423 peer_max_blocks, num_of_blocks, 424 cl_ntoh64(osm_node_get_node_guid(p_node)), 425 osm_physp_get_port_num(peer)); 426 enforce = FALSE; 427 } 428 429 if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce)) 430 port_info_set = TRUE; 431 432 if (enforce == FALSE) 433 return port_info_set; 434 435 p_peer_pkey_tbl->used_blocks = p_pkey_tbl->used_blocks; 436 for (block_index = 0; block_index < p_pkey_tbl->used_blocks; 437 block_index++) { 438 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); 439 if (!block) 440 block = &empty_block; 441 442 peer_block = 443 osm_pkey_tbl_block_get(p_peer_pkey_tbl, block_index); 444 if (!peer_block 445 || memcmp(peer_block, block, sizeof(*peer_block))) { 446 status = 447 pkey_mgr_update_pkey_entry(sm, peer, block, 448 block_index); 449 if (status == IB_SUCCESS) 450 ret_val = TRUE; 451 else 452 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: " 453 "pkey_mgr_update_pkey_entry() failed to update " 454 "pkey table block %d for node 0x%016" 455 PRIx64 " port %u\n", block_index, 456 cl_ntoh64(osm_node_get_node_guid 457 (p_node)), 458 osm_physp_get_port_num(peer)); 459 } 460 } 461 462 if (ret_val) 463 OSM_LOG(p_log, OSM_LOG_DEBUG, 464 "Pkey table was updated for node 0x%016" PRIx64 465 " port %u\n", 466 cl_ntoh64(osm_node_get_node_guid(p_node)), 467 osm_physp_get_port_num(peer)); 468 469 if (port_info_set) 470 return TRUE; 471 return ret_val; 472} 473 474/********************************************************************** 475 **********************************************************************/ 476osm_signal_t osm_pkey_mgr_process(IN osm_opensm_t * p_osm) 477{ 478 cl_qmap_t *p_tbl; 479 cl_map_item_t *p_next; 480 osm_prtn_t *p_prtn; 481 osm_port_t *p_port; 482 osm_signal_t signal = OSM_SIGNAL_DONE; 483 484 CL_ASSERT(p_osm); 485 486 OSM_LOG_ENTER(&p_osm->log); 487 488 CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock); 489 490 if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) { 491 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: " 492 "osm_prtn_make_partitions() failed\n"); 493 goto _err; 494 } 495 496 /* populate the pending pkey entries by scanning all partitions */ 497 p_tbl = &p_osm->subn.prtn_pkey_tbl; 498 p_next = cl_qmap_head(p_tbl); 499 while (p_next != cl_qmap_end(p_tbl)) { 500 p_prtn = (osm_prtn_t *) p_next; 501 p_next = cl_qmap_next(p_next); 502 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm, 503 p_prtn, FALSE); 504 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm, 505 p_prtn, TRUE); 506 } 507 508 /* calculate and set new pkey tables */ 509 p_tbl = &p_osm->subn.port_guid_tbl; 510 p_next = cl_qmap_head(p_tbl); 511 while (p_next != cl_qmap_end(p_tbl)) { 512 p_port = (osm_port_t *) p_next; 513 p_next = cl_qmap_next(p_next); 514 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port)) 515 signal = OSM_SIGNAL_DONE_PENDING; 516 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH) 517 && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm, 518 &p_osm->subn, p_port, 519 !p_osm->subn.opt. 520 no_partition_enforcement)) 521 signal = OSM_SIGNAL_DONE_PENDING; 522 } 523 524_err: 525 CL_PLOCK_RELEASE(&p_osm->lock); 526 OSM_LOG_EXIT(&p_osm->log); 527 return (signal); 528} 529