ibmirror.c revision 326169
1/* 2 * Copyright (c) 2004-2009 Voltaire Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#if HAVE_CONFIG_H 35# include <config.h> 36#endif /* HAVE_CONFIG_H */ 37 38#include <stdio.h> 39#include <stdlib.h> 40#include <unistd.h> 41#include <getopt.h> 42#include <netinet/in.h> 43 44#include <infiniband/umad.h> 45#include <infiniband/mad.h> 46 47#include "ibdiag_common.h" 48 49#define IB_MLX_VENDOR_CLASS 10 50 51/* Vendor specific Attribute IDs */ 52#define IB_MLX_IS3_GENERAL_INFO 0x17 53 54#define MAX_SWITCH_PORTS (36+1) 55static char mtx_ports[MAX_SWITCH_PORTS] = {0}; 56static char mrx_ports[MAX_SWITCH_PORTS] = {0}; 57static char str[4096]; 58static uint8_t buf[256]; 59 60#define ATTRID_PM_ROUTE 0xff30 61#define ATTRID_PM_FILTER 0xff31 62#define ATTRID_PM_PORTS 0xff32 63#define ATTRID_LOSSY_CFG 0xff80 64 65enum mirror_type { 66 MT_DISABLED = 0, 67 MT_MIRROR_NATIVE = 2, 68 MT_DROP = 5, 69 MT_MIRROR_ENCAP = 6, 70 MT_MIRROR_DROP = 7 71}; 72 73enum mirror_port { 74 MP_DISABLED = 0, 75 MP_MIRROR_FILTER = 1, 76 MP_MIRROR_ALWAYS = 2, 77 MP_MIRROR_FILTER_NOT = 3, 78 MT_MIRROR_AS_RX = 1 79}; 80 81#define PM_ENCAP_ETHERTYPE 0x1123 82 83struct ibmad_port *srcport; 84 85typedef struct { 86 uint16_t hw_revision; 87 uint16_t device_id; 88 uint8_t reserved[24]; 89 uint32_t uptime; 90} is3_hw_info_t; 91 92typedef struct { 93 uint8_t resv1; 94 uint8_t major; 95 uint8_t minor; 96 uint8_t sub_minor; 97 uint32_t build_id; 98 uint8_t month; 99 uint8_t day; 100 uint16_t year; 101 uint16_t resv2; 102 uint16_t hour; 103 uint8_t psid[16]; 104 uint32_t ini_file_version; 105} is3_fw_info_t; 106 107typedef struct { 108 uint8_t resv1; 109 uint8_t major; 110 uint8_t minor; 111 uint8_t sub_minor; 112 uint8_t resv2[28]; 113} is3_sw_info_t; 114 115typedef struct { 116 uint8_t reserved[8]; 117 is3_hw_info_t hw_info; 118 is3_fw_info_t fw_info; 119 is3_sw_info_t sw_info; 120} is3_general_info_t; 121 122typedef struct { 123 uint16_t ignore_buffer_mask; 124 uint16_t ignore_credit_mask; 125} lossy_config_t; 126 127static int mirror_query, mirror_dport, mirror_dlid, mirror_clear, mirror_sl, lossy_set; 128static int set_mtx, set_mrx, packet_size = 0xfff; 129 130static int parse_ports(char *ports_str, char *ports_array) 131{ 132 int num, i; 133 char *str = strdup(ports_str); 134 char *token = strtok(str, ","); 135 for (i = 0; i < MAX_SWITCH_PORTS && token; i++) { 136 num = strtoul(token, NULL, 0); 137 if (num > 0 && num < MAX_SWITCH_PORTS) 138 ports_array[num] = 1; 139 140 token = strtok(NULL, ","); 141 } 142 free(str); 143 return 0; 144} 145 146void port_mirror_route(ib_portid_t * portid, int query, int clear) 147{ 148 int mirror_type; 149 150 memset(&buf, 0, sizeof(buf)); 151 152 if (clear) { 153 if (!smp_set_via(buf, portid, ATTRID_PM_ROUTE, 0, 0, srcport)) 154 IBEXIT("Clear port mirror route set failed"); 155 return; 156 } 157 158 if (query) { 159 if (!smp_query_via(buf, portid, ATTRID_PM_ROUTE, 0, 0, srcport)) 160 IBEXIT("Read port mirror route get failed"); 161 mad_decode_field(buf, IB_PMR_MT_F, &mirror_type); 162 if (mirror_type == MT_MIRROR_ENCAP && mirror_dlid == 0) 163 mad_decode_field(buf, IB_PMR_LRH_DLID_F, &mirror_dlid); 164 if (mirror_type == MT_MIRROR_NATIVE && mirror_dport == 0) 165 mad_decode_field(buf, IB_PMR_NM_PORT_F, &mirror_dport); 166 goto Exit; 167 } 168 169 /* Port Mirror Route */ 170 mad_set_field(buf, 0, IB_PMR_ENCAP_RAW_ETH_TYPE_F, PM_ENCAP_ETHERTYPE); 171 172 if (mirror_dlid == 0) { 173 /* Can not truncate mirrored packets in local mode */ 174 mad_set_field(buf, 0, IB_PMR_MAX_MIRROR_LEN_F, 0xfff); 175 mad_set_field(buf, 0, IB_PMR_MT_F, MT_MIRROR_NATIVE); 176 mad_set_field(buf, 0, IB_PMR_NM_PORT_F, mirror_dport); 177 } 178 else { /* remote mirror */ 179 /* convert size to dwords */ 180 packet_size = packet_size / 4 + 1; 181 mad_set_field(buf, 0, IB_PMR_MAX_MIRROR_LEN_F, packet_size); 182 mad_set_field(buf, 0, IB_PMR_MT_F, MT_MIRROR_ENCAP); 183 mad_set_field(buf, 0, IB_PMR_LRH_SL_F, mirror_sl); 184 mad_set_field(buf, 0, IB_PMR_LRH_DLID_F, mirror_dlid); 185 mad_set_field(buf, 0, IB_PMR_LRH_SLID_F, portid->lid); 186 } 187 188 if (!smp_set_via(buf, portid, ATTRID_PM_ROUTE, 0, 0, srcport)) 189 IBEXIT("port mirror route set failed"); 190 191Exit: 192 mad_dump_portmirror_route(str, sizeof str, buf, sizeof buf); 193 printf("Port Mirror Route\n%s", str); 194} 195 196void port_mirror_ports(ib_portid_t * portid, int query, int clear) 197{ 198 int p, rqf, tqf, rqv, tqv; 199 200 memset(&buf, 0, sizeof(buf)); 201 202 if (clear) { 203 if (!smp_set_via(buf, portid, ATTRID_PM_PORTS, 0, 0, srcport)) 204 IBEXIT("Clear port mirror ports set failed"); 205 return; 206 } 207 208 if (query) { 209 if (!smp_query_via(buf, portid, ATTRID_PM_PORTS, 0, 0, srcport)) 210 IBEXIT("Read port mirror ports get failed"); 211 goto Exit; 212 } 213 214 /* Port Mirror Ports */ 215 rqf = IB_PMP_RQ_1_F; 216 tqf = IB_PMP_TQ_1_F; 217 218 for (p = 1; p < MAX_SWITCH_PORTS; p++) { 219 rqv = mrx_ports[p] ? MP_MIRROR_ALWAYS : MP_DISABLED; 220 tqv = mtx_ports[p] ? MP_MIRROR_ALWAYS : MT_MIRROR_AS_RX; 221 mad_set_field(buf, 0, rqf, rqv); 222 mad_set_field(buf, 0, tqf, tqv); 223 rqf += 2; 224 tqf += 2; 225 } 226 227 if (!smp_set_via(buf, portid, ATTRID_PM_PORTS, 0, 0, srcport)) 228 IBEXIT("port mirror ports set failed"); 229 230Exit: 231 mad_dump_portmirror_ports(str, sizeof str, buf, sizeof buf); 232 printf("Port Mirror Ports\n%s", str); 233} 234 235int get_out_port(ib_portid_t* portid) 236{ 237 int block; 238 int offset; 239 240 if (mirror_dlid) { 241 block = mirror_dlid / IB_SMP_DATA_SIZE; 242 offset = mirror_dlid - block * IB_SMP_DATA_SIZE; 243 /* get out port from lft */ 244 if (!smp_query_via(buf, portid, IB_ATTR_LINEARFORWTBL, block, 0, srcport)) 245 IBEXIT("linear forwarding table get failed"); 246 block = mirror_dlid / IB_SMP_DATA_SIZE; 247 offset = mirror_dlid - block * IB_SMP_DATA_SIZE; 248 return buf[offset]; 249 } 250 else 251 return mirror_dport; 252} 253 254int get_peer(ib_portid_t* portid, int outport, int* peerlid, int* peerport) 255{ 256 ib_portid_t selfportid = { 0 }; 257 ib_portid_t peerportid = { 0 }; 258 int selfport = 0; 259 260 /* set peerportid for peer port */ 261 memcpy(&peerportid, portid, sizeof(peerportid)); 262 peerportid.drpath.cnt = 1; 263 peerportid.drpath.p[1] = outport; 264 if (ib_resolve_self_via(&selfportid, &selfport, 0, srcport) < 0) 265 IBEXIT("failed to resolve self portid"); 266 peerportid.drpath.drslid = (uint16_t) selfportid.lid; 267 peerportid.drpath.drdlid = 0xffff; 268 if (!smp_query_via(buf, &peerportid, IB_ATTR_PORT_INFO, 0, 0, srcport)) 269 IBEXIT("get peer portinfo failed - unable to configure lossy\n"); 270 271 mad_decode_field(buf, IB_PORT_LID_F, peerlid); 272 mad_decode_field(buf, IB_PORT_LOCAL_PORT_F, peerport); 273 274 return 0; 275} 276 277int get_mirror_vl(ib_portid_t* portid, int outport) 278{ 279 ib_slvl_table_t * p_slvl_tbl; 280 int portnum; 281 int vl; 282 283 /* hack; assume all sl2vl mappings are the same for any in port and outport */ 284 portnum = (1 << 8) | outport; 285 286 /* get sl2vl mapping */ 287 if (!smp_query_via(buf, portid, IB_ATTR_SLVL_TABLE, portnum, 0, srcport)) 288 IBEXIT("slvl query failed"); 289 290 p_slvl_tbl = (ib_slvl_table_t *) buf; 291 vl = ib_slvl_table_get(p_slvl_tbl, mirror_sl); 292 printf("mirror_sl %d, mirror_vl %d\n", mirror_sl, vl); 293 return vl; 294} 295 296int lossy_config(ib_portid_t* portid, int query, int clear) 297{ 298 int outport; 299 int peerport; 300 int attr_mod; 301 uint8_t mirror_vl; 302 ib_portid_t peerportid = { 0 }; 303 ib_portid_t * p_portid; 304 lossy_config_t local_lossy_cfg; 305 lossy_config_t peer_lossy_cfg; 306 lossy_config_t lossy_cfg; 307 308 outport = get_out_port(portid); 309 if (outport == 0) 310 IBEXIT("get_out_port failed, mirror_dlid and mirror_dport are 0"); 311 312 get_peer(portid, outport, &peerportid.lid, &peerport); 313 314 printf("local lid %d / port %d\n", portid->lid, outport); 315 printf("peer lid %d / port %d\n", peerportid.lid, peerport); 316 317 mirror_vl = get_mirror_vl(portid, outport); 318 319 /* read local lossy configuration */ 320 if (!smp_query_via(buf, portid, ATTRID_LOSSY_CFG, outport, 0, srcport)) 321 IBEXIT("get lossy config from lid %d port %d failed - not supported\n", 322 portid->lid, outport); 323 memcpy(&local_lossy_cfg, buf, sizeof(local_lossy_cfg)); 324 325 /* read peer lossy configuration */ 326 if (!smp_query_via(buf, &peerportid, ATTRID_LOSSY_CFG, peerport, 0, srcport)) 327 IBEXIT("get lossy config from lid %d port %d failed - not supported\n", 328 peerportid.lid, peerport); 329 memcpy(&peer_lossy_cfg, buf, sizeof(peer_lossy_cfg)); 330 331 if (query) { 332 printf("local port lid %d port %d ignore_buffer 0x%04x, ignore_credit 0x%04x\n", 333 portid->lid, outport, 334 ntohs(local_lossy_cfg.ignore_buffer_mask), ntohs(local_lossy_cfg.ignore_credit_mask)); 335 printf("peer port lid %d port %d ignore_buffer 0x%04x, ignore_credit 0x%04x\n", 336 peerportid.lid, peerport, 337 ntohs(peer_lossy_cfg.ignore_buffer_mask), ntohs(peer_lossy_cfg.ignore_credit_mask)); 338 return 0; 339 } 340 341 /* VLs 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 */ 342 /* ignore Buf Overrun ignore Credits */ 343 /* when mirror activated set ignore buffer overrun on peer port */ 344 /* when mirror is de-activated clear ignore credits on local port */ 345 memset(&buf, 0, sizeof(buf)); 346 if (clear) { 347 p_portid = portid; 348 attr_mod = outport; 349 } else { 350 /* set buffer overrun on peer port */ 351 p_portid = &peerportid; 352 attr_mod = peerport; 353 lossy_cfg.ignore_buffer_mask = htons(1<<mirror_vl); 354 lossy_cfg.ignore_credit_mask = 0; 355 memcpy(&buf, &lossy_cfg, sizeof(lossy_cfg)); 356 } 357 if (!smp_set_via(buf, p_portid, ATTRID_LOSSY_CFG, attr_mod, 0, srcport)) 358 IBEXIT("%s lossy config on lid %d failed\n", clear?"clear":"set", p_portid->lid); 359 360 /* when mirror activated set ignore credit on local port */ 361 /* when mirror de-activated clear buffer overrun on peer */ 362 memset(&buf, 0, sizeof(buf)); 363 if (clear) { 364 p_portid = &peerportid; 365 attr_mod = peerport; 366 } else { 367 /* set ignore credit on local port */ 368 p_portid = portid; 369 attr_mod = outport; 370 lossy_cfg.ignore_credit_mask = htons(1<<mirror_vl); 371 lossy_cfg.ignore_buffer_mask = 0; 372 memcpy(&buf, &lossy_cfg, sizeof(lossy_cfg)); 373 } 374 if (!smp_set_via(buf, p_portid, ATTRID_LOSSY_CFG, attr_mod, 0, srcport)) 375 IBEXIT("%s lossy config on lid %d failed\n", clear?"clear":"set", p_portid->lid); 376 377 return 0; 378} 379 380int mirror_config(ib_portid_t* portid, int query, int clear) 381{ 382 port_mirror_route(portid, query, clear); 383 /* port_mirror_filter(portid, query, clear); */ 384 port_mirror_ports(portid, query, clear); 385 386 return 0; 387} 388 389static int process_opt(void *context, int ch, char *optarg) 390{ 391 switch (ch) { 392 case 'p': 393 mirror_dport = strtoul(optarg, NULL, 0); 394 break; 395 case 'S': 396 packet_size = strtoul(optarg, NULL, 0); 397 break; 398 case 'l': 399 mirror_sl = strtoul(optarg, NULL, 0); 400 break; 401 case 'L': 402 mirror_dlid = strtoul(optarg, NULL, 0); 403 break; 404 case 'R': 405 set_mrx = 1; 406 if (-1 == parse_ports(optarg, mrx_ports)) 407 return -1; 408 break; 409 case 'T': 410 set_mtx = 1; 411 if (-1 == parse_ports(optarg, mtx_ports)) 412 return -1; 413 break; 414 case 'D': 415 mirror_clear = 1; 416 break; 417 case 'Q': 418 mirror_query = 1; 419 break; 420 case 'y': 421 lossy_set = 1; 422 break; 423 default: 424 return -1; 425 } 426 return 0; 427} 428 429int main(int argc, char **argv) 430{ 431 int mgmt_classes[4] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS, 432 IB_MLX_VENDOR_CLASS 433 }; 434 ib_portid_t portid = { 0 }; 435 int port = 0; 436 ib_vendor_call_t call; 437 is3_general_info_t *gi; 438 uint32_t fw_ver; 439 char op_str[32]; 440 441 const struct ibdiag_opt opts[] = { 442 {"dport", 'p', 1, "<port>", "set mirror destination port"}, 443 {"dlid", 'L', 1, "<dlid>", "set mirror destination LID"}, 444 {"sl", 'l', 1, "<sl>", "set mirror SL"}, 445 {"size", 'S', 1, "<size>", "set packet size"}, 446 {"rxports", 'R', 1, NULL, "mirror receive port list"}, 447 {"txports", 'T', 1, NULL, "mirror transmit port list"}, 448 {"clear", 'D', 0, NULL, "clear ports mirroring"}, 449 {"query", 'Q', 0, NULL, "read mirror configuration"}, 450 {"lossy", 'y', 0, NULL, "set lossy configuration on out port"}, 451 {0} 452 }; 453 454 char usage_args[] = "<lid>"; 455 const char *usage_examples[] = { 456 "-R 1,2,3 -T 2,5 -l1 -L25 -S100 <lid>\t# configure mirror ports", 457 "-D <lid> \t# clear mirror configuration", 458 "-Q <lid>\t# read mirror configuration", 459 NULL 460 }; 461 462 ibdiag_process_opts(argc, argv, NULL, "GDLs", opts, process_opt, 463 usage_args, usage_examples); 464 465 argc -= optind; 466 argv += optind; 467 468 if (argc == 0) 469 ibdiag_show_usage(); 470 471 srcport = mad_rpc_open_port(ibd_ca, ibd_ca_port, mgmt_classes, 4); 472 if (!srcport) 473 IBEXIT("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port); 474 475 if (argc) { 476 if (ib_resolve_portid_str_via(&portid, argv[0], ibd_dest_type, 477 ibd_sm_id, srcport) < 0) 478 IBEXIT("can't resolve destination port %s", argv[0]); 479 } 480 481 482 memset(&buf, 0, sizeof(buf)); 483 memset(&call, 0, sizeof(call)); 484 call.mgmt_class = IB_MLX_VENDOR_CLASS; 485 call.method = IB_MAD_METHOD_GET; 486 call.timeout = ibd_timeout; 487 call.attrid = IB_MLX_IS3_GENERAL_INFO; 488 if (!ib_vendor_call_via(&buf, &portid, &call, srcport)) 489 IBEXIT("failed to read vendor info"); 490 gi = (is3_general_info_t *) & buf; 491 if (ntohs(gi->hw_info.device_id) != 0x1b3) 492 IBEXIT("device id 0x%x does not support mirroring", ntohs(gi->hw_info.device_id)); 493 494 fw_ver = gi->fw_info.major * 100000 + gi->fw_info.minor * 1000 + gi->fw_info.sub_minor; 495 printf("FW version %08d\n", fw_ver); 496 if (lossy_set && fw_ver < 704000) 497 IBEXIT("FW version %d.%d.%d does not support lossy config", 498 gi->fw_info.major, gi->fw_info.minor, gi->fw_info.sub_minor); 499 500 if (ibdebug) { 501 printf( "switch_lid = %d\n" 502 "mirror_clear = %d\n" 503 "mirror_dlid = %d\n" 504 "mirror_sl = %d\n" 505 "mirror_port = %d\n", 506 portid.lid, mirror_clear, mirror_dlid, 507 mirror_sl, mirror_dport); 508 509 for (port = 1; port < MAX_SWITCH_PORTS; port++) { 510 if (mtx_ports[port]) 511 printf("TX: %d\n",port); 512 else if(mrx_ports[port]) 513 printf("RX: %d\n",port); 514 } 515 } 516 517 if (mirror_clear) 518 strcpy(op_str, "Clear"); 519 else if (mirror_query) 520 strcpy(op_str, "Read"); 521 else if (!mirror_dport && !mirror_dlid) 522 IBEXIT("Mirror remote LID and local port are zero"); 523 else if (!set_mtx && !set_mrx) 524 IBEXIT("Mirror Rx and Tx ports not selected"); 525 else 526 strcpy(op_str, "Set"); 527 528 printf("\n%s Mirror Configuration\n", op_str); 529 mirror_config(&portid, mirror_query, mirror_clear); 530 531 if (lossy_set) { 532 printf("%s Lossy Configuration\n", op_str); 533 lossy_config(&portid, mirror_query, mirror_clear); 534 } 535 536 mad_rpc_close_port(srcport); 537 exit(0); 538} 539