1/* 2 * Copyright (c) 2006 Voltaire, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34#include <stdio.h> 35#include <string.h> 36#include <errno.h> 37#include <inttypes.h> 38 39#include <infiniband/umad.h> 40#include <infiniband/mad.h> 41 42#define info(fmt, arg...) fprintf(stderr, "INFO: " fmt, ##arg ) 43#define err(fmt, arg...) fprintf(stderr, "ERR: " fmt, ##arg ) 44#ifdef NOISY_DEBUG 45#define dbg(fmt, arg...) fprintf(stderr, "DBG: " fmt, ##arg ) 46#else 47#define dbg(fmt, arg...) 48#endif 49 50#define TMO 100 51 52/* Multicast Member Record Component Masks */ 53#define IB_MCR_COMPMASK_MGID (1ULL<<0) 54#define IB_MCR_COMPMASK_PORT_GID (1ULL<<1) 55#define IB_MCR_COMPMASK_QKEY (1ULL<<2) 56#define IB_MCR_COMPMASK_MLID (1ULL<<3) 57#define IB_MCR_COMPMASK_MTU_SEL (1ULL<<4) 58#define IB_MCR_COMPMASK_MTU (1ULL<<5) 59#define IB_MCR_COMPMASK_TCLASS (1ULL<<6) 60#define IB_MCR_COMPMASK_PKEY (1ULL<<7) 61#define IB_MCR_COMPMASK_RATE_SEL (1ULL<<8) 62#define IB_MCR_COMPMASK_RATE (1ULL<<9) 63#define IB_MCR_COMPMASK_LIFE_SEL (1ULL<<10) 64#define IB_MCR_COMPMASK_LIFE (1ULL<<11) 65#define IB_MCR_COMPMASK_SL (1ULL<<12) 66#define IB_MCR_COMPMASK_FLOW (1ULL<<13) 67#define IB_MCR_COMPMASK_HOP (1ULL<<14) 68#define IB_MCR_COMPMASK_SCOPE (1ULL<<15) 69#define IB_MCR_COMPMASK_JOIN_STATE (1ULL<<16) 70#define IB_MCR_COMPMASK_PROXY (1ULL<<17) 71 72static ibmad_gid_t mgid_ipoib = { 73 0xff, 0x12, 0x40, 0x1b, 0xff, 0xff, 0x00, 0x00, 74 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff 75}; 76 77uint64_t build_mcm_rec(uint8_t *data, ibmad_gid_t mgid, ibmad_gid_t port_gid) 78{ 79 memset(data, 0, IB_SA_DATA_SIZE); 80 mad_set_array(data, 0, IB_SA_MCM_MGID_F, mgid); 81 mad_set_array(data, 0, IB_SA_MCM_PORTGID_F, port_gid); 82 mad_set_field(data, 0, IB_SA_MCM_JOIN_STATE_F, 1); 83 84 return IB_MCR_COMPMASK_MGID|IB_MCR_COMPMASK_PORT_GID| 85 IB_MCR_COMPMASK_JOIN_STATE; 86} 87 88static void build_mcm_rec_umad(void *umad, ib_portid_t *dport, int method, 89 uint64_t comp_mask, uint8_t *data) 90{ 91 ib_rpc_t rpc; 92 93 memset(&rpc, 0, sizeof(rpc)); 94 rpc.mgtclass = IB_SA_CLASS; 95 rpc.method = method; 96 rpc.attr.id = IB_SA_ATTR_MCRECORD; 97 rpc.attr.mod = 0; // ??? 98 rpc.mask = comp_mask; 99 rpc.datasz = IB_SA_DATA_SIZE; 100 rpc.dataoffs = IB_SA_DATA_OFFS; 101 102 mad_build_pkt(umad, &rpc, dport, NULL, data); 103} 104 105static int rereg_send(int port, int agent, ib_portid_t *dport, 106 uint8_t *umad, int len, int method, ibmad_gid_t port_gid) 107{ 108 uint8_t data[IB_SA_DATA_SIZE]; 109 uint64_t comp_mask; 110 111 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid); 112 113 build_mcm_rec_umad(umad, dport, method, comp_mask, data); 114 if(umad_send(port, agent, umad, len, TMO, 0) < 0) { 115 err("umad_send leave failed: %s\n", strerror(errno)); 116 return -1; 117 } 118 dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method, 119 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 120 121 return 0; 122} 123 124static int rereg_port_gid(int port, int agent, ib_portid_t *dport, 125 uint8_t *umad, int len, ibmad_gid_t port_gid) 126{ 127 uint8_t data[IB_SA_DATA_SIZE]; 128 uint64_t comp_mask; 129 130 comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid); 131 132 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE, 133 comp_mask, data); 134 if(umad_send(port, agent, umad, len, TMO, 0) < 0) { 135 err("umad_send leave failed: %s\n", strerror(errno)); 136 return -1; 137 } 138 dbg("umad_send leave: tid = 0x%016" PRIx64 "\n", 139 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 140 141 build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET, 142 comp_mask, data); 143 if(umad_send(port, agent, umad, len, TMO, 0) < 0) { 144 err("umad_send join failed: %s\n", strerror(errno)); 145 return -1; 146 } 147 dbg("umad_send join: tid = 0x%016" PRIx64 "\n", 148 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F)); 149 150 return 0; 151} 152 153struct guid_trid { 154 ibmad_gid_t gid; 155 uint64_t guid; 156 uint64_t trid; 157}; 158 159static int rereg_send_all(int port, int agent, ib_portid_t *dport, 160 struct guid_trid *list, unsigned cnt) 161{ 162 uint8_t *umad; 163 int len = umad_size() + 256; 164 int i, ret; 165 166 info("rereg_send_all... cnt = %u\n", cnt); 167 168 umad = calloc(1, len); 169 if (!umad) { 170 err("cannot alloc mem for umad: %s\n", strerror(errno)); 171 return -1; 172 } 173 174 for (i = 0; i < cnt; i++) { 175 ret = rereg_port_gid(port, agent, dport, umad, len, list[i].gid); 176 if (ret < 0) { 177 err("rereg_send_all: rereg_port_gid 0x%016" PRIx64 178 " failed\n", list[i].guid); 179 continue; 180 } 181 list[i].trid = mad_get_field64(umad_get_mad(umad), 0, 182 IB_MAD_TRID_F); 183 } 184 185 info("rereg_send_all: sent %u requests\n", cnt*2); 186 187 free(umad); 188 189 return 0; 190} 191 192#if 0 193static int rereg_mcm_rec_send(int port, int agent, ib_portid_t *dport, int cnt) 194{ 195 ib_portid_t portid; 196 ibmad_gid_t port_gid; 197 uint8_t *umad; 198 int len, ret = 0; 199 200 ib_resolve_self(&portid, NULL, &port_gid); 201 202 len = umad_size() + 256; 203 umad = calloc(1, len); 204 if (!umad) { 205 err("cannot alloc mem for umad: %s\n", strerror(errno)); 206 return -1; 207 } 208 209 while(cnt--) { 210 if (!rereg_port_gid(port, agent, dport, umad, len, port_gid)) 211 ret += 2; 212 } 213 214 free(umad); 215 216 return ret; 217} 218#endif 219 220static int rereg_recv(int port, int agent, ib_portid_t *dport, 221 uint8_t *umad, int length, int tmo) 222{ 223 int ret, retry = 0; 224 int len = length; 225 226 while((ret = umad_recv(port, umad, &len, tmo)) < 0 && 227 errno == ETIMEDOUT) { 228 if (retry++ > 3) 229 return 0; 230 } 231 if (ret < 0) { 232 err("umad_recv %d failed: %s\n", ret, strerror(errno)); 233 return -1; 234 } 235 dbg("umad_recv (retries %d), tid = 0x%016" PRIx64 ": len = %d, status = %d\n", 236 retry, 237 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), 238 len, umad_status(umad)); 239 240 return 1; 241} 242 243static int rereg_recv_all(int port, int agent, ib_portid_t *dport, 244 struct guid_trid *list, unsigned cnt) 245{ 246 uint8_t *umad, *mad; 247 int len = umad_size() + 256; 248 uint64_t trid; 249 unsigned n, method, status; 250 int i; 251 252 info("rereg_recv_all...\n"); 253 254 umad = calloc(1, len); 255 if (!umad) { 256 err("cannot alloc mem for umad: %s\n", strerror(errno)); 257 return -1; 258 } 259 260 n = 0; 261 while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) { 262 dbg("rereg_recv_all: done %d\n", n); 263 n++; 264 mad = umad_get_mad(umad); 265 266 method = mad_get_field(mad, 0, IB_MAD_METHOD_F); 267 status = mad_get_field(mad, 0, IB_MAD_STATUS_F); 268 269 if (status) 270 dbg("MAD status %x, method %x\n", status, method); 271 272 if (status && 273 (method&0x7f) == (IB_MAD_METHOD_GET_RESPONSE&0x7f)) { 274 trid = mad_get_field64(mad, 0, IB_MAD_TRID_F); 275 for (i = 0; i < cnt; i++) 276 if (trid == list[i].trid) 277 break; 278 if (i == cnt) { 279 err("cannot find trid 0x%016" PRIx64 "\n", 280 trid); 281 continue; 282 } 283 info("guid 0x%016" PRIx64 ": method = %x status = %x. Resending\n", 284 ntohll(list[i].guid), method, status); 285 rereg_port_gid(port, agent, dport, umad, len, 286 list[i].gid); 287 list[i].trid = mad_get_field64(umad_get_mad(umad), 0, 288 IB_MAD_TRID_F); 289 } 290 } 291 292 info("rereg_recv_all: got %u responses\n", n); 293 294 free(umad); 295 return 0; 296} 297 298static int rereg_query_all(int port, int agent, ib_portid_t *dport, 299 struct guid_trid *list, unsigned cnt) 300{ 301 uint8_t *umad, *mad; 302 int len = umad_size() + 256; 303 unsigned method, status; 304 int i, ret; 305 306 info("rereg_query_all...\n"); 307 308 umad = calloc(1, len); 309 if (!umad) { 310 err("cannot alloc mem for umad: %s\n", strerror(errno)); 311 return -1; 312 } 313 314 for ( i = 0; i < cnt; i++ ) { 315 ret = rereg_send(port, agent, dport, umad, len, 316 IB_MAD_METHOD_GET, list[i].gid); 317 if (ret < 0) { 318 err("query_all: rereg_send failed.\n"); 319 continue; 320 } 321 322 ret = rereg_recv(port, agent, dport, umad, len, TMO); 323 if (ret < 0) { 324 err("query_all: rereg_recv failed.\n"); 325 continue; 326 } 327 328 mad = umad_get_mad(umad); 329 330 method = mad_get_field(mad, 0, IB_MAD_METHOD_F); 331 status = mad_get_field(mad, 0, IB_MAD_STATUS_F); 332 333 if (status) 334 info("guid 0x%016" PRIx64 ": status %x, method %x\n", 335 ntohll(list[i].guid), status, method); 336 } 337 338 info("rereg_query_all: %u queried.\n", cnt); 339 340 free(umad); 341 return 0; 342} 343 344#if 0 345static int rereg_mcm_rec_recv(int port, int agent, int cnt) 346{ 347 uint8_t *umad, *mad; 348 int len = umad_size() + 256; 349 int i; 350 351 umad = calloc(1, len); 352 if (!umad) { 353 err("cannot alloc mem for umad: %s\n", strerror(errno)); 354 return -1; 355 } 356 357 for ( i = 0; i < cnt; i++ ) { 358 int retry; 359 retry = 0; 360 while (umad_recv(port, umad, &len, TMO) < 0 && 361 errno == ETIMEDOUT) 362 if (retry++ > 3) { 363 err("umad_recv %d failed: %s\n", 364 i, strerror(errno)); 365 free(umad); 366 return -1; 367 } 368 dbg("umad_recv %d (retries %d), tid = 0x%016" PRIx64 ": len = %d, status = %d\n", 369 i, retry, 370 mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), 371 len, umad_status(umad)); 372 mad = umad_get_mad(umad); 373 } 374 375 free(umad); 376 return 0; 377} 378#endif 379 380#define MAX_CLIENTS 50 381 382static int rereg_and_test_port(char *guid_file, int port, int agent, ib_portid_t *dport, int timeout) 383{ 384 char line[256]; 385 FILE *f; 386 ibmad_gid_t port_gid; 387 uint64_t prefix = htonll(0xfe80000000000000llu); 388 uint64_t guid = htonll(0x0002c90200223825llu); 389 struct guid_trid *list; 390 int i = 0; 391 392 list = calloc(MAX_CLIENTS, sizeof(*list)); 393 if (!list) { 394 err("cannot alloc mem for guid/trid list: %s\n", strerror(errno)); 395 return -1; 396 } 397 398 f = fopen(guid_file, "r"); 399 if (!f) { 400 err("cannot open %s: %s\n", guid_file, strerror(errno)); 401 return -1; 402 } 403 404 while (fgets(line, sizeof(line), f)) { 405 guid = strtoull(line, NULL, 0); 406 guid = htonll(guid); 407 memcpy(&port_gid[0], &prefix, 8); 408 memcpy(&port_gid[8], &guid, 8); 409 410 list[i].guid = guid; 411 memcpy(list[i].gid, port_gid, sizeof(list[i].gid)); 412 list[i].trid = 0; 413 if (++i >= MAX_CLIENTS) 414 break; 415 } 416 fclose(f); 417 418 rereg_send_all(port, agent, dport, list, i); 419 rereg_recv_all(port, agent, dport, list, i); 420 421 rereg_query_all(port, agent, dport, list, i); 422 423 free(list); 424 return 0; 425} 426 427int main(int argc, char **argv) 428{ 429 char *guid_file = "port_guids.list"; 430 int mgmt_classes[2] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS}; 431 ib_portid_t dport_id; 432 int port, agent; 433 uint8_t *umad, *mad; 434 int len; 435 436 if (argc > 1) 437 guid_file = argv[1]; 438 439 madrpc_init(NULL, 0, mgmt_classes, 2); 440 441#if 1 442 ib_resolve_smlid(&dport_id, TMO); 443#else 444 memset(&dport_id, 0, sizeof(dport_id)); 445 dport_id.lid = 1; 446#endif 447 dport_id.qp = 1; 448 if (!dport_id.qkey) 449 dport_id.qkey = IB_DEFAULT_QP1_QKEY; 450 451 452 len = umad_size() + 256; 453 umad = calloc(1, len); 454 if (!umad) { 455 err("cannot alloc mem for umad: %s\n", strerror(errno)); 456 return -1; 457 } 458 459#if 1 460 port = madrpc_portid(); 461#else 462 ret = umad_init(); 463 464 port = umad_open_port(NULL, 0); 465 if (port < 0) { 466 err("umad_open_port failed: %s\n", strerror(errno)); 467 return port; 468 } 469#endif 470 471 agent = umad_register(port, IB_SA_CLASS, 2, 0, NULL); 472 473#if 0 474 int cnt; 475 cnt = rereg_mcm_rec_send(port, agent, &dport_id, cnt); 476 477 rereg_recv_all(port, agent, &dport_id); 478#else 479 rereg_and_test_port(guid_file, port, agent, &dport_id, TMO); 480#endif 481 mad = umad_get_mad(umad); 482 483 free(umad); 484 umad_unregister(port, agent); 485 umad_close_port(port); 486 umad_done(); 487 488 return 0; 489} 490