1[PATCH 4/4] libibverbs: Update examples 2 3Since RDMAoE requires usage of GRH, update ibv_*_pinpong examples to accept 4GIDs. GIDs are given as an index to the local port's table and are exchanged 5between the client and the server through the socket connection. 6 7Signed-off-by: Eli Cohen <eli@mellanox.co.il> 8--- 9 examples/devinfo.c | 14 ++++++++ 10 examples/pingpong.c | 31 ++++++++++++++++++ 11 examples/pingpong.h | 4 ++ 12 examples/rc_pingpong.c | 79 +++++++++++++++++++++++++++++++++++----------- 13 examples/srq_pingpong.c | 72 ++++++++++++++++++++++++++++++++---------- 14 examples/uc_pingpong.c | 70 ++++++++++++++++++++++++++++++++--------- 15 examples/ud_pingpong.c | 69 ++++++++++++++++++++++++++++++----------- 16 7 files changed, 269 insertions(+), 70 deletions(-) 17 18Index: libibverbs/examples/devinfo.c 19=================================================================== 20--- libibverbs.orig/examples/devinfo.c 2010-02-08 15:04:24.369329000 +0200 21+++ libibverbs/examples/devinfo.c 2010-03-17 14:08:48.404754000 +0200 22@@ -184,6 +184,19 @@ static int print_all_port_gids(struct ib 23 return rc; 24 } 25 26+static const char *link_layer_str(uint8_t link_layer) 27+{ 28+ switch (link_layer) { 29+ case IBV_LINK_LAYER_UNSPECIFIED: 30+ case IBV_LINK_LAYER_INFINIBAND: 31+ return "IB"; 32+ case IBV_LINK_LAYER_ETHERNET: 33+ return "Ethernet"; 34+ default: 35+ return "Unknown"; 36+ } 37+} 38+ 39 static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) 40 { 41 struct ibv_context *ctx; 42@@ -284,6 +297,7 @@ static int print_hca_cap(struct ibv_devi 43 printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid); 44 printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid); 45 printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc); 46+ printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer)); 47 48 if (verbose) { 49 printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz); 50Index: libibverbs/examples/pingpong.c 51=================================================================== 52--- libibverbs.orig/examples/pingpong.c 2010-02-08 15:04:24.372330000 +0200 53+++ libibverbs/examples/pingpong.c 2010-03-17 14:08:48.433754000 +0200 54@@ -31,6 +31,10 @@ 55 */ 56 57 #include "pingpong.h" 58+#include <arpa/inet.h> 59+#include <stdlib.h> 60+#include <stdio.h> 61+#include <string.h> 62 63 enum ibv_mtu pp_mtu_to_enum(int mtu) 64 { 65@@ -53,3 +57,30 @@ uint16_t pp_get_local_lid(struct ibv_con 66 67 return attr.lid; 68 } 69+ 70+int pp_get_port_info(struct ibv_context *context, int port, 71+ struct ibv_port_attr *attr) 72+{ 73+ return ibv_query_port(context, port, attr); 74+} 75+ 76+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid) 77+{ 78+ char tmp[9]; 79+ uint32_t v32; 80+ int i; 81+ 82+ for (tmp[8] = 0, i = 0; i < 4; ++i) { 83+ memcpy(tmp, wgid + i * 8, 8); 84+ sscanf(tmp, "%x", &v32); 85+ *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); 86+ } 87+} 88+ 89+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]) 90+{ 91+ int i; 92+ 93+ for (i = 0; i < 4; ++i) 94+ sprintf(&wgid[i * 8], "%08x", htonl(*(uint32_t *)(gid->raw + i * 4))); 95+} 96Index: libibverbs/examples/pingpong.h 97=================================================================== 98--- libibverbs.orig/examples/pingpong.h 2010-02-08 15:04:24.375328000 +0200 99+++ libibverbs/examples/pingpong.h 2010-03-17 14:08:48.443756000 +0200 100@@ -37,5 +37,9 @@ 101 102 enum ibv_mtu pp_mtu_to_enum(int mtu); 103 uint16_t pp_get_local_lid(struct ibv_context *context, int port); 104+int pp_get_port_info(struct ibv_context *context, int port, 105+ struct ibv_port_attr *attr); 106+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid); 107+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]); 108 109 #endif /* IBV_PINGPONG_H */ 110Index: libibverbs/examples/rc_pingpong.c 111=================================================================== 112--- libibverbs.orig/examples/rc_pingpong.c 2010-02-08 15:04:24.378329000 +0200 113+++ libibverbs/examples/rc_pingpong.c 2010-03-17 14:11:16.145313000 +0200 114@@ -67,17 +67,19 @@ struct pingpong_context { 115 int size; 116 int rx_depth; 117 int pending; 118+ struct ibv_port_attr portinfo; 119 }; 120 121 struct pingpong_dest { 122 int lid; 123 int qpn; 124 int psn; 125+ union ibv_gid gid; 126 }; 127 128 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 129 enum ibv_mtu mtu, int sl, 130- struct pingpong_dest *dest) 131+ struct pingpong_dest *dest, int sgid_idx) 132 { 133 struct ibv_qp_attr attr = { 134 .qp_state = IBV_QPS_RTR, 135@@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpon 136 .port_num = port 137 } 138 }; 139+ 140+ if (dest->gid.global.interface_id) { 141+ attr.ah_attr.is_global = 1; 142+ attr.ah_attr.grh.hop_limit = 1; 143+ attr.ah_attr.grh.dgid = dest->gid; 144+ attr.ah_attr.grh.sgid_index = sgid_idx; 145+ } 146 if (ibv_modify_qp(ctx->qp, &attr, 147 IBV_QP_STATE | 148 IBV_QP_AV | 149@@ -135,10 +144,11 @@ static struct pingpong_dest *pp_client_e 150 .ai_socktype = SOCK_STREAM 151 }; 152 char *service; 153- char msg[sizeof "0000:000000:000000"]; 154+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 155 int n; 156 int sockfd = -1; 157 struct pingpong_dest *rem_dest = NULL; 158+ char gid[33]; 159 160 if (asprintf(&service, "%d", port) < 0) 161 return NULL; 162@@ -169,7 +179,8 @@ static struct pingpong_dest *pp_client_e 163 return NULL; 164 } 165 166- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 167+ gid_to_wire_gid(&my_dest->gid, gid); 168+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 169 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 170 fprintf(stderr, "Couldn't send local address\n"); 171 goto out; 172@@ -187,7 +198,8 @@ static struct pingpong_dest *pp_client_e 173 if (!rem_dest) 174 goto out; 175 176- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 177+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 178+ wire_gid_to_gid(gid, &rem_dest->gid); 179 180 out: 181 close(sockfd); 182@@ -197,7 +209,8 @@ out: 183 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 184 int ib_port, enum ibv_mtu mtu, 185 int port, int sl, 186- const struct pingpong_dest *my_dest) 187+ const struct pingpong_dest *my_dest, 188+ int sgid_idx) 189 { 190 struct addrinfo *res, *t; 191 struct addrinfo hints = { 192@@ -206,10 +219,11 @@ static struct pingpong_dest *pp_server_e 193 .ai_socktype = SOCK_STREAM 194 }; 195 char *service; 196- char msg[sizeof "0000:000000:000000"]; 197+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 198 int n; 199 int sockfd = -1, connfd; 200 struct pingpong_dest *rem_dest = NULL; 201+ char gid[33]; 202 203 if (asprintf(&service, "%d", port) < 0) 204 return NULL; 205@@ -263,16 +277,19 @@ static struct pingpong_dest *pp_server_e 206 if (!rem_dest) 207 goto out; 208 209- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 210+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 211+ wire_gid_to_gid(gid, &rem_dest->gid); 212 213- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { 214+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 215 fprintf(stderr, "Couldn't connect to remote QP\n"); 216 free(rem_dest); 217 rem_dest = NULL; 218 goto out; 219 } 220 221- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 222+ 223+ gid_to_wire_gid(&my_dest->gid, gid); 224+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 225 if (write(connfd, msg, sizeof msg) != sizeof msg) { 226 fprintf(stderr, "Couldn't send local address\n"); 227 free(rem_dest); 228@@ -289,11 +306,11 @@ out: 229 230 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 231 int rx_depth, int port, 232- int use_event) 233+ int use_event, int is_server) 234 { 235 struct pingpong_context *ctx; 236 237- ctx = malloc(sizeof *ctx); 238+ ctx = calloc(1, sizeof *ctx); 239 if (!ctx) 240 return NULL; 241 242@@ -306,7 +323,7 @@ static struct pingpong_context *pp_init_ 243 return NULL; 244 } 245 246- memset(ctx->buf, 0, size); 247+ memset(ctx->buf, 0x7b + is_server, size); 248 249 ctx->context = ibv_open_device(ib_dev); 250 if (!ctx->context) { 251@@ -481,6 +498,7 @@ static void usage(const char *argv0) 252 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 253 printf(" -l, --sl=<sl> service level value\n"); 254 printf(" -e, --events sleep on CQ events (default poll)\n"); 255+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 256 } 257 258 int main(int argc, char *argv[]) 259@@ -504,6 +522,8 @@ int main(int argc, char *argv[]) 260 int rcnt, scnt; 261 int num_cq_events = 0; 262 int sl = 0; 263+ int gidx = -1; 264+ char gid[33]; 265 266 srand48(getpid() * time(NULL)); 267 268@@ -520,10 +540,11 @@ int main(int argc, char *argv[]) 269 { .name = "iters", .has_arg = 1, .val = 'n' }, 270 { .name = "sl", .has_arg = 1, .val = 'l' }, 271 { .name = "events", .has_arg = 0, .val = 'e' }, 272+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 273 { 0 } 274 }; 275 276- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); 277+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 278 if (c == -1) 279 break; 280 281@@ -576,6 +597,10 @@ int main(int argc, char *argv[]) 282 ++use_event; 283 break; 284 285+ case 'g': 286+ gidx = strtol(optarg, NULL, 0); 287+ break; 288+ 289 default: 290 usage(argv[0]); 291 return 1; 292@@ -615,7 +640,7 @@ int main(int argc, char *argv[]) 293 } 294 } 295 296- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 297+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername); 298 if (!ctx) 299 return 1; 300 301@@ -631,30 +656,47 @@ int main(int argc, char *argv[]) 302 return 1; 303 } 304 305- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 306- my_dest.qpn = ctx->qp->qp_num; 307- my_dest.psn = lrand48() & 0xffffff; 308- if (!my_dest.lid) { 309+ 310+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 311+ fprintf(stderr, "Couldn't get port info\n"); 312+ return 1; 313+ } 314+ 315+ my_dest.lid = ctx->portinfo.lid; 316+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 317 fprintf(stderr, "Couldn't get local LID\n"); 318 return 1; 319 } 320 321- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 322- my_dest.lid, my_dest.qpn, my_dest.psn); 323+ if (gidx >= 0) { 324+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 325+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 326+ return 1; 327+ } 328+ } else 329+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 330+ 331+ my_dest.qpn = ctx->qp->qp_num; 332+ my_dest.psn = lrand48() & 0xffffff; 333+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 334+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 335+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 336+ 337 338 if (servername) 339 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 340 else 341- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); 342+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 343 344 if (!rem_dest) 345 return 1; 346 347- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 348- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 349+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 350+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 351+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 352 353 if (servername) 354- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) 355+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 356 return 1; 357 358 ctx->pending = PINGPONG_RECV_WRID; 359@@ -706,6 +748,7 @@ int main(int argc, char *argv[]) 360 fprintf(stderr, "poll CQ failed %d\n", ne); 361 return 1; 362 } 363+ 364 } while (!use_event && ne < 1); 365 366 for (i = 0; i < ne; ++i) { 367Index: libibverbs/examples/srq_pingpong.c 368=================================================================== 369--- libibverbs.orig/examples/srq_pingpong.c 2010-02-08 15:04:24.382329000 +0200 370+++ libibverbs/examples/srq_pingpong.c 2010-03-17 14:13:22.332220000 +0200 371@@ -71,17 +71,19 @@ struct pingpong_context { 372 int num_qp; 373 int rx_depth; 374 int pending[MAX_QP]; 375+ struct ibv_port_attr portinfo; 376 }; 377 378 struct pingpong_dest { 379 int lid; 380 int qpn; 381 int psn; 382+ union ibv_gid gid; 383 }; 384 385 static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu, 386 int sl, const struct pingpong_dest *my_dest, 387- const struct pingpong_dest *dest) 388+ const struct pingpong_dest *dest, int sgid_idx) 389 { 390 int i; 391 392@@ -101,6 +103,13 @@ static int pp_connect_ctx(struct pingpon 393 .port_num = port 394 } 395 }; 396+ 397+ if (dest->gid.global.interface_id) { 398+ attr.ah_attr.is_global = 1; 399+ attr.ah_attr.grh.hop_limit = 1; 400+ attr.ah_attr.grh.dgid = dest->gid; 401+ attr.ah_attr.grh.sgid_index = sgid_idx; 402+ } 403 if (ibv_modify_qp(ctx->qp[i], &attr, 404 IBV_QP_STATE | 405 IBV_QP_AV | 406@@ -143,12 +152,13 @@ static struct pingpong_dest *pp_client_e 407 .ai_socktype = SOCK_STREAM 408 }; 409 char *service; 410- char msg[sizeof "0000:000000:000000"]; 411+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 412 int n; 413 int r; 414 int i; 415 int sockfd = -1; 416 struct pingpong_dest *rem_dest = NULL; 417+ char gid[33]; 418 419 if (asprintf(&service, "%d", port) < 0) 420 return NULL; 421@@ -180,7 +190,8 @@ static struct pingpong_dest *pp_client_e 422 } 423 424 for (i = 0; i < MAX_QP; ++i) { 425- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 426+ gid_to_wire_gid(&my_dest[i].gid, gid); 427+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 428 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 429 fprintf(stderr, "Couldn't send local address\n"); 430 goto out; 431@@ -204,8 +215,9 @@ static struct pingpong_dest *pp_client_e 432 n += r; 433 } 434 435- sscanf(msg, "%x:%x:%x", 436- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); 437+ sscanf(msg, "%x:%x:%x:%s", 438+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); 439+ wire_gid_to_gid(gid, &rem_dest[i].gid); 440 } 441 442 write(sockfd, "done", sizeof "done"); 443@@ -218,7 +230,8 @@ out: 444 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 445 int ib_port, enum ibv_mtu mtu, 446 int port, int sl, 447- const struct pingpong_dest *my_dest) 448+ const struct pingpong_dest *my_dest, 449+ int sgid_idx) 450 { 451 struct addrinfo *res, *t; 452 struct addrinfo hints = { 453@@ -227,12 +240,13 @@ static struct pingpong_dest *pp_server_e 454 .ai_socktype = SOCK_STREAM 455 }; 456 char *service; 457- char msg[sizeof "0000:000000:000000"]; 458+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 459 int n; 460 int r; 461 int i; 462 int sockfd = -1, connfd; 463 struct pingpong_dest *rem_dest = NULL; 464+ char gid[33]; 465 466 if (asprintf(&service, "%d", port) < 0) 467 return NULL; 468@@ -292,11 +306,12 @@ static struct pingpong_dest *pp_server_e 469 n += r; 470 } 471 472- sscanf(msg, "%x:%x:%x", 473- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); 474+ sscanf(msg, "%x:%x:%x:%s", 475+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); 476+ wire_gid_to_gid(gid, &rem_dest[i].gid); 477 } 478 479- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) { 480+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, sgid_idx)) { 481 fprintf(stderr, "Couldn't connect to remote QP\n"); 482 free(rem_dest); 483 rem_dest = NULL; 484@@ -304,7 +319,8 @@ static struct pingpong_dest *pp_server_e 485 } 486 487 for (i = 0; i < MAX_QP; ++i) { 488- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 489+ gid_to_wire_gid(&my_dest[i].gid, gid); 490+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 491 if (write(connfd, msg, sizeof msg) != sizeof msg) { 492 fprintf(stderr, "Couldn't send local address\n"); 493 free(rem_dest); 494@@ -327,7 +343,7 @@ static struct pingpong_context *pp_init_ 495 struct pingpong_context *ctx; 496 int i; 497 498- ctx = malloc(sizeof *ctx); 499+ ctx = calloc(1, sizeof *ctx); 500 if (!ctx) 501 return NULL; 502 503@@ -551,6 +567,7 @@ static void usage(const char *argv0) 504 printf(" -n, --iters=<iters> number of exchanges per QP(default 1000)\n"); 505 printf(" -l, --sl=<sl> service level value\n"); 506 printf(" -e, --events sleep on CQ events (default poll)\n"); 507+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 508 } 509 510 int main(int argc, char *argv[]) 511@@ -578,6 +595,8 @@ int main(int argc, char *argv[]) 512 int i; 513 int num_cq_events = 0; 514 int sl = 0; 515+ int gidx = -1; 516+ char gid[33]; 517 518 srand48(getpid() * time(NULL)); 519 520@@ -595,10 +614,11 @@ int main(int argc, char *argv[]) 521 { .name = "iters", .has_arg = 1, .val = 'n' }, 522 { .name = "sl", .has_arg = 1, .val = 'l' }, 523 { .name = "events", .has_arg = 0, .val = 'e' }, 524+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 525 { 0 } 526 }; 527 528- c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:e", long_options, NULL); 529+ c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", long_options, NULL); 530 if (c == -1) 531 break; 532 533@@ -655,6 +675,10 @@ int main(int argc, char *argv[]) 534 ++use_event; 535 break; 536 537+ case 'g': 538+ gidx = strtol(optarg, NULL, 0); 539+ break; 540+ 541 default: 542 usage(argv[0]); 543 return 1; 544@@ -722,33 +746,50 @@ int main(int argc, char *argv[]) 545 546 memset(my_dest, 0, sizeof my_dest); 547 548+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 549+ fprintf(stderr, "Couldn't get port info\n"); 550+ return 1; 551+ } 552 for (i = 0; i < num_qp; ++i) { 553 my_dest[i].qpn = ctx->qp[i]->qp_num; 554 my_dest[i].psn = lrand48() & 0xffffff; 555- my_dest[i].lid = pp_get_local_lid(ctx->context, ib_port); 556- if (!my_dest[i].lid) { 557+ my_dest[i].lid = ctx->portinfo.lid; 558+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest[i].lid) { 559 fprintf(stderr, "Couldn't get local LID\n"); 560 return 1; 561 } 562 563- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 564- my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 565+ if (gidx >= 0) { 566+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest[i].gid)) { 567+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 568+ return 1; 569+ } 570+ } else 571+ memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid); 572+ 573+ inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid); 574+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 575+ my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 576 } 577 578 if (servername) 579 rem_dest = pp_client_exch_dest(servername, port, my_dest); 580 else 581- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest); 582+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest, gidx); 583 584 if (!rem_dest) 585 return 1; 586 587- for (i = 0; i < num_qp; ++i) 588- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 589- rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn); 590+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 591+ 592+ for (i = 0; i < num_qp; ++i) { 593+ inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid); 594+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 595+ rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, gid); 596+ } 597 598 if (servername) 599- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) 600+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, gidx)) 601 return 1; 602 603 if (servername) 604Index: libibverbs/examples/uc_pingpong.c 605=================================================================== 606--- libibverbs.orig/examples/uc_pingpong.c 2010-02-08 15:04:24.386328000 +0200 607+++ libibverbs/examples/uc_pingpong.c 2010-03-17 14:14:23.573114000 +0200 608@@ -67,17 +67,19 @@ struct pingpong_context { 609 int size; 610 int rx_depth; 611 int pending; 612+ struct ibv_port_attr portinfo; 613 }; 614 615 struct pingpong_dest { 616 int lid; 617 int qpn; 618 int psn; 619+ union ibv_gid gid; 620 }; 621 622 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 623 enum ibv_mtu mtu, int sl, 624- struct pingpong_dest *dest) 625+ struct pingpong_dest *dest, int sgid_idx) 626 { 627 struct ibv_qp_attr attr = { 628 .qp_state = IBV_QPS_RTR, 629@@ -92,6 +94,14 @@ static int pp_connect_ctx(struct pingpon 630 .port_num = port 631 } 632 }; 633+ 634+ if (dest->gid.global.interface_id) { 635+ attr.ah_attr.is_global = 1; 636+ attr.ah_attr.grh.hop_limit = 1; 637+ attr.ah_attr.grh.dgid = dest->gid; 638+ attr.ah_attr.grh.sgid_index = sgid_idx; 639+ } 640+ 641 if (ibv_modify_qp(ctx->qp, &attr, 642 IBV_QP_STATE | 643 IBV_QP_AV | 644@@ -123,10 +133,11 @@ static struct pingpong_dest *pp_client_e 645 .ai_socktype = SOCK_STREAM 646 }; 647 char *service; 648- char msg[sizeof "0000:000000:000000"]; 649+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 650 int n; 651 int sockfd = -1; 652 struct pingpong_dest *rem_dest = NULL; 653+ char gid[33]; 654 655 if (asprintf(&service, "%d", port) < 0) 656 return NULL; 657@@ -157,7 +168,8 @@ static struct pingpong_dest *pp_client_e 658 return NULL; 659 } 660 661- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 662+ gid_to_wire_gid(&my_dest->gid, gid); 663+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 664 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 665 fprintf(stderr, "Couldn't send local address\n"); 666 goto out; 667@@ -175,7 +187,8 @@ static struct pingpong_dest *pp_client_e 668 if (!rem_dest) 669 goto out; 670 671- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 672+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 673+ wire_gid_to_gid(gid, &rem_dest->gid); 674 675 out: 676 close(sockfd); 677@@ -185,7 +198,8 @@ out: 678 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 679 int ib_port, enum ibv_mtu mtu, 680 int port, int sl, 681- const struct pingpong_dest *my_dest) 682+ const struct pingpong_dest *my_dest, 683+ int sgid_idx) 684 { 685 struct addrinfo *res, *t; 686 struct addrinfo hints = { 687@@ -194,10 +208,11 @@ static struct pingpong_dest *pp_server_e 688 .ai_socktype = SOCK_STREAM 689 }; 690 char *service; 691- char msg[sizeof "0000:000000:000000"]; 692+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 693 int n; 694 int sockfd = -1, connfd; 695 struct pingpong_dest *rem_dest = NULL; 696+ char gid[33]; 697 698 if (asprintf(&service, "%d", port) < 0) 699 return NULL; 700@@ -251,16 +266,18 @@ static struct pingpong_dest *pp_server_e 701 if (!rem_dest) 702 goto out; 703 704- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 705+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 706+ wire_gid_to_gid(gid, &rem_dest->gid); 707 708- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { 709+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 710 fprintf(stderr, "Couldn't connect to remote QP\n"); 711 free(rem_dest); 712 rem_dest = NULL; 713 goto out; 714 } 715 716- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 717+ gid_to_wire_gid(&my_dest->gid, gid); 718+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 719 if (write(connfd, msg, sizeof msg) != sizeof msg) { 720 fprintf(stderr, "Couldn't send local address\n"); 721 free(rem_dest); 722@@ -281,7 +298,7 @@ static struct pingpong_context *pp_init_ 723 { 724 struct pingpong_context *ctx; 725 726- ctx = malloc(sizeof *ctx); 727+ ctx = calloc(1, sizeof *ctx); 728 if (!ctx) 729 return NULL; 730 731@@ -469,6 +486,7 @@ static void usage(const char *argv0) 732 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 733 printf(" -l, --sl=<sl> service level value\n"); 734 printf(" -e, --events sleep on CQ events (default poll)\n"); 735+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 736 } 737 738 int main(int argc, char *argv[]) 739@@ -492,6 +510,8 @@ int main(int argc, char *argv[]) 740 int rcnt, scnt; 741 int num_cq_events = 0; 742 int sl = 0; 743+ int gidx = -1; 744+ char gid[33]; 745 746 srand48(getpid() * time(NULL)); 747 748@@ -508,10 +528,11 @@ int main(int argc, char *argv[]) 749 { .name = "iters", .has_arg = 1, .val = 'n' }, 750 { .name = "sl", .has_arg = 1, .val = 'l' }, 751 { .name = "events", .has_arg = 0, .val = 'e' }, 752+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 753 { 0 } 754 }; 755 756- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); 757+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 758 if (c == -1) 759 break; 760 761@@ -564,6 +585,10 @@ int main(int argc, char *argv[]) 762 ++use_event; 763 break; 764 765+ case 'g': 766+ gidx = strtol(optarg, NULL, 0); 767+ break; 768+ 769 default: 770 usage(argv[0]); 771 return 1; 772@@ -619,30 +644,45 @@ int main(int argc, char *argv[]) 773 return 1; 774 } 775 776- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 777- my_dest.qpn = ctx->qp->qp_num; 778- my_dest.psn = lrand48() & 0xffffff; 779- if (!my_dest.lid) { 780+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 781+ fprintf(stderr, "Couldn't get port info\n"); 782+ return 1; 783+ } 784+ 785+ my_dest.lid = ctx->portinfo.lid; 786+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 787 fprintf(stderr, "Couldn't get local LID\n"); 788 return 1; 789 } 790 791- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 792- my_dest.lid, my_dest.qpn, my_dest.psn); 793+ if (gidx >= 0) { 794+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 795+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 796+ return 1; 797+ } 798+ } else 799+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 800+ 801+ my_dest.qpn = ctx->qp->qp_num; 802+ my_dest.psn = lrand48() & 0xffffff; 803+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 804+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 805+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 806 807 if (servername) 808 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 809 else 810- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); 811+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 812 813 if (!rem_dest) 814 return 1; 815 816- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 817- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 818+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 819+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 820+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 821 822 if (servername) 823- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) 824+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 825 return 1; 826 827 ctx->pending = PINGPONG_RECV_WRID; 828Index: libibverbs/examples/ud_pingpong.c 829=================================================================== 830--- libibverbs.orig/examples/ud_pingpong.c 2010-02-08 15:04:24.389329000 +0200 831+++ libibverbs/examples/ud_pingpong.c 2010-03-17 14:08:48.502754000 +0200 832@@ -68,16 +68,18 @@ struct pingpong_context { 833 int size; 834 int rx_depth; 835 int pending; 836+ struct ibv_port_attr portinfo; 837 }; 838 839 struct pingpong_dest { 840 int lid; 841 int qpn; 842 int psn; 843+ union ibv_gid gid; 844 }; 845 846 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 847- int sl, struct pingpong_dest *dest) 848+ int sl, struct pingpong_dest *dest, int sgid_idx) 849 { 850 struct ibv_ah_attr ah_attr = { 851 .is_global = 0, 852@@ -105,6 +107,13 @@ static int pp_connect_ctx(struct pingpon 853 return 1; 854 } 855 856+ if (dest->gid.global.interface_id) { 857+ ah_attr.is_global = 1; 858+ ah_attr.grh.hop_limit = 1; 859+ ah_attr.grh.dgid = dest->gid; 860+ ah_attr.grh.sgid_index = sgid_idx; 861+ } 862+ 863 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); 864 if (!ctx->ah) { 865 fprintf(stderr, "Failed to create AH\n"); 866@@ -123,10 +132,11 @@ static struct pingpong_dest *pp_client_e 867 .ai_socktype = SOCK_STREAM 868 }; 869 char *service; 870- char msg[sizeof "0000:000000:000000"]; 871+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 872 int n; 873 int sockfd = -1; 874 struct pingpong_dest *rem_dest = NULL; 875+ char gid[33]; 876 877 if (asprintf(&service, "%d", port) < 0) 878 return NULL; 879@@ -157,7 +167,8 @@ static struct pingpong_dest *pp_client_e 880 return NULL; 881 } 882 883- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 884+ gid_to_wire_gid(&my_dest->gid, gid); 885+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 886 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 887 fprintf(stderr, "Couldn't send local address\n"); 888 goto out; 889@@ -175,7 +186,8 @@ static struct pingpong_dest *pp_client_e 890 if (!rem_dest) 891 goto out; 892 893- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 894+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 895+ wire_gid_to_gid(gid, &rem_dest->gid); 896 897 out: 898 close(sockfd); 899@@ -184,7 +196,8 @@ out: 900 901 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 902 int ib_port, int port, int sl, 903- const struct pingpong_dest *my_dest) 904+ const struct pingpong_dest *my_dest, 905+ int sgid_idx) 906 { 907 struct addrinfo *res, *t; 908 struct addrinfo hints = { 909@@ -193,10 +206,11 @@ static struct pingpong_dest *pp_server_e 910 .ai_socktype = SOCK_STREAM 911 }; 912 char *service; 913- char msg[sizeof "0000:000000:000000"]; 914+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 915 int n; 916 int sockfd = -1, connfd; 917 struct pingpong_dest *rem_dest = NULL; 918+ char gid[33]; 919 920 if (asprintf(&service, "%d", port) < 0) 921 return NULL; 922@@ -250,16 +264,18 @@ static struct pingpong_dest *pp_server_e 923 if (!rem_dest) 924 goto out; 925 926- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 927+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 928+ wire_gid_to_gid(gid, &rem_dest->gid); 929 930- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest)) { 931+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { 932 fprintf(stderr, "Couldn't connect to remote QP\n"); 933 free(rem_dest); 934 rem_dest = NULL; 935 goto out; 936 } 937 938- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 939+ gid_to_wire_gid(&my_dest->gid, gid); 940+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 941 if (write(connfd, msg, sizeof msg) != sizeof msg) { 942 fprintf(stderr, "Couldn't send local address\n"); 943 free(rem_dest); 944@@ -474,10 +490,11 @@ static void usage(const char *argv0) 945 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 946 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 947 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 948- printf(" -s, --size=<size> size of message to exchange (default 2048)\n"); 949+ printf(" -s, --size=<size> size of message to exchange (default 1024)\n"); 950 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 951 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 952 printf(" -e, --events sleep on CQ events (default poll)\n"); 953+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 954 } 955 956 int main(int argc, char *argv[]) 957@@ -492,7 +509,7 @@ int main(int argc, char *argv[]) 958 char *servername = NULL; 959 int port = 18515; 960 int ib_port = 1; 961- int size = 2048; 962+ int size = 1024; 963 int rx_depth = 500; 964 int iters = 1000; 965 int use_event = 0; 966@@ -500,6 +517,8 @@ int main(int argc, char *argv[]) 967 int rcnt, scnt; 968 int num_cq_events = 0; 969 int sl = 0; 970+ int gidx = -1; 971+ char gid[33]; 972 973 srand48(getpid() * time(NULL)); 974 975@@ -515,10 +534,11 @@ int main(int argc, char *argv[]) 976 { .name = "iters", .has_arg = 1, .val = 'n' }, 977 { .name = "sl", .has_arg = 1, .val = 'l' }, 978 { .name = "events", .has_arg = 0, .val = 'e' }, 979+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 980 { 0 } 981 }; 982 983- c = getopt_long(argc, argv, "p:d:i:s:r:n:l:e", long_options, NULL); 984+ c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL); 985 if (c == -1) 986 break; 987 988@@ -563,6 +583,10 @@ int main(int argc, char *argv[]) 989 ++use_event; 990 break; 991 992+ case 'g': 993+ gidx = strtol(optarg, NULL, 0); 994+ break; 995+ 996 default: 997 usage(argv[0]); 998 return 1; 999@@ -618,30 +642,41 @@ int main(int argc, char *argv[]) 1000 return 1; 1001 } 1002 1003- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 1004- my_dest.qpn = ctx->qp->qp_num; 1005- my_dest.psn = lrand48() & 0xffffff; 1006- if (!my_dest.lid) { 1007- fprintf(stderr, "Couldn't get local LID\n"); 1008+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 1009+ fprintf(stderr, "Couldn't get port info\n"); 1010 return 1; 1011 } 1012+ my_dest.lid = ctx->portinfo.lid; 1013+ 1014+ my_dest.qpn = ctx->qp->qp_num; 1015+ my_dest.psn = lrand48() & 0xffffff; 1016+ 1017+ if (gidx >= 0) { 1018+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 1019+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 1020+ return 1; 1021+ } 1022+ } else 1023+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 1024 1025- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 1026- my_dest.lid, my_dest.qpn, my_dest.psn); 1027+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 1028+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", 1029+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 1030 1031 if (servername) 1032 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 1033 else 1034- rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest); 1035+ rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx); 1036 1037 if (!rem_dest) 1038 return 1; 1039 1040- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 1041- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 1042+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 1043+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 1044+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 1045 1046 if (servername) 1047- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest)) 1048+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) 1049 return 1; 1050 1051 ctx->pending = PINGPONG_RECV_WRID; 1052