1276238Shselasky[PATCH 4/4] libibverbs: Update examples 2276238Shselasky 3276238ShselaskySince RDMAoE requires usage of GRH, update ibv_*_pinpong examples to accept 4276238ShselaskyGIDs. GIDs are given as an index to the local port's table and are exchanged 5276238Shselaskybetween the client and the server through the socket connection. 6276238Shselasky 7276238ShselaskySigned-off-by: Eli Cohen <eli@mellanox.co.il> 8276238Shselasky--- 9276238Shselasky examples/devinfo.c | 14 ++++++++ 10 examples/pingpong.c | 31 ++++++++++++++++++ 11 examples/pingpong.h | 4 ++ 12 examples/rc_pingpong.c | 79 +++++++++++++++++++++++++++++++++++----------- 13 examples/srq_pingpong.c | 72 ++++++++++++++++++++++++++++++++---------- 14 examples/uc_pingpong.c | 70 ++++++++++++++++++++++++++++++++--------- 15 examples/ud_pingpong.c | 69 ++++++++++++++++++++++++++++++----------- 16 7 files changed, 269 insertions(+), 70 deletions(-) 17 18Index: libibverbs/examples/devinfo.c 19=================================================================== 20--- libibverbs.orig/examples/devinfo.c 2010-02-08 15:04:24.369329000 +0200 21+++ libibverbs/examples/devinfo.c 2010-03-17 14:08:48.404754000 +0200 22@@ -184,6 +184,19 @@ static int print_all_port_gids(struct ib 23 return rc; 24 } 25 26+static const char *link_layer_str(uint8_t link_layer) 27+{ 28+ switch (link_layer) { 29+ case IBV_LINK_LAYER_UNSPECIFIED: 30+ case IBV_LINK_LAYER_INFINIBAND: 31+ return "IB"; 32+ case IBV_LINK_LAYER_ETHERNET: 33+ return "Ethernet"; 34+ default: 35+ return "Unknown"; 36+ } 37+} 38+ 39 static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) 40 { 41 struct ibv_context *ctx; 42@@ -284,6 +297,7 @@ static int print_hca_cap(struct ibv_devi 43 printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid); 44 printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid); 45 printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc); 46+ printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer)); 47 48 if (verbose) { 49 printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz); 50Index: libibverbs/examples/pingpong.c 51=================================================================== 52--- libibverbs.orig/examples/pingpong.c 2010-02-08 15:04:24.372330000 +0200 53+++ libibverbs/examples/pingpong.c 2010-03-17 14:08:48.433754000 +0200 54@@ -31,6 +31,10 @@ 55 */ 56 57 #include "pingpong.h" 58+#include <arpa/inet.h> 59+#include <stdlib.h> 60+#include <stdio.h> 61+#include <string.h> 62 63 enum ibv_mtu pp_mtu_to_enum(int mtu) 64 { 65@@ -53,3 +57,30 @@ uint16_t pp_get_local_lid(struct ibv_con 66 67 return attr.lid; 68 } 69+ 70+int pp_get_port_info(struct ibv_context *context, int port, 71+ struct ibv_port_attr *attr) 72+{ 73+ return ibv_query_port(context, port, attr); 74+} 75+ 76+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid) 77+{ 78+ char tmp[9]; 79+ uint32_t v32; 80+ int i; 81+ 82+ for (tmp[8] = 0, i = 0; i < 4; ++i) { 83+ memcpy(tmp, wgid + i * 8, 8); 84+ sscanf(tmp, "%x", &v32); 85+ *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); 86+ } 87+} 88+ 89+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]) 90+{ 91+ int i; 92+ 93+ for (i = 0; i < 4; ++i) 94+ sprintf(&wgid[i * 8], "%08x", htonl(*(uint32_t *)(gid->raw + i * 4))); 95+} 96Index: libibverbs/examples/pingpong.h 97=================================================================== 98--- libibverbs.orig/examples/pingpong.h 2010-02-08 15:04:24.375328000 +0200 99+++ libibverbs/examples/pingpong.h 2010-03-17 14:08:48.443756000 +0200 100@@ -37,5 +37,9 @@ 101 102 enum ibv_mtu pp_mtu_to_enum(int mtu); 103 uint16_t pp_get_local_lid(struct ibv_context *context, int port); 104+int pp_get_port_info(struct ibv_context *context, int port, 105+ struct ibv_port_attr *attr); 106+void wire_gid_to_gid(const char *wgid, union ibv_gid *gid); 107+void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]); 108 109 #endif /* IBV_PINGPONG_H */ 110Index: libibverbs/examples/rc_pingpong.c 111=================================================================== 112--- libibverbs.orig/examples/rc_pingpong.c 2010-02-08 15:04:24.378329000 +0200 113+++ libibverbs/examples/rc_pingpong.c 2010-03-17 14:11:16.145313000 +0200 114@@ -67,17 +67,19 @@ struct pingpong_context { 115 int size; 116 int rx_depth; 117 int pending; 118+ struct ibv_port_attr portinfo; 119 }; 120 121 struct pingpong_dest { 122 int lid; 123 int qpn; 124 int psn; 125+ union ibv_gid gid; 126 }; 127 128 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 129 enum ibv_mtu mtu, int sl, 130- struct pingpong_dest *dest) 131+ struct pingpong_dest *dest, int sgid_idx) 132 { 133 struct ibv_qp_attr attr = { 134 .qp_state = IBV_QPS_RTR, 135@@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpon 136 .port_num = port 137 } 138 }; 139+ 140+ if (dest->gid.global.interface_id) { 141+ attr.ah_attr.is_global = 1; 142+ attr.ah_attr.grh.hop_limit = 1; 143+ attr.ah_attr.grh.dgid = dest->gid; 144+ attr.ah_attr.grh.sgid_index = sgid_idx; 145+ } 146 if (ibv_modify_qp(ctx->qp, &attr, 147 IBV_QP_STATE | 148 IBV_QP_AV | 149@@ -135,10 +144,11 @@ static struct pingpong_dest *pp_client_e 150 .ai_socktype = SOCK_STREAM 151 }; 152 char *service; 153- char msg[sizeof "0000:000000:000000"]; 154+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 155 int n; 156 int sockfd = -1; 157 struct pingpong_dest *rem_dest = NULL; 158+ char gid[33]; 159 160 if (asprintf(&service, "%d", port) < 0) 161 return NULL; 162@@ -169,7 +179,8 @@ static struct pingpong_dest *pp_client_e 163 return NULL; 164 } 165 166- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 167+ gid_to_wire_gid(&my_dest->gid, gid); 168+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 169 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 170 fprintf(stderr, "Couldn't send local address\n"); 171 goto out; 172@@ -187,7 +198,8 @@ static struct pingpong_dest *pp_client_e 173 if (!rem_dest) 174 goto out; 175 176- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 177+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 178+ wire_gid_to_gid(gid, &rem_dest->gid); 179 180 out: 181 close(sockfd); 182@@ -197,7 +209,8 @@ out: 183 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 184 int ib_port, enum ibv_mtu mtu, 185 int port, int sl, 186- const struct pingpong_dest *my_dest) 187+ const struct pingpong_dest *my_dest, 188+ int sgid_idx) 189 { 190 struct addrinfo *res, *t; 191 struct addrinfo hints = { 192@@ -206,10 +219,11 @@ static struct pingpong_dest *pp_server_e 193 .ai_socktype = SOCK_STREAM 194 }; 195 char *service; 196- char msg[sizeof "0000:000000:000000"]; 197+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 198 int n; 199 int sockfd = -1, connfd; 200 struct pingpong_dest *rem_dest = NULL; 201+ char gid[33]; 202 203 if (asprintf(&service, "%d", port) < 0) 204 return NULL; 205@@ -263,16 +277,19 @@ static struct pingpong_dest *pp_server_e 206 if (!rem_dest) 207 goto out; 208 209- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 210+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 211+ wire_gid_to_gid(gid, &rem_dest->gid); 212 213- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { 214+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 215 fprintf(stderr, "Couldn't connect to remote QP\n"); 216 free(rem_dest); 217 rem_dest = NULL; 218 goto out; 219 } 220 221- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 222+ 223+ gid_to_wire_gid(&my_dest->gid, gid); 224+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 225 if (write(connfd, msg, sizeof msg) != sizeof msg) { 226 fprintf(stderr, "Couldn't send local address\n"); 227 free(rem_dest); 228@@ -289,11 +306,11 @@ out: 229 230 static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, 231 int rx_depth, int port, 232- int use_event) 233+ int use_event, int is_server) 234 { 235 struct pingpong_context *ctx; 236 237- ctx = malloc(sizeof *ctx); 238+ ctx = calloc(1, sizeof *ctx); 239 if (!ctx) 240 return NULL; 241 242@@ -306,7 +323,7 @@ static struct pingpong_context *pp_init_ 243 return NULL; 244 } 245 246- memset(ctx->buf, 0, size); 247+ memset(ctx->buf, 0x7b + is_server, size); 248 249 ctx->context = ibv_open_device(ib_dev); 250 if (!ctx->context) { 251@@ -481,6 +498,7 @@ static void usage(const char *argv0) 252 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 253 printf(" -l, --sl=<sl> service level value\n"); 254 printf(" -e, --events sleep on CQ events (default poll)\n"); 255+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 256 } 257 258 int main(int argc, char *argv[]) 259@@ -504,6 +522,8 @@ int main(int argc, char *argv[]) 260 int rcnt, scnt; 261 int num_cq_events = 0; 262 int sl = 0; 263+ int gidx = -1; 264+ char gid[33]; 265 266 srand48(getpid() * time(NULL)); 267 268@@ -520,10 +540,11 @@ int main(int argc, char *argv[]) 269 { .name = "iters", .has_arg = 1, .val = 'n' }, 270 { .name = "sl", .has_arg = 1, .val = 'l' }, 271 { .name = "events", .has_arg = 0, .val = 'e' }, 272+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 273 { 0 } 274 }; 275 276- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); 277+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 278 if (c == -1) 279 break; 280 281@@ -576,6 +597,10 @@ int main(int argc, char *argv[]) 282 ++use_event; 283 break; 284 285+ case 'g': 286+ gidx = strtol(optarg, NULL, 0); 287+ break; 288+ 289 default: 290 usage(argv[0]); 291 return 1; 292@@ -615,7 +640,7 @@ int main(int argc, char *argv[]) 293 } 294 } 295 296- ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); 297+ ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername); 298 if (!ctx) 299 return 1; 300 301@@ -631,30 +656,47 @@ int main(int argc, char *argv[]) 302 return 1; 303 } 304 305- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 306- my_dest.qpn = ctx->qp->qp_num; 307- my_dest.psn = lrand48() & 0xffffff; 308- if (!my_dest.lid) { 309+ 310+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 311+ fprintf(stderr, "Couldn't get port info\n"); 312+ return 1; 313+ } 314+ 315+ my_dest.lid = ctx->portinfo.lid; 316+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 317 fprintf(stderr, "Couldn't get local LID\n"); 318 return 1; 319 } 320 321- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 322- my_dest.lid, my_dest.qpn, my_dest.psn); 323+ if (gidx >= 0) { 324+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 325+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 326+ return 1; 327+ } 328+ } else 329+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 330+ 331+ my_dest.qpn = ctx->qp->qp_num; 332+ my_dest.psn = lrand48() & 0xffffff; 333+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 334+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 335+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 336+ 337 338 if (servername) 339 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 340 else 341- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); 342+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 343 344 if (!rem_dest) 345 return 1; 346 347- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 348- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 349+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 350+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 351+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 352 353 if (servername) 354- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) 355+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 356 return 1; 357 358 ctx->pending = PINGPONG_RECV_WRID; 359@@ -706,6 +748,7 @@ int main(int argc, char *argv[]) 360 fprintf(stderr, "poll CQ failed %d\n", ne); 361 return 1; 362 } 363+ 364 } while (!use_event && ne < 1); 365 366 for (i = 0; i < ne; ++i) { 367Index: libibverbs/examples/srq_pingpong.c 368=================================================================== 369--- libibverbs.orig/examples/srq_pingpong.c 2010-02-08 15:04:24.382329000 +0200 370+++ libibverbs/examples/srq_pingpong.c 2010-03-17 14:13:22.332220000 +0200 371@@ -71,17 +71,19 @@ struct pingpong_context { 372 int num_qp; 373 int rx_depth; 374 int pending[MAX_QP]; 375+ struct ibv_port_attr portinfo; 376 }; 377 378 struct pingpong_dest { 379 int lid; 380 int qpn; 381 int psn; 382+ union ibv_gid gid; 383 }; 384 385 static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu, 386 int sl, const struct pingpong_dest *my_dest, 387- const struct pingpong_dest *dest) 388+ const struct pingpong_dest *dest, int sgid_idx) 389 { 390 int i; 391 392@@ -101,6 +103,13 @@ static int pp_connect_ctx(struct pingpon 393 .port_num = port 394 } 395 }; 396+ 397+ if (dest->gid.global.interface_id) { 398+ attr.ah_attr.is_global = 1; 399+ attr.ah_attr.grh.hop_limit = 1; 400+ attr.ah_attr.grh.dgid = dest->gid; 401+ attr.ah_attr.grh.sgid_index = sgid_idx; 402+ } 403 if (ibv_modify_qp(ctx->qp[i], &attr, 404 IBV_QP_STATE | 405 IBV_QP_AV | 406@@ -143,12 +152,13 @@ static struct pingpong_dest *pp_client_e 407 .ai_socktype = SOCK_STREAM 408 }; 409 char *service; 410- char msg[sizeof "0000:000000:000000"]; 411+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 412 int n; 413 int r; 414 int i; 415 int sockfd = -1; 416 struct pingpong_dest *rem_dest = NULL; 417+ char gid[33]; 418 419 if (asprintf(&service, "%d", port) < 0) 420 return NULL; 421@@ -180,7 +190,8 @@ static struct pingpong_dest *pp_client_e 422 } 423 424 for (i = 0; i < MAX_QP; ++i) { 425- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 426+ gid_to_wire_gid(&my_dest[i].gid, gid); 427+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 428 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 429 fprintf(stderr, "Couldn't send local address\n"); 430 goto out; 431@@ -204,8 +215,9 @@ static struct pingpong_dest *pp_client_e 432 n += r; 433 } 434 435- sscanf(msg, "%x:%x:%x", 436- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); 437+ sscanf(msg, "%x:%x:%x:%s", 438+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); 439+ wire_gid_to_gid(gid, &rem_dest[i].gid); 440 } 441 442 write(sockfd, "done", sizeof "done"); 443@@ -218,7 +230,8 @@ out: 444 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 445 int ib_port, enum ibv_mtu mtu, 446 int port, int sl, 447- const struct pingpong_dest *my_dest) 448+ const struct pingpong_dest *my_dest, 449+ int sgid_idx) 450 { 451 struct addrinfo *res, *t; 452 struct addrinfo hints = { 453@@ -227,12 +240,13 @@ static struct pingpong_dest *pp_server_e 454 .ai_socktype = SOCK_STREAM 455 }; 456 char *service; 457- char msg[sizeof "0000:000000:000000"]; 458+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 459 int n; 460 int r; 461 int i; 462 int sockfd = -1, connfd; 463 struct pingpong_dest *rem_dest = NULL; 464+ char gid[33]; 465 466 if (asprintf(&service, "%d", port) < 0) 467 return NULL; 468@@ -292,11 +306,12 @@ static struct pingpong_dest *pp_server_e 469 n += r; 470 } 471 472- sscanf(msg, "%x:%x:%x", 473- &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); 474+ sscanf(msg, "%x:%x:%x:%s", 475+ &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); 476+ wire_gid_to_gid(gid, &rem_dest[i].gid); 477 } 478 479- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) { 480+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, sgid_idx)) { 481 fprintf(stderr, "Couldn't connect to remote QP\n"); 482 free(rem_dest); 483 rem_dest = NULL; 484@@ -304,7 +319,8 @@ static struct pingpong_dest *pp_server_e 485 } 486 487 for (i = 0; i < MAX_QP; ++i) { 488- sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 489+ gid_to_wire_gid(&my_dest[i].gid, gid); 490+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 491 if (write(connfd, msg, sizeof msg) != sizeof msg) { 492 fprintf(stderr, "Couldn't send local address\n"); 493 free(rem_dest); 494@@ -327,7 +343,7 @@ static struct pingpong_context *pp_init_ 495 struct pingpong_context *ctx; 496 int i; 497 498- ctx = malloc(sizeof *ctx); 499+ ctx = calloc(1, sizeof *ctx); 500 if (!ctx) 501 return NULL; 502 503@@ -551,6 +567,7 @@ static void usage(const char *argv0) 504 printf(" -n, --iters=<iters> number of exchanges per QP(default 1000)\n"); 505 printf(" -l, --sl=<sl> service level value\n"); 506 printf(" -e, --events sleep on CQ events (default poll)\n"); 507+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 508 } 509 510 int main(int argc, char *argv[]) 511@@ -578,6 +595,8 @@ int main(int argc, char *argv[]) 512 int i; 513 int num_cq_events = 0; 514 int sl = 0; 515+ int gidx = -1; 516+ char gid[33]; 517 518 srand48(getpid() * time(NULL)); 519 520@@ -595,10 +614,11 @@ int main(int argc, char *argv[]) 521 { .name = "iters", .has_arg = 1, .val = 'n' }, 522 { .name = "sl", .has_arg = 1, .val = 'l' }, 523 { .name = "events", .has_arg = 0, .val = 'e' }, 524+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 525 { 0 } 526 }; 527 528- c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:e", long_options, NULL); 529+ c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", long_options, NULL); 530 if (c == -1) 531 break; 532 533@@ -655,6 +675,10 @@ int main(int argc, char *argv[]) 534 ++use_event; 535 break; 536 537+ case 'g': 538+ gidx = strtol(optarg, NULL, 0); 539+ break; 540+ 541 default: 542 usage(argv[0]); 543 return 1; 544@@ -722,33 +746,50 @@ int main(int argc, char *argv[]) 545 546 memset(my_dest, 0, sizeof my_dest); 547 548+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 549+ fprintf(stderr, "Couldn't get port info\n"); 550+ return 1; 551+ } 552 for (i = 0; i < num_qp; ++i) { 553 my_dest[i].qpn = ctx->qp[i]->qp_num; 554 my_dest[i].psn = lrand48() & 0xffffff; 555- my_dest[i].lid = pp_get_local_lid(ctx->context, ib_port); 556- if (!my_dest[i].lid) { 557+ my_dest[i].lid = ctx->portinfo.lid; 558+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest[i].lid) { 559 fprintf(stderr, "Couldn't get local LID\n"); 560 return 1; 561 } 562 563- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 564- my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); 565+ if (gidx >= 0) { 566+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest[i].gid)) { 567+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 568+ return 1; 569+ } 570+ } else 571+ memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid); 572+ 573+ inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid); 574+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 575+ my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); 576 } 577 578 if (servername) 579 rem_dest = pp_client_exch_dest(servername, port, my_dest); 580 else 581- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest); 582+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest, gidx); 583 584 if (!rem_dest) 585 return 1; 586 587- for (i = 0; i < num_qp; ++i) 588- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 589- rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn); 590+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 591+ 592+ for (i = 0; i < num_qp; ++i) { 593+ inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid); 594+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 595+ rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, gid); 596+ } 597 598 if (servername) 599- if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) 600+ if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, gidx)) 601 return 1; 602 603 if (servername) 604Index: libibverbs/examples/uc_pingpong.c 605=================================================================== 606--- libibverbs.orig/examples/uc_pingpong.c 2010-02-08 15:04:24.386328000 +0200 607+++ libibverbs/examples/uc_pingpong.c 2010-03-17 14:14:23.573114000 +0200 608@@ -67,17 +67,19 @@ struct pingpong_context { 609 int size; 610 int rx_depth; 611 int pending; 612+ struct ibv_port_attr portinfo; 613 }; 614 615 struct pingpong_dest { 616 int lid; 617 int qpn; 618 int psn; 619+ union ibv_gid gid; 620 }; 621 622 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 623 enum ibv_mtu mtu, int sl, 624- struct pingpong_dest *dest) 625+ struct pingpong_dest *dest, int sgid_idx) 626 { 627 struct ibv_qp_attr attr = { 628 .qp_state = IBV_QPS_RTR, 629@@ -92,6 +94,14 @@ static int pp_connect_ctx(struct pingpon 630 .port_num = port 631 } 632 }; 633+ 634+ if (dest->gid.global.interface_id) { 635+ attr.ah_attr.is_global = 1; 636+ attr.ah_attr.grh.hop_limit = 1; 637+ attr.ah_attr.grh.dgid = dest->gid; 638+ attr.ah_attr.grh.sgid_index = sgid_idx; 639+ } 640+ 641 if (ibv_modify_qp(ctx->qp, &attr, 642 IBV_QP_STATE | 643 IBV_QP_AV | 644@@ -123,10 +133,11 @@ static struct pingpong_dest *pp_client_e 645 .ai_socktype = SOCK_STREAM 646 }; 647 char *service; 648- char msg[sizeof "0000:000000:000000"]; 649+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 650 int n; 651 int sockfd = -1; 652 struct pingpong_dest *rem_dest = NULL; 653+ char gid[33]; 654 655 if (asprintf(&service, "%d", port) < 0) 656 return NULL; 657@@ -157,7 +168,8 @@ static struct pingpong_dest *pp_client_e 658 return NULL; 659 } 660 661- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 662+ gid_to_wire_gid(&my_dest->gid, gid); 663+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 664 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 665 fprintf(stderr, "Couldn't send local address\n"); 666 goto out; 667@@ -175,7 +187,8 @@ static struct pingpong_dest *pp_client_e 668 if (!rem_dest) 669 goto out; 670 671- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 672+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 673+ wire_gid_to_gid(gid, &rem_dest->gid); 674 675 out: 676 close(sockfd); 677@@ -185,7 +198,8 @@ out: 678 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 679 int ib_port, enum ibv_mtu mtu, 680 int port, int sl, 681- const struct pingpong_dest *my_dest) 682+ const struct pingpong_dest *my_dest, 683+ int sgid_idx) 684 { 685 struct addrinfo *res, *t; 686 struct addrinfo hints = { 687@@ -194,10 +208,11 @@ static struct pingpong_dest *pp_server_e 688 .ai_socktype = SOCK_STREAM 689 }; 690 char *service; 691- char msg[sizeof "0000:000000:000000"]; 692+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 693 int n; 694 int sockfd = -1, connfd; 695 struct pingpong_dest *rem_dest = NULL; 696+ char gid[33]; 697 698 if (asprintf(&service, "%d", port) < 0) 699 return NULL; 700@@ -251,16 +266,18 @@ static struct pingpong_dest *pp_server_e 701 if (!rem_dest) 702 goto out; 703 704- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 705+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 706+ wire_gid_to_gid(gid, &rem_dest->gid); 707 708- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { 709+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { 710 fprintf(stderr, "Couldn't connect to remote QP\n"); 711 free(rem_dest); 712 rem_dest = NULL; 713 goto out; 714 } 715 716- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 717+ gid_to_wire_gid(&my_dest->gid, gid); 718+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 719 if (write(connfd, msg, sizeof msg) != sizeof msg) { 720 fprintf(stderr, "Couldn't send local address\n"); 721 free(rem_dest); 722@@ -281,7 +298,7 @@ static struct pingpong_context *pp_init_ 723 { 724 struct pingpong_context *ctx; 725 726- ctx = malloc(sizeof *ctx); 727+ ctx = calloc(1, sizeof *ctx); 728 if (!ctx) 729 return NULL; 730 731@@ -469,6 +486,7 @@ static void usage(const char *argv0) 732 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 733 printf(" -l, --sl=<sl> service level value\n"); 734 printf(" -e, --events sleep on CQ events (default poll)\n"); 735+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 736 } 737 738 int main(int argc, char *argv[]) 739@@ -492,6 +510,8 @@ int main(int argc, char *argv[]) 740 int rcnt, scnt; 741 int num_cq_events = 0; 742 int sl = 0; 743+ int gidx = -1; 744+ char gid[33]; 745 746 srand48(getpid() * time(NULL)); 747 748@@ -508,10 +528,11 @@ int main(int argc, char *argv[]) 749 { .name = "iters", .has_arg = 1, .val = 'n' }, 750 { .name = "sl", .has_arg = 1, .val = 'l' }, 751 { .name = "events", .has_arg = 0, .val = 'e' }, 752+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 753 { 0 } 754 }; 755 756- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); 757+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); 758 if (c == -1) 759 break; 760 761@@ -564,6 +585,10 @@ int main(int argc, char *argv[]) 762 ++use_event; 763 break; 764 765+ case 'g': 766+ gidx = strtol(optarg, NULL, 0); 767+ break; 768+ 769 default: 770 usage(argv[0]); 771 return 1; 772@@ -619,30 +644,45 @@ int main(int argc, char *argv[]) 773 return 1; 774 } 775 776- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 777- my_dest.qpn = ctx->qp->qp_num; 778- my_dest.psn = lrand48() & 0xffffff; 779- if (!my_dest.lid) { 780+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 781+ fprintf(stderr, "Couldn't get port info\n"); 782+ return 1; 783+ } 784+ 785+ my_dest.lid = ctx->portinfo.lid; 786+ if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { 787 fprintf(stderr, "Couldn't get local LID\n"); 788 return 1; 789 } 790 791- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 792- my_dest.lid, my_dest.qpn, my_dest.psn); 793+ if (gidx >= 0) { 794+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 795+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 796+ return 1; 797+ } 798+ } else 799+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 800+ 801+ my_dest.qpn = ctx->qp->qp_num; 802+ my_dest.psn = lrand48() & 0xffffff; 803+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 804+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 805+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 806 807 if (servername) 808 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 809 else 810- rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); 811+ rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); 812 813 if (!rem_dest) 814 return 1; 815 816- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 817- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 818+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 819+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 820+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 821 822 if (servername) 823- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) 824+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) 825 return 1; 826 827 ctx->pending = PINGPONG_RECV_WRID; 828Index: libibverbs/examples/ud_pingpong.c 829=================================================================== 830--- libibverbs.orig/examples/ud_pingpong.c 2010-02-08 15:04:24.389329000 +0200 831+++ libibverbs/examples/ud_pingpong.c 2010-03-17 14:08:48.502754000 +0200 832@@ -68,16 +68,18 @@ struct pingpong_context { 833 int size; 834 int rx_depth; 835 int pending; 836+ struct ibv_port_attr portinfo; 837 }; 838 839 struct pingpong_dest { 840 int lid; 841 int qpn; 842 int psn; 843+ union ibv_gid gid; 844 }; 845 846 static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, 847- int sl, struct pingpong_dest *dest) 848+ int sl, struct pingpong_dest *dest, int sgid_idx) 849 { 850 struct ibv_ah_attr ah_attr = { 851 .is_global = 0, 852@@ -105,6 +107,13 @@ static int pp_connect_ctx(struct pingpon 853 return 1; 854 } 855 856+ if (dest->gid.global.interface_id) { 857+ ah_attr.is_global = 1; 858+ ah_attr.grh.hop_limit = 1; 859+ ah_attr.grh.dgid = dest->gid; 860+ ah_attr.grh.sgid_index = sgid_idx; 861+ } 862+ 863 ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); 864 if (!ctx->ah) { 865 fprintf(stderr, "Failed to create AH\n"); 866@@ -123,10 +132,11 @@ static struct pingpong_dest *pp_client_e 867 .ai_socktype = SOCK_STREAM 868 }; 869 char *service; 870- char msg[sizeof "0000:000000:000000"]; 871+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 872 int n; 873 int sockfd = -1; 874 struct pingpong_dest *rem_dest = NULL; 875+ char gid[33]; 876 877 if (asprintf(&service, "%d", port) < 0) 878 return NULL; 879@@ -157,7 +167,8 @@ static struct pingpong_dest *pp_client_e 880 return NULL; 881 } 882 883- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 884+ gid_to_wire_gid(&my_dest->gid, gid); 885+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 886 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 887 fprintf(stderr, "Couldn't send local address\n"); 888 goto out; 889@@ -175,7 +186,8 @@ static struct pingpong_dest *pp_client_e 890 if (!rem_dest) 891 goto out; 892 893- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 894+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 895+ wire_gid_to_gid(gid, &rem_dest->gid); 896 897 out: 898 close(sockfd); 899@@ -184,7 +196,8 @@ out: 900 901 static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, 902 int ib_port, int port, int sl, 903- const struct pingpong_dest *my_dest) 904+ const struct pingpong_dest *my_dest, 905+ int sgid_idx) 906 { 907 struct addrinfo *res, *t; 908 struct addrinfo hints = { 909@@ -193,10 +206,11 @@ static struct pingpong_dest *pp_server_e 910 .ai_socktype = SOCK_STREAM 911 }; 912 char *service; 913- char msg[sizeof "0000:000000:000000"]; 914+ char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; 915 int n; 916 int sockfd = -1, connfd; 917 struct pingpong_dest *rem_dest = NULL; 918+ char gid[33]; 919 920 if (asprintf(&service, "%d", port) < 0) 921 return NULL; 922@@ -250,16 +264,18 @@ static struct pingpong_dest *pp_server_e 923 if (!rem_dest) 924 goto out; 925 926- sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); 927+ sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); 928+ wire_gid_to_gid(gid, &rem_dest->gid); 929 930- if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest)) { 931+ if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { 932 fprintf(stderr, "Couldn't connect to remote QP\n"); 933 free(rem_dest); 934 rem_dest = NULL; 935 goto out; 936 } 937 938- sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); 939+ gid_to_wire_gid(&my_dest->gid, gid); 940+ sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 941 if (write(connfd, msg, sizeof msg) != sizeof msg) { 942 fprintf(stderr, "Couldn't send local address\n"); 943 free(rem_dest); 944@@ -474,10 +490,11 @@ static void usage(const char *argv0) 945 printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n"); 946 printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n"); 947 printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n"); 948- printf(" -s, --size=<size> size of message to exchange (default 2048)\n"); 949+ printf(" -s, --size=<size> size of message to exchange (default 1024)\n"); 950 printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n"); 951 printf(" -n, --iters=<iters> number of exchanges (default 1000)\n"); 952 printf(" -e, --events sleep on CQ events (default poll)\n"); 953+ printf(" -g, --gid-idx=<gid index> local port gid index\n"); 954 } 955 956 int main(int argc, char *argv[]) 957@@ -492,7 +509,7 @@ int main(int argc, char *argv[]) 958 char *servername = NULL; 959 int port = 18515; 960 int ib_port = 1; 961- int size = 2048; 962+ int size = 1024; 963 int rx_depth = 500; 964 int iters = 1000; 965 int use_event = 0; 966@@ -500,6 +517,8 @@ int main(int argc, char *argv[]) 967 int rcnt, scnt; 968 int num_cq_events = 0; 969 int sl = 0; 970+ int gidx = -1; 971+ char gid[33]; 972 973 srand48(getpid() * time(NULL)); 974 975@@ -515,10 +534,11 @@ int main(int argc, char *argv[]) 976 { .name = "iters", .has_arg = 1, .val = 'n' }, 977 { .name = "sl", .has_arg = 1, .val = 'l' }, 978 { .name = "events", .has_arg = 0, .val = 'e' }, 979+ { .name = "gid-idx", .has_arg = 1, .val = 'g' }, 980 { 0 } 981 }; 982 983- c = getopt_long(argc, argv, "p:d:i:s:r:n:l:e", long_options, NULL); 984+ c = getopt_long(argc, argv, "p:d:i:s:r:n:l:eg:", long_options, NULL); 985 if (c == -1) 986 break; 987 988@@ -563,6 +583,10 @@ int main(int argc, char *argv[]) 989 ++use_event; 990 break; 991 992+ case 'g': 993+ gidx = strtol(optarg, NULL, 0); 994+ break; 995+ 996 default: 997 usage(argv[0]); 998 return 1; 999@@ -618,30 +642,41 @@ int main(int argc, char *argv[]) 1000 return 1; 1001 } 1002 1003- my_dest.lid = pp_get_local_lid(ctx->context, ib_port); 1004- my_dest.qpn = ctx->qp->qp_num; 1005- my_dest.psn = lrand48() & 0xffffff; 1006- if (!my_dest.lid) { 1007- fprintf(stderr, "Couldn't get local LID\n"); 1008+ if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { 1009+ fprintf(stderr, "Couldn't get port info\n"); 1010 return 1; 1011 } 1012+ my_dest.lid = ctx->portinfo.lid; 1013+ 1014+ my_dest.qpn = ctx->qp->qp_num; 1015+ my_dest.psn = lrand48() & 0xffffff; 1016+ 1017+ if (gidx >= 0) { 1018+ if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { 1019+ fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); 1020+ return 1; 1021+ } 1022+ } else 1023+ memset(&my_dest.gid, 0, sizeof my_dest.gid); 1024 1025- printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 1026- my_dest.lid, my_dest.qpn, my_dest.psn); 1027+ inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); 1028+ printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x: GID %s\n", 1029+ my_dest.lid, my_dest.qpn, my_dest.psn, gid); 1030 1031 if (servername) 1032 rem_dest = pp_client_exch_dest(servername, port, &my_dest); 1033 else 1034- rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest); 1035+ rem_dest = pp_server_exch_dest(ctx, ib_port, port, sl, &my_dest, gidx); 1036 1037 if (!rem_dest) 1038 return 1; 1039 1040- printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", 1041- rem_dest->lid, rem_dest->qpn, rem_dest->psn); 1042+ inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); 1043+ printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", 1044+ rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); 1045 1046 if (servername) 1047- if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest)) 1048+ if (pp_connect_ctx(ctx, ib_port, my_dest.psn, sl, rem_dest, gidx)) 1049 return 1; 1050 1051 ctx->pending = PINGPONG_RECV_WRID; 1052