dapl_tavor_hca.c revision 9517:b4839b0aa7a4
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#include <sys/types.h> 28#include <sys/socket.h> 29#include <sys/sockio.h> 30#include <sys/param.h> 31#include <sys/stat.h> 32#include <netinet/in.h> 33#include <arpa/inet.h> 34#include <net/if.h> 35#include <unistd.h> 36#include <stdlib.h> 37#include <string.h> 38#include <errno.h> 39#include <libdevinfo.h> 40 41#include "dapl.h" 42#include "dapl_adapter_util.h" 43#include "dapl_tavor_ibtf_impl.h" 44#include "dapl_hca_util.h" 45#include "dapl_name_service.h" 46#define IF_NAME "ibd" 47#define MAX_HCAS 64 48#define PROP_HCA_GUID "hca-guid" 49#define PROP_PORT_NUM "port-number" 50#define PROP_PORT_PKEY "port-pkey" 51 52#define DEVDAPLT "/dev/daplt" 53 54/* function prototypes */ 55static DAT_RETURN dapli_process_tavor_node(di_node_t node, int *hca_idx, 56 int try_blueflame); 57static DAT_RETURN dapli_process_ibd_node(di_node_t node, DAPL_HCA *hca_ptr, 58 int hca_idx); 59 60#if defined(IBHOSTS_NAMING) 61#include <stdio.h> 62static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx); 63#endif /* IBHOSTS_NAMING */ 64 65static DAPL_OS_LOCK g_tavor_state_lock; 66static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS]; 67DAPL_OS_LOCK g_tavor_uar_lock; 68 69DAT_RETURN 70dapli_init_hca( 71 IN DAPL_HCA *hca_ptr) 72{ 73 di_node_t root_node; 74 di_node_t hca_node; 75 di_node_t ibd_node; 76 DAT_RETURN dat_status = DAT_SUCCESS; 77 int hca_idx = 0; 78 int ia_instance; 79 int check_for_bf = 0; 80 81 ia_instance = (int)dapl_os_strtol(hca_ptr->name + strlen(IF_NAME), 82 NULL, 0); 83 84 root_node = di_init("/", DINFOCPYALL); 85 if (root_node == DI_NODE_NIL) { 86 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 87 "init_hca: di_init failed %s\n", strerror(errno)); 88 return (DAT_INTERNAL_ERROR); 89 } 90 91 ibd_node = di_drv_first_node(IF_NAME, root_node); 92 while (ibd_node != DI_NODE_NIL) { 93 /* find the ibd node matching our ianame */ 94 if (di_instance(ibd_node) == ia_instance) { 95 break; 96 } 97 ibd_node = di_drv_next_node(ibd_node); 98 } 99 100 if (ibd_node == DI_NODE_NIL) { 101 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 102 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 103 "init_hcas: ibd%d di_node not found\n", ia_instance); 104 goto bail; 105 } 106 107 hca_node = di_parent_node(ibd_node); 108 if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name(hca_node), 109 "tavor", strlen("tavor")) == 0)) 110 dapls_init_funcs_tavor(hca_ptr); 111 else if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name 112 (hca_node), "arbel", strlen("arbel")) == 0)) 113 dapls_init_funcs_arbel(hca_ptr); 114 else if ((hca_node != DI_NODE_NIL) && (strncmp(di_driver_name 115 (hca_node), "hermon", strlen("hermon")) == 0)) { 116 dapls_init_funcs_hermon(hca_ptr); 117 check_for_bf = 1; 118 } else { 119 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 120 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 121 "init_hcas: ibd%d hca_node not found\n", ia_instance); 122 goto bail; 123 } 124 125 dat_status = dapli_process_tavor_node(hca_node, &hca_idx, check_for_bf); 126 if (dat_status != DAT_SUCCESS) { 127 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 128 "init_hcas: ibd%d process_tavor_node failed(0x%x)\n", 129 ia_instance, dat_status); 130 goto bail; 131 } 132 133#if defined(IBHOSTS_NAMING) 134 if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) { 135 /* no entries were found */ 136 dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0); 137 } 138#else 139 dat_status = dapli_process_ibd_node(ibd_node, hca_ptr, hca_idx); 140#endif 141 if (dat_status != DAT_SUCCESS) { 142 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 143 "init_hcas: ibd%d process_ibd_node failed(0x%x)\n", 144 ia_instance, dat_status); 145 goto bail; 146 } 147 148 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 149 "init_hcas: done ibd%d\n", ia_instance); 150 151bail: 152 di_fini(root_node); 153 return (dat_status); 154} 155 156static DAT_RETURN 157dapli_process_tavor_node(di_node_t node, int *hca_idx, int try_blueflame) 158{ 159 char *dev_path; 160 char path_buf[MAXPATHLEN]; 161 int idx, fd; 162#ifndef _LP64 163 int tmpfd; 164#endif 165 size_t pagesize; 166 void *mapaddr; 167 pid_t cur_pid; 168 off64_t uarpg_offset; 169 170 dapl_os_lock(&g_tavor_state_lock); 171 172 for (idx = 0; idx < MAX_HCAS; idx++) { 173 /* 174 * page size == 0 means this entry is not occupied 175 */ 176 if (g_tavor_state[idx].uarpg_size == 0) { 177 break; 178 } 179 } 180 if (idx == MAX_HCAS) { 181 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 182 "process_tavor: all hcas are being used!\n"); 183 dapl_os_unlock(&g_tavor_state_lock); 184 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 185 } 186 dev_path = di_devfs_path(node); 187 188 /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */ 189 if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 > 190 MAXPATHLEN) { 191 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 192 "process_tavor: devfs path %s is too long\n", 193 dev_path); 194 di_devfs_path_free(dev_path); 195 dapl_os_unlock(&g_tavor_state_lock); 196 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 197 } 198 (void) dapl_os_strcpy(path_buf, "/devices"); 199 (void) dapl_os_strcat(path_buf, dev_path); 200 (void) dapl_os_strcat(path_buf, ":devctl"); 201 di_devfs_path_free(dev_path); 202 203 pagesize = (size_t)sysconf(_SC_PAGESIZE); 204 if (pagesize == 0) { 205 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 206 "process_tavor: page_size == 0\n"); 207 dapl_os_unlock(&g_tavor_state_lock); 208 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 209 } 210 cur_pid = getpid(); 211 212 fd = open(path_buf, O_RDWR); 213 if (fd < 0) { 214 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 215 "process_tavor: cannot open %s: %s\n", 216 path_buf, strerror(errno)); 217 dapl_os_unlock(&g_tavor_state_lock); 218 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 219 } 220#ifndef _LP64 221 /* 222 * libc can't handle fd's greater than 255, in order to 223 * ensure that these values remain available make fd > 255. 224 * Note: not needed for LP64 225 */ 226 tmpfd = fcntl(fd, F_DUPFD, 256); 227 if (tmpfd < 0) { 228 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 229 "process_tavor: cannot F_DUPFD: %s\n", strerror(errno)); 230 } else { 231 (void) close(fd); 232 fd = tmpfd; 233 } 234#endif /* _LP64 */ 235 236 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 237 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 238 "process_tavor: cannot F_SETFD: %s\n", strerror(errno)); 239 (void) close(fd); 240 dapl_os_unlock(&g_tavor_state_lock); 241 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 242 } 243 244 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 245 MLNX_UMAP_UARPG_RSRC) * pagesize; 246 247 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 248 MAP_SHARED, fd, uarpg_offset); 249 if (mapaddr == MAP_FAILED) { 250 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 251 "process_tavor: mmap failed %s\n", strerror(errno)); 252 (void) close(fd); 253 dapl_os_unlock(&g_tavor_state_lock); 254 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 255 } 256 257 g_tavor_state[idx].hca_fd = fd; 258 g_tavor_state[idx].uarpg_baseaddr = mapaddr; 259 g_tavor_state[idx].uarpg_size = pagesize; 260 261 if (try_blueflame == 0) 262 goto done; 263 264 /* Try to do the Hermon Blueflame page mapping */ 265 uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) | 266 MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize; 267 268 mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE, 269 MAP_SHARED, fd, uarpg_offset); 270 if (mapaddr == MAP_FAILED) { 271 /* This is not considered to be fatal. Charge on! */ 272 dapl_dbg_log(DAPL_DBG_TYPE_WARN, 273 "process_tavor: mmap of blueflame page failed %s\n", 274 strerror(errno)); 275 } else { 276 g_tavor_state[idx].bf_pg_baseaddr = mapaddr; 277 } 278done: 279 dapl_os_unlock(&g_tavor_state_lock); 280 281 *hca_idx = idx; 282 283 return (DAT_SUCCESS); 284} 285 286static DAT_RETURN 287dapli_process_ibd_node(di_node_t node, DAPL_HCA *hca_ptr, int hca_idx) 288{ 289 di_prop_t prop; 290 ib_guid_t hca_guid = 0; 291 struct lifreq lifreq; 292 uint32_t port_num = 0; 293 uint32_t partition_key = 0; 294 int instance, sfd, retval, af; 295 int tmp; 296 int digits; 297 char *drv_name; 298 char addr_buf[64]; 299 300 prop = di_prop_next(node, DI_PROP_NIL); 301 while (prop != DI_PROP_NIL) { 302 char *prop_name; 303 uchar_t *bytep; 304 int *intp, count; 305 306 prop_name = di_prop_name(prop); 307 count = 0; 308 309 if (strcmp(prop_name, PROP_HCA_GUID) == 0) { 310 count = di_prop_bytes(prop, &bytep); 311 dapl_os_assert(count == sizeof (ib_guid_t)); 312 (void) dapl_os_memcpy((void *)&hca_guid, (void *)bytep, 313 sizeof (ib_guid_t)); 314 } else if (strcmp(prop_name, PROP_PORT_NUM) == 0) { 315 count = di_prop_ints(prop, &intp); 316 dapl_os_assert(count == 1); 317 port_num = (uint32_t)intp[0]; 318 } else if (strcmp(prop_name, PROP_PORT_PKEY) == 0) { 319 count = di_prop_ints(prop, &intp); 320 dapl_os_assert(count == 1); 321 partition_key = (uint32_t)intp[0]; 322 } 323 prop = di_prop_next(node, prop); 324 } 325 if (hca_guid == 0 || port_num == 0 || partition_key == 0) { 326 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 327 "process_ibd: invalid properties: guid 0x%016llx, " 328 "port %d, pkey 0x%08x\n", hca_guid, port_num, 329 partition_key); 330 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 331 } 332 333 /* 334 * if an interface has both v4 and v6 addresses plumbed, 335 * we'll take the v4 address. 336 */ 337 af = AF_INET; 338again: 339 sfd = socket(af, SOCK_DGRAM, 0); 340 if (sfd < 0) { 341 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 342 "process_ibd: socket failed: %s\n", strerror(errno)); 343 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 344 } 345 instance = di_instance(node); 346 drv_name = di_driver_name(node); 347 348 /* calculate the number of digits in instance */ 349 tmp = instance; 350 digits = 0; 351 do { 352 tmp = tmp / 10; 353 digits++; 354 } while (tmp > 0); 355 /* check if name will fit in lifr_name */ 356 if (dapl_os_strlen(drv_name) + digits + 1 > LIFNAMSIZ) { 357 (void) close(sfd); 358 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 359 "process_ibd: if name overflow %s:%d\n", 360 drv_name, instance); 361 return (DAT_ERROR(DAT_INVALID_PARAMETER, 0)); 362 } 363 364 (void) dapl_os_strcpy(lifreq.lifr_name, drv_name); 365 (void) sprintf(&lifreq.lifr_name[dapl_os_strlen(drv_name)], "%d", 366 instance); 367 retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq); 368 if (retval < 0) { 369 (void) close(sfd); 370 if (af == AF_INET6) { 371 /* 372 * the interface is not plumbed. 373 */ 374 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 375 "process_ibd: %s: ip address not found\n", 376 lifreq.lifr_name); 377 return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0)); 378 } else { 379 /* 380 * we've failed to find a v4 address. now 381 * let's try v6. 382 */ 383 af = AF_INET6; 384 goto again; 385 } 386 } 387 (void) close(sfd); 388 389 hca_ptr->hca_ibd_inst = instance; 390 hca_ptr->tavor_idx = hca_idx; 391 hca_ptr->node_GUID = hca_guid; 392 hca_ptr->port_num = port_num; 393 hca_ptr->partition_key = partition_key; 394 (void) dapl_os_memcpy((void *)&hca_ptr->hca_address, 395 (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address)); 396 hca_ptr->max_inline_send = dapls_tavor_max_inline(); 397 398 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 399 "process_ibd: interface %s, hca guid 0x%016llx, port %d, " 400 "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_guid, 401 port_num, partition_key, dapls_inet_ntop( 402 (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64)); 403 return (DAT_SUCCESS); 404} 405 406void 407dapls_ib_state_init(void) 408{ 409 int i; 410 411 (void) dapl_os_lock_init(&g_tavor_state_lock); 412 (void) dapl_os_lock_init(&g_tavor_uar_lock); 413 (void) dapl_os_lock_init(&dapls_ib_dbp_lock); 414 415 for (i = 0; i < MAX_HCAS; i++) { 416 g_tavor_state[i].hca_fd = 0; 417 g_tavor_state[i].uarpg_baseaddr = NULL; 418 g_tavor_state[i].uarpg_size = 0; 419 g_tavor_state[i].bf_pg_baseaddr = NULL; 420 } 421} 422 423void 424dapls_ib_state_fini(void) 425{ 426 int i, count = 0; 427 428 /* 429 * Uinitialize the per hca instance state 430 */ 431 dapl_os_lock(&g_tavor_state_lock); 432 for (i = 0; i < MAX_HCAS; i++) { 433 if (g_tavor_state[i].uarpg_size == 0) { 434 dapl_os_assert(g_tavor_state[i].uarpg_baseaddr == 435 NULL); 436 continue; 437 } 438 if (munmap(g_tavor_state[i].uarpg_baseaddr, 439 g_tavor_state[i].uarpg_size) < 0) { 440 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 441 "ib_state_fini: " 442 "munmap(0x%p, 0x%llx) failed(%d)\n", 443 g_tavor_state[i].uarpg_baseaddr, 444 g_tavor_state[i].uarpg_size, errno); 445 } 446 if ((g_tavor_state[i].bf_pg_baseaddr != NULL) && 447 (munmap(g_tavor_state[i].bf_pg_baseaddr, 448 g_tavor_state[i].uarpg_size) < 0)) { 449 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 450 "ib_state_fini: " 451 "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n", 452 g_tavor_state[i].bf_pg_baseaddr, 453 g_tavor_state[i].uarpg_size, errno); 454 } 455 456 (void) close(g_tavor_state[i].hca_fd); 457 count++; 458 } 459 dapl_os_unlock(&g_tavor_state_lock); 460 461 dapl_os_lock_destroy(&g_tavor_uar_lock); 462 dapl_os_lock_destroy(&g_tavor_state_lock); 463 dapl_os_lock_destroy(&dapls_ib_dbp_lock); 464 465 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 466 "ib_state_fini: cleaned %d hcas\n", count); 467} 468 469/* 470 * dapls_ib_open_hca 471 * 472 * Open HCA 473 * 474 * Input: 475 * *hca_ptr pointer to hca device 476 * *ib_hca_handle_p pointer to provide HCA handle 477 * 478 * Output: 479 * none 480 * 481 * Return: 482 * DAT_SUCCESS 483 * DAT_INSUFFICIENT_RESOURCES 484 * 485 */ 486DAT_RETURN 487dapls_ib_open_hca( 488 IN DAPL_HCA *hca_ptr, 489 OUT ib_hca_handle_t *ib_hca_handle_p) 490{ 491 dapl_ia_create_t args; 492 DAT_RETURN dat_status; 493 struct dapls_ib_hca_handle *hca_p; 494 int fd; 495#ifndef _LP64 496 int tmpfd; 497#endif 498 int retval; 499 struct sockaddr *s; 500 struct sockaddr_in6 *v6addr; 501 struct sockaddr_in *v4addr; 502 dapl_ia_addr_t *sap; 503 504 dat_status = dapli_init_hca(hca_ptr); 505 if (dat_status != DAT_SUCCESS) { 506 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 507 "dapls_ib_open_hca: init_hca failed %d\n", dat_status); 508 return (dat_status); 509 } 510 511 fd = open(DEVDAPLT, O_RDONLY); 512 if (fd < 0) { 513 return (DAT_INSUFFICIENT_RESOURCES); 514 } 515 516#ifndef _LP64 517 /* 518 * libc can't handle fd's greater than 255, in order to 519 * ensure that these values remain available make fd > 255. 520 * Note: not needed for LP64 521 */ 522 tmpfd = fcntl(fd, F_DUPFD, 256); 523 if (tmpfd < 0) { 524 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 525 "dapls_ib_open_hca: cannot F_DUPFD: %s\n", 526 strerror(errno)); 527 } else { 528 (void) close(fd); 529 fd = tmpfd; 530 } 531#endif /* _LP64 */ 532 533 if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) { 534 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 535 "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno)); 536 (void) close(fd); 537 return (DAT_INTERNAL_ERROR); 538 } 539 540 hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc( 541 sizeof (struct dapls_ib_hca_handle)); 542 if (hca_p == NULL) { 543 (void) close(fd); 544 return (DAT_INSUFFICIENT_RESOURCES); 545 } 546 547 args.ia_guid = hca_ptr->node_GUID; 548 args.ia_port = hca_ptr->port_num; 549 args.ia_pkey = hca_ptr->partition_key; 550 args.ia_version = DAPL_IF_VERSION; 551 (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES); 552 553 /* pass down local ip address to be stored in SA */ 554 s = (struct sockaddr *)&hca_ptr->hca_address; 555 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 556 sap = (dapl_ia_addr_t *)args.ia_sadata; 557 switch (s->sa_family) { 558 case AF_INET: 559 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 560 v4addr = (struct sockaddr_in *)s; 561 sap->iad_v4 = v4addr->sin_addr; 562 break; 563 case AF_INET6: 564 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 565 v6addr = (struct sockaddr_in6 *)s; 566 sap->iad_v6 = v6addr->sin6_addr; 567 break; 568 default: 569 break; /* fall through */ 570 } 571 572 retval = ioctl(fd, DAPL_IA_CREATE, &args); 573 if (retval != 0) { 574 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 575 "open_hca: ia_create failed, fd %d, " 576 "guid 0x%016llx, port %d, pkey 0x%x, version %d\n", 577 fd, args.ia_guid, args.ia_port, args.ia_pkey, 578 args.ia_version); 579 580 dapl_os_free(hca_p, sizeof (*hca_p)); 581 (void) close(fd); 582 return (dapls_convert_error(errno, retval)); 583 } 584 585 hca_p->ia_fd = fd; 586 hca_p->ia_rnum = args.ia_resnum; 587 hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd; 588 hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr; 589 hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr; 590 hca_p->ia_bf_toggle = 0; 591 *ib_hca_handle_p = hca_p; 592 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 593 "open_hca: ia_created, hca_p 0x%p, fd %d, " 594 "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n", 595 hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID, 596 hca_ptr->port_num, hca_ptr->partition_key); 597 598 return (DAT_SUCCESS); 599} 600 601/* 602 * dapls_ib_close_hca 603 * 604 * Open HCA 605 * 606 * Input: 607 * ib_hca_handle provide HCA handle 608 * 609 * Output: 610 * none 611 * 612 * Return: 613 * DAT_SUCCESS 614 * DAT_INSUFFICIENT_RESOURCES 615 * 616 */ 617DAT_RETURN 618dapls_ib_close_hca( 619 IN ib_hca_handle_t ib_hca_handle) 620{ 621 if (ib_hca_handle == NULL) { 622 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 623 "close_hca: ib_hca_handle == NULL\n"); 624 return (DAT_SUCCESS); 625 } 626 dapl_dbg_log(DAPL_DBG_TYPE_UTIL, 627 "close_hca: closing hca 0x%p, fd %d, rnum %d\n", 628 ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum); 629 630 (void) close(ib_hca_handle->ia_fd); 631 dapl_os_free((void *)ib_hca_handle, 632 sizeof (struct dapls_ib_hca_handle)); 633 return (DAT_SUCCESS); 634} 635 636#if defined(IBHOSTS_NAMING) 637#define LINE_LEN 256 638static int 639dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx) 640{ 641 char line_buf[LINE_LEN]; 642 char host_buf[LINE_LEN]; 643 char localhost[LINE_LEN]; 644 ib_guid_t prefix; 645 ib_guid_t guid; 646 FILE *fp; 647 int count = 0; 648 DAPL_HCA *hca_ptr; 649 650 fp = fopen("/etc/dapl/ibhosts", "r"); 651 if (fp == NULL) { 652 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 653 "fake_ibds: ibhosts not found!\n"); 654 return (0); 655 } 656 if (gethostname(localhost, LINE_LEN) != 0) { 657 dapl_dbg_log(DAPL_DBG_TYPE_ERR, 658 "fake_ibds: hostname not found!\n"); 659 return (0); 660 } 661 while (!feof(fp)) { 662 (void) fgets(line_buf, LINE_LEN, fp); 663 sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid); 664 (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1); 665 if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) { 666 guid &= 0xfffffffffffffff0; 667 hca_ptr->hca_ibd_inst = count + 1; 668 hca_ptr->tavor_idx = hca_idx; 669 hca_ptr->node_GUID = guid; 670 hca_ptr->port_num = count + 1; 671 hca_ptr->partition_key = 0x0000ffff; 672 count++; 673 } 674 if (count >= 2) break; 675 } 676 (void) fclose(fp); 677 return (count); 678} 679 680#endif /* IBHOSTS_NAMING */ 681