1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved. 24 */ 25 26/* 27 * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31/* 32 * 33 * MODULE: dapl_osd.c 34 * 35 * PURPOSE: Operating System Dependent layer 36 * Description: 37 * Provide OS dependent functions with a canonical DAPL 38 * interface. Designed to be portable and hide OS specific quirks 39 * of common functions. 40 * 41 * 42 * $Id: dapl_osd.c,v 1.26 2003/07/31 14:04:18 jlentini Exp $ 43 */ 44 45#include "dapl_osd.h" 46#include "dapl.h" 47#include "dapl_hca_util.h" 48#include "dapl_ia_util.h" 49#include "dapl_rmr_util.h" 50#include "dapl_lmr_util.h" 51#include "dapl_pz_util.h" 52#include "dapl_ep_util.h" 53#include "dapl_cr_util.h" 54#include "dapl_evd_util.h" 55#include "dapl_sp_util.h" 56#include "dapl_adapter_util.h" 57#include "dapl_provider.h" 58#include "dapl_hash.h" 59#include "dapl_debug.h" 60 61#include <sys/time.h> 62#include <stdlib.h> /* needed for getenv() */ 63#include <pthread.h> /* needed for pthread_atfork() */ 64#include <signal.h> /* needed for thread setup */ 65 66static void dapls_osd_fork_cleanup(void); 67 68/* 69 * dapl_osd_init 70 * 71 * Do Linux initialization: 72 * - Set up fork handler to clean up DAPL resources in the child 73 * process after a fork(). 74 * 75 * Input: 76 * none 77 * 78 * Returns: 79 * DAT_SUCCESS 80 */ 81void 82dapl_os_init() 83{ 84 int status; 85 86 /* 87 * Set up fork control 88 */ 89 status = pthread_atfork(NULL, NULL, dapls_osd_fork_cleanup); 90 if (status != 0) { 91 dapl_dbg_log(DAPL_DBG_TYPE_WARN, 92 "WARNING: pthread_atfork %d\n", status); 93 } 94} 95 96 97/* 98 * dapl_os_get_time 99 * 100 * Return 64 bit value of current time in microseconds. 101 * 102 * Input: 103 * loc User location to place current time 104 * 105 * Returns: 106 * DAT_SUCCESS 107 */ 108 109DAT_RETURN 110dapl_os_get_time( 111 OUT DAPL_OS_TIMEVAL * loc) 112{ 113 struct timeval tv; 114 struct timezone tz; 115 116 117 (void) gettimeofday(&tv, &tz); 118 *loc = ((DAT_UINT64)(tv.tv_sec) * 1000000L) + (DAT_UINT64) tv.tv_usec; 119 120 return (DAT_SUCCESS); 121} 122 123 124/* 125 * dapl_os_get__env_bool 126 * 127 * Return boolean value of passed in environment variable: 1 if present, 128 * 0 if not 129 * 130 * Input: 131 * 132 * 133 * Returns: 134 * TRUE or FALSE 135 */ 136int 137dapl_os_get_env_bool( 138 char *env_str) 139{ 140 char *env_var; 141 142 env_var = getenv(env_str); 143 if (env_var != NULL) { 144 return (1); 145 } 146 147 return (0); 148} 149 150 151/* 152 * dapl_os_get_env_val 153 * 154 * Update val to value of passed in environment variable if present 155 * 156 * Input: 157 * env_str 158 * def_val default value if environment variable does not exist 159 * 160 * Returns: 161 * TRUE or FALSE 162 */ 163int 164dapl_os_get_env_val( 165 char *env_str, 166 int def_val) 167{ 168 char *env_var; 169 170 env_var = getenv(env_str); 171 if (env_var != NULL) { 172 def_val = strtol(env_var, NULL, 0); 173 } 174 175 return (def_val); 176} 177 178 179/* 180 * Wait object routines 181 */ 182 183/* 184 * dapl_os_wait_object_init 185 * 186 * Initialize a wait object 187 * 188 * Input: 189 * wait_obj 190 * 191 * Returns: 192 * DAT_SUCCESS 193 * DAT_INTERNAL_ERROR 194 */ 195DAT_RETURN 196dapl_os_wait_object_init( 197 IN DAPL_OS_WAIT_OBJECT *wait_obj) 198{ 199 wait_obj->signaled = DAT_FALSE; 200 if (0 != pthread_cond_init(&wait_obj->cv, NULL)) { 201 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 202 } 203 204 /* Always returns 0. */ 205 (void) pthread_mutex_init(&wait_obj->lock, NULL); 206 207 return (DAT_SUCCESS); 208} 209 210 211/* 212 * Wait on the supplied wait object, up to the specified time_out. 213 * A timeout of DAT_TIMEOUT_INFINITE will wait indefinitely. 214 * Timeout should be specified in micro seconds. 215 * 216 * Functional returns: 217 * DAT_SUCCESS -- another thread invoked dapl_os_wait object_wakeup 218 * DAT_INVALID_STATE -- someone else is already waiting in this wait 219 * object. 220 * only one waiter is allowed at a time. 221 * DAT_ABORT -- another thread invoked dapl_os_wait_object_destroy 222 * DAT_TIMEOUT -- the specified time limit was reached. 223 */ 224 225DAT_RETURN 226dapl_os_wait_object_wait( 227 IN DAPL_OS_WAIT_OBJECT *wait_obj, 228 IN DAT_TIMEOUT timeout_val) 229{ 230 DAT_RETURN dat_status; 231 int pthread_status; 232 struct timespec future; 233 234 dat_status = DAT_SUCCESS; 235 pthread_status = 0; 236 237 if (timeout_val != DAT_TIMEOUT_INFINITE) { 238 struct timeval now; 239 struct timezone tz; 240 unsigned int microsecs; 241 242 (void) gettimeofday(&now, &tz); 243 microsecs = now.tv_usec + (timeout_val % 1000000); 244 if (microsecs > 1000000) { 245 now.tv_sec = now.tv_sec + timeout_val / 1000000 + 1; 246 now.tv_usec = microsecs - 1000000; 247 } else { 248 now.tv_sec = now.tv_sec + timeout_val / 1000000; 249 now.tv_usec = microsecs; 250 } 251 252 /* Convert timeval to timespec */ 253 future.tv_sec = now.tv_sec; 254 future.tv_nsec = now.tv_usec * 1000; 255 256 (void) pthread_mutex_lock(&wait_obj->lock); 257 while (wait_obj->signaled == DAT_FALSE && pthread_status == 0) { 258 pthread_status = pthread_cond_timedwait( 259 &wait_obj->cv, &wait_obj->lock, &future); 260 261 /* 262 * No need to reset &future if we go around the loop; 263 * It's an absolute time. 264 */ 265 } 266 /* Reset the signaled status if we were woken up. */ 267 if (pthread_status == 0) { 268 wait_obj->signaled = DAT_FALSE; 269 } 270 (void) pthread_mutex_unlock(&wait_obj->lock); 271 } else { 272 (void) pthread_mutex_lock(&wait_obj->lock); 273 while (wait_obj->signaled == DAT_FALSE && pthread_status == 0) { 274 pthread_status = pthread_cond_wait( 275 &wait_obj->cv, &wait_obj->lock); 276 } 277 /* Reset the signaled status if we were woken up. */ 278 if (pthread_status == 0) { 279 wait_obj->signaled = DAT_FALSE; 280 } 281 (void) pthread_mutex_unlock(&wait_obj->lock); 282 } 283 284 if (ETIMEDOUT == pthread_status) { 285 dat_status = DAT_ERROR(DAT_TIMEOUT_EXPIRED, 0); 286 } else if (0 != pthread_status) { 287 dat_status = DAT_ERROR(DAT_INTERNAL_ERROR, 0); 288 } 289 290 return (dat_status); 291} 292 293 294/* 295 * dapl_os_wait_object_wakeup 296 * 297 * Wakeup a thread waiting on a wait object 298 * 299 * Input: 300 * wait_obj 301 * 302 * Returns: 303 * DAT_SUCCESS 304 * DAT_INTERNAL_ERROR 305 */ 306DAT_RETURN 307dapl_os_wait_object_wakeup( 308 IN DAPL_OS_WAIT_OBJECT *wait_obj) 309{ 310 (void) pthread_mutex_lock(&wait_obj->lock); 311 wait_obj->signaled = DAT_TRUE; 312 (void) pthread_mutex_unlock(&wait_obj->lock); 313 if (0 != pthread_cond_signal(&wait_obj->cv)) { 314 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 315 } 316 317 return (DAT_SUCCESS); 318} 319 320 321/* 322 * dapl_os_wait_object_destroy 323 * 324 * Destroy a wait object 325 * 326 * Input: 327 * wait_obj 328 * 329 * Returns: 330 * DAT_SUCCESS 331 * DAT_INTERNAL_ERROR 332 */ 333DAT_RETURN 334dapl_os_wait_object_destroy( 335 IN DAPL_OS_WAIT_OBJECT *wait_obj) 336{ 337 if (0 != pthread_cond_destroy(&wait_obj->cv)) { 338 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 339 } 340 if (0 != pthread_mutex_destroy(&wait_obj->lock)) { 341 return (DAT_ERROR(DAT_INTERNAL_ERROR, 0)); 342 } 343 344 return (DAT_SUCCESS); 345} 346 347 348/* 349 * dapls_osd_fork_cleanup 350 * 351 * Update val to value of passed in environment variable if present 352 * 353 * Input: 354 * env_str 355 * val Updated if environment variable exists 356 * 357 * Returns: 358 * TRUE or FALSE 359 */ 360void 361dapls_osd_fork_cleanup(void) 362{ 363 DAPL_PROVIDER_LIST_NODE *cur_node; 364 DAPL_HCA *hca_ptr; 365 DAPL_IA *ia_ptr; 366 DAPL_LMR *lmr_ptr; 367 DAPL_RMR *rmr_ptr; 368 DAPL_PZ *pz_ptr; 369 DAPL_CR *cr_ptr; 370 DAPL_EP *ep_ptr; 371 DAPL_EVD *evd_ptr; 372 DAT_EP_PARAM *param; 373 DAPL_SP *sp_ptr; 374 375 while (NULL != g_dapl_provider_list.head) { 376 cur_node = g_dapl_provider_list.head; 377 g_dapl_provider_list.head = cur_node->next; 378 379 hca_ptr = (DAPL_HCA *) cur_node->data.extension; 380 381 /* 382 * Walk the list of IA ptrs & clean up. This is purposely 383 * a destructive list walk, we really don't want to preserve 384 * any of it. 385 */ 386 while (!dapl_llist_is_empty(&hca_ptr->ia_list_head)) { 387 ia_ptr = (DAPL_IA *) 388 dapl_llist_peek_head(&hca_ptr->ia_list_head); 389 390 /* 391 * The rest of the cleanup code is similar to 392 * dapl_ia_close, the big difference is that we don't 393 * release IB resources, only memory; the underlying IB 394 * subsystem doesn't deal with fork at all, so leave 395 * IB handles alone. 396 */ 397 while (!dapl_llist_is_empty(&ia_ptr->rmr_list_head)) { 398 rmr_ptr = (DAPL_RMR *) 399 dapl_llist_peek_head(&ia_ptr-> 400 rmr_list_head); 401 if (rmr_ptr->param.lmr_triplet. 402 virtual_address != 0) { 403 (void) dapl_os_atomic_dec(&rmr_ptr-> 404 lmr->lmr_ref_count); 405 rmr_ptr->param.lmr_triplet. 406 virtual_address = 0; 407 } 408 dapl_os_atomic_dec(&rmr_ptr->pz->pz_ref_count); 409 dapl_ia_unlink_rmr(rmr_ptr->header.owner_ia, 410 rmr_ptr); 411 dapl_rmr_dealloc(rmr_ptr); 412 } 413 414 while (!dapl_llist_is_empty(&ia_ptr->rsp_list_head)) { 415 sp_ptr = (DAPL_SP *) dapl_llist_peek_head( 416 &ia_ptr->rsp_list_head); 417 dapl_os_atomic_dec(&((DAPL_EVD *)sp_ptr-> 418 evd_handle)->evd_ref_count); 419 dapls_ia_unlink_sp(ia_ptr, sp_ptr); 420 dapls_sp_free_sp(sp_ptr); 421 } 422 423 while (!dapl_llist_is_empty(&ia_ptr->ep_list_head)) { 424 ep_ptr = (DAPL_EP *) dapl_llist_peek_head( 425 &ia_ptr->ep_list_head); 426 param = &ep_ptr->param; 427 if (param->pz_handle != NULL) { 428 dapl_os_atomic_dec(&((DAPL_PZ *)param-> 429 pz_handle)->pz_ref_count); 430 } 431 if (param->recv_evd_handle != NULL) { 432 dapl_os_atomic_dec(&((DAPL_EVD *)param-> 433 recv_evd_handle)->evd_ref_count); 434 } 435 if (param->request_evd_handle) { 436 dapl_os_atomic_dec(&((DAPL_EVD *)param-> 437 request_evd_handle)->evd_ref_count); 438 } 439 if (param->connect_evd_handle != NULL) { 440 dapl_os_atomic_dec(&((DAPL_EVD *)param-> 441 connect_evd_handle)->evd_ref_count); 442 } 443 444 /* ...and free the resource */ 445 dapl_ia_unlink_ep(ia_ptr, ep_ptr); 446 dapl_ep_dealloc(ep_ptr); 447 } 448 449 while (!dapl_llist_is_empty(&ia_ptr->lmr_list_head)) { 450 lmr_ptr = (DAPL_LMR *) dapl_llist_peek_head( 451 &ia_ptr->lmr_list_head); 452 453 (void) dapls_hash_remove(lmr_ptr->header. 454 owner_ia->hca_ptr->lmr_hash_table, 455 lmr_ptr->param.lmr_context, NULL); 456 457 pz_ptr = (DAPL_PZ *) lmr_ptr->param.pz_handle; 458 dapl_os_atomic_dec(&pz_ptr->pz_ref_count); 459 dapl_ia_unlink_lmr(lmr_ptr->header.owner_ia, 460 lmr_ptr); 461 dapl_lmr_dealloc(lmr_ptr); 462 } 463 464 while (!dapl_llist_is_empty(&ia_ptr->psp_list_head)) { 465 sp_ptr = (DAPL_SP *) dapl_llist_peek_head( 466 &ia_ptr->psp_list_head); 467 while (!dapl_llist_is_empty(&sp_ptr-> 468 cr_list_head)) { 469 cr_ptr = (DAPL_CR *) 470 dapl_llist_peek_head( 471 &sp_ptr->cr_list_head); 472 dapl_sp_remove_cr(sp_ptr, cr_ptr); 473 dapls_cr_free(cr_ptr); 474 } 475 476 dapls_ia_unlink_sp(ia_ptr, sp_ptr); 477 dapl_os_atomic_dec(&((DAPL_EVD *)sp_ptr-> 478 evd_handle)->evd_ref_count); 479 dapls_sp_free_sp(sp_ptr); 480 } 481 482 while (!dapl_llist_is_empty(&ia_ptr->pz_list_head)) { 483 pz_ptr = (DAPL_PZ *) 484 dapl_llist_peek_head(&ia_ptr->pz_list_head); 485 dapl_ia_unlink_pz(pz_ptr->header.owner_ia, 486 pz_ptr); 487 dapl_pz_dealloc(pz_ptr); 488 } 489 490 while (!dapl_llist_is_empty(&ia_ptr->evd_list_head)) { 491 evd_ptr = (DAPL_EVD *) dapl_llist_peek_head( 492 &ia_ptr->evd_list_head); 493 dapl_ia_unlink_evd(evd_ptr->header.owner_ia, 494 evd_ptr); 495 /* 496 * reset the cq_handle to avoid having it 497 * removed 498 */ 499 evd_ptr->ib_cq_handle = IB_INVALID_HANDLE; 500 (void) dapls_evd_dealloc(evd_ptr); 501 } 502 503 dapl_hca_unlink_ia(ia_ptr->hca_ptr, ia_ptr); 504 /* 505 * asycn error evd was taken care of above, reset the 506 * pointer 507 */ 508 ia_ptr->async_error_evd = NULL; 509 dapls_ia_free(ia_ptr); 510 } /* end while( ia_ptr != NULL ) */ 511 512 513 dapl_os_free(cur_node, sizeof (DAPL_PROVIDER_LIST_NODE)); 514 } /* end while (NULL != g_dapl_provider_list.head) */ 515} 516 517 518/* 519 * Local variables: 520 * c-indent-level: 4 521 * c-basic-offset: 4 522 * tab-width: 8 523 * End: 524 */ 525