1/* 2 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36/* 37 * Abstract: 38 * Declaration of dispatcher abstraction. 39 */ 40 41#ifndef _CL_DISPATCHER_H_ 42#define _CL_DISPATCHER_H_ 43 44#include <complib/cl_atomic.h> 45#include <complib/cl_threadpool.h> 46#include <complib/cl_qlist.h> 47#include <complib/cl_qpool.h> 48#include <complib/cl_spinlock.h> 49#include <complib/cl_ptr_vector.h> 50 51#ifdef __cplusplus 52# define BEGIN_C_DECLS extern "C" { 53# define END_C_DECLS } 54#else /* !__cplusplus */ 55# define BEGIN_C_DECLS 56# define END_C_DECLS 57#endif /* __cplusplus */ 58 59BEGIN_C_DECLS 60/****h* Component Library/Dispatcher 61* NAME 62* Dispatcher 63* 64* DESCRIPTION 65* The Dispatcher provides a facility for message routing to 66* asynchronous worker threads. 67* 68* The Dispatcher functions operate on a cl_dispatcher_t structure 69* which should be treated as opaque and should be manipulated 70* only through the provided functions. 71* 72* SEE ALSO 73* Structures: 74* cl_dispatcher_t 75* 76* Initialization/Destruction: 77* cl_disp_construct, cl_disp_init, cl_disp_shutdown, cl_disp_destroy 78* 79* Manipulation: 80* cl_disp_post, cl_disp_reset, cl_disp_wait_on 81*********/ 82/****s* Component Library: Dispatcher/cl_disp_msgid_t 83* NAME 84* cl_disp_msgid_t 85* 86* DESCRIPTION 87* Defines the type of dispatcher messages. 88* 89* SYNOPSIS 90*/ 91typedef uint32_t cl_disp_msgid_t; 92/**********/ 93 94/****s* Component Library: Dispatcher/CL_DISP_MSGID_NONE 95* NAME 96* CL_DISP_MSGID_NONE 97* 98* DESCRIPTION 99* Defines a message value that means "no message". 100* This value is used during registration by Dispatcher clients 101* that do not wish to receive messages. 102* 103* No Dispatcher message is allowed to have this value. 104* 105* SYNOPSIS 106*/ 107#define CL_DISP_MSGID_NONE 0xFFFFFFFF 108/**********/ 109 110/****s* Component Library: Dispatcher/CL_DISP_INVALID_HANDLE 111* NAME 112* CL_DISP_INVALID_HANDLE 113* 114* DESCRIPTION 115* Defines the value of an invalid Dispatcher registration handle. 116* 117* SYNOPSIS 118*/ 119#define CL_DISP_INVALID_HANDLE ((cl_disp_reg_handle_t)0) 120/*********/ 121 122/****f* Component Library: Dispatcher/cl_pfn_msgrcv_cb_t 123* NAME 124* cl_pfn_msgrcv_cb_t 125* 126* DESCRIPTION 127* This typedef defines the prototype for client functions invoked 128* by the Dispatcher. The Dispatcher calls the corresponding 129* client function when delivering a message to the client. 130* 131* The client function must be reentrant if the user creates a 132* Dispatcher with more than one worker thread. 133* 134* SYNOPSIS 135*/ 136typedef void 137 (*cl_pfn_msgrcv_cb_t) (IN void *context, IN void *p_data); 138/* 139* PARAMETERS 140* context 141* [in] Client specific context specified in a call to 142* cl_disp_register 143* 144* p_data 145* [in] Pointer to the client specific data payload 146* of this message. 147* 148* RETURN VALUE 149* This function does not return a value. 150* 151* NOTES 152* This typedef provides a function prototype reference for 153* the function provided by Dispatcher clients as a parameter 154* to the cl_disp_register function. 155* 156* SEE ALSO 157* Dispatcher, cl_disp_register 158*********/ 159 160/****f* Component Library: Dispatcher/cl_pfn_msgdone_cb_t 161* NAME 162* cl_pfn_msgdone_cb_t 163* 164* DESCRIPTION 165* This typedef defines the prototype for client functions invoked 166* by the Dispatcher. The Dispatcher calls the corresponding 167* client function after completing delivery of a message. 168* 169* The client function must be reentrant if the user creates a 170* Dispatcher with more than one worker thread. 171* 172* SYNOPSIS 173*/ 174typedef void 175 (*cl_pfn_msgdone_cb_t) (IN void *context, IN void *p_data); 176/* 177* PARAMETERS 178* context 179* [in] Client specific context specified in a call to 180* cl_disp_post 181* 182* p_data 183* [in] Pointer to the client specific data payload 184* of this message. 185* 186* RETURN VALUE 187* This function does not return a value. 188* 189* NOTES 190* This typedef provides a function prototype reference for 191* the function provided by Dispatcher clients as a parameter 192* to the cl_disp_post function. 193* 194* SEE ALSO 195* Dispatcher, cl_disp_post 196*********/ 197 198/****s* Component Library: Dispatcher/cl_dispatcher_t 199* NAME 200* cl_dispatcher_t 201* 202* DESCRIPTION 203* Dispatcher structure. 204* 205* The Dispatcher is thread safe. 206* 207* The cl_dispatcher_t structure should be treated as opaque and should 208* be manipulated only through the provided functions. 209* 210* SYNOPSIS 211*/ 212typedef struct _cl_dispatcher { 213 cl_spinlock_t lock; 214 cl_ptr_vector_t reg_vec; 215 cl_qlist_t reg_list; 216 cl_thread_pool_t worker_threads; 217 cl_qlist_t msg_fifo; 218 cl_qpool_t msg_pool; 219 uint64_t last_msg_queue_time_us; 220} cl_dispatcher_t; 221/* 222* FIELDS 223* reg_vec 224* Vector of registration info objects. Indexed by message msg_id. 225* 226* lock 227* Spinlock to guard internal structures. 228* 229* msg_fifo 230* FIFO of messages being processed by the Dispatcher. New 231* messages are posted to the tail of the FIFO. Worker threads 232* pull messages from the front. 233* 234* worker_threads 235* Thread pool of worker threads to dispose of posted messages. 236* 237* msg_pool 238* Pool of message objects to be processed through the FIFO. 239* 240* reg_count 241* Count of the number of registrants. 242* 243* state 244* Indicates the state of the object. 245* 246* last_msg_queue_time_us 247* The time that the last message spent in the Q in usec 248* 249* SEE ALSO 250* Dispatcher 251*********/ 252 253/****s* Component Library: Dispatcher/cl_disp_reg_info_t 254* NAME 255* cl_disp_reg_info_t 256* 257* DESCRIPTION 258* Defines the dispatcher registration object structure. 259* 260* The cl_disp_reg_info_t structure is for internal use by the 261* Dispatcher only. 262* 263* SYNOPSIS 264*/ 265typedef struct _cl_disp_reg_info { 266 cl_list_item_t list_item; 267 cl_pfn_msgrcv_cb_t pfn_rcv_callback; 268 const void *context; 269 atomic32_t ref_cnt; 270 cl_disp_msgid_t msg_id; 271 cl_dispatcher_t *p_disp; 272} cl_disp_reg_info_t; 273/* 274* FIELDS 275* pfn_rcv_callback 276* Client's message receive callback. 277* 278* context 279* Client's context for message receive callback. 280* 281* rcv_thread_count 282* Number of threads currently in the receive callback. 283* 284* msg_done_thread_count 285* Number of threads currently in the message done callback. 286* 287* state 288* State of this registration object. 289* DISP_REGSTATE_INIT: initialized and inactive 290* DISP_REGSTATE_ACTIVE: in active use 291* DISP_REGSTATE_UNREGPEND: unregistration is pending 292* 293* msg_id 294* Dispatcher message msg_id value for this registration object. 295* 296* p_disp 297* Pointer to parent Dispatcher. 298* 299* SEE ALSO 300*********/ 301 302/****s* Component Library: Dispatcher/cl_disp_msg_t 303* NAME 304* cl_disp_msg_t 305* 306* DESCRIPTION 307* Defines the dispatcher message structure. 308* 309* The cl_disp_msg_t structure is for internal use by the 310* Dispatcher only. 311* 312* SYNOPSIS 313*/ 314typedef struct _cl_disp_msg { 315 cl_pool_item_t item; 316 const void *p_data; 317 cl_disp_reg_info_t *p_src_reg; 318 cl_disp_reg_info_t *p_dest_reg; 319 cl_pfn_msgdone_cb_t pfn_xmt_callback; 320 uint64_t in_time; 321 const void *context; 322} cl_disp_msg_t; 323/* 324* FIELDS 325* item 326* List & Pool linkage. Must be first element in the structure!! 327* 328* msg_id 329* The message's numberic ID value. 330* 331* p_data 332* Pointer to the data payload for this message. The payload 333* is opaque to the Dispatcher. 334* 335* p_reg_info 336* Pointer to the registration info of the sender. 337* 338* pfn_xmt_callback 339* Client's message done callback. 340* 341* in_time 342* The absolute time the message was inserted into the queue 343* 344* context 345* Client's message done callback context. 346* 347* SEE ALSO 348*********/ 349 350/****s* Component Library: Dispatcher/cl_disp_reg_info_t 351* NAME 352* cl_disp_reg_info_t 353* 354* DESCRIPTION 355* Defines the Dispatcher registration handle. This handle 356* should be treated as opaque by the client. 357* 358* SYNOPSIS 359*/ 360typedef const struct _cl_disp_reg_info *cl_disp_reg_handle_t; 361/**********/ 362 363/****f* Component Library: Dispatcher/cl_disp_construct 364* NAME 365* cl_disp_construct 366* 367* DESCRIPTION 368* This function constructs a Dispatcher object. 369* 370* SYNOPSIS 371*/ 372void cl_disp_construct(IN cl_dispatcher_t * const p_disp); 373/* 374* PARAMETERS 375* p_disp 376* [in] Pointer to a Dispatcher. 377* 378* RETURN VALUE 379* This function does not return a value. 380* 381* NOTES 382* Allows calling cl_disp_init and cl_disp_destroy. 383* 384* SEE ALSO 385* Dispatcher, cl_disp_init, cl_disp_destroy 386*********/ 387 388/****f* Component Library: Dispatcher/cl_disp_init 389* NAME 390* cl_disp_init 391* 392* DESCRIPTION 393* This function initializes a Dispatcher object. 394* 395* SYNOPSIS 396*/ 397cl_status_t 398cl_disp_init(IN cl_dispatcher_t * const p_disp, 399 IN const uint32_t thread_count, IN const char *const name); 400/* 401* PARAMETERS 402* p_disp 403* [in] Pointer to a Dispatcher. 404* 405* thread_count 406* [in] The number of worker threads to create in this Dispatcher. 407* A value of 0 causes the Dispatcher to create one worker thread 408* per CPU in the system. When the Dispatcher is created with 409* only one thread, the Dispatcher guarantees to deliver posted 410* messages in order. When the Dispatcher is created with more 411* than one thread, messages may be delivered out of order. 412* 413* name 414* [in] Name to associate with the threads. The name may be up to 16 415* characters, including a terminating null character. All threads 416* created in the Dispatcher have the same name. 417* 418* RETURN VALUE 419* CL_SUCCESS if the operation is successful. 420* 421* SEE ALSO 422* Dispatcher, cl_disp_destoy, cl_disp_register, cl_disp_unregister, 423* cl_disp_post 424*********/ 425 426/****f* Component Library: Dispatcher/cl_disp_shutdown 427* NAME 428* cl_disp_shutdown 429* 430* DESCRIPTION 431* This function shutdown a Dispatcher object. So it unreg all messages and 432* clears the fifo and waits for the threads to exit 433* 434* SYNOPSIS 435*/ 436void cl_disp_shutdown(IN cl_dispatcher_t * const p_disp); 437/* 438* PARAMETERS 439* p_disp 440* [in] Pointer to a Dispatcher. 441* 442* RETURN VALUE 443* This function does not return a value. 444* 445* NOTES 446* This function does not returns until all worker threads 447* have exited client callback functions and been successfully 448* shutdowned. 449* 450* SEE ALSO 451* Dispatcher, cl_disp_construct, cl_disp_init 452*********/ 453 454/****f* Component Library: Dispatcher/cl_disp_destroy 455* NAME 456* cl_disp_destroy 457* 458* DESCRIPTION 459* This function destroys a Dispatcher object. 460* 461* SYNOPSIS 462*/ 463void cl_disp_destroy(IN cl_dispatcher_t * const p_disp); 464/* 465* PARAMETERS 466* p_disp 467* [in] Pointer to a Dispatcher. 468* 469* RETURN VALUE 470* This function does not return a value. 471* 472* SEE ALSO 473* Dispatcher, cl_disp_construct, cl_disp_init 474*********/ 475 476/****f* Component Library: Dispatcher/cl_disp_register 477* NAME 478* cl_disp_register 479* 480* DESCRIPTION 481* This function registers a client with a Dispatcher object. 482* 483* SYNOPSIS 484*/ 485cl_disp_reg_handle_t 486cl_disp_register(IN cl_dispatcher_t * const p_disp, 487 IN const cl_disp_msgid_t msg_id, 488 IN cl_pfn_msgrcv_cb_t pfn_callback OPTIONAL, 489 IN const void *const context); 490/* 491* PARAMETERS 492* p_disp 493* [in] Pointer to a Dispatcher. 494* 495* msg_id 496* [in] Numberic message ID for which the client is registering. 497* If the client does not wish to receive any messages, 498* (a send-only client) then the caller should set this value 499* to CL_DISP_MSGID_NONE. For efficiency, numeric message msg_id 500* values should start with 0 and should be contiguous, or nearly so. 501* 502* pfn_callback 503* [in] Message receive callback. The Dispatcher calls this 504* function after receiving a posted message with the 505* appropriate message msg_id value. Send-only clients may specify 506* NULL for this value. 507* 508* context 509* [in] Client context value passed to the cl_pfn_msgrcv_cb_t 510* function. 511* 512* RETURN VALUE 513* On success a Dispatcher registration handle. 514* CL_CL_DISP_INVALID_HANDLE otherwise. 515* 516* SEE ALSO 517* Dispatcher, cl_disp_unregister, cl_disp_post 518*********/ 519 520/****f* Component Library: Dispatcher/cl_disp_unregister 521* NAME 522* cl_disp_unregister 523* 524* DESCRIPTION 525* This function unregisters a client from a Dispatcher. 526* 527* SYNOPSIS 528*/ 529void cl_disp_unregister(IN const cl_disp_reg_handle_t handle); 530/* 531* PARAMETERS 532* handle 533* [in] cl_disp_reg_handle_t value return by cl_disp_register. 534* 535* RETURN VALUE 536* This function does not return a value. 537* 538* NOTES 539* This function will not return until worker threads have exited 540* the callback functions for this client. Do not invoke this 541* function from a callback. 542* 543* SEE ALSO 544* Dispatcher, cl_disp_register 545*********/ 546 547/****f* Component Library: Dispatcher/cl_disp_post 548* NAME 549* cl_disp_post 550* 551* DESCRIPTION 552* This function posts a message to a Dispatcher object. 553* 554* SYNOPSIS 555*/ 556cl_status_t 557cl_disp_post(IN const cl_disp_reg_handle_t handle, 558 IN const cl_disp_msgid_t msg_id, 559 IN const void *const p_data, 560 IN cl_pfn_msgdone_cb_t pfn_callback OPTIONAL, 561 IN const void *const context); 562/* 563* PARAMETERS 564* handle 565* [in] cl_disp_reg_handle_t value return by cl_disp_register. 566* 567* msg_id 568* [in] Numeric message msg_id value associated with this message. 569* 570* p_data 571* [in] Data payload for this message. 572* 573* pfn_callback 574* [in] Pointer to a cl_pfn_msgdone_cb_t function. 575* The Dispatcher calls this function after the message has been 576* processed by the recipient. 577* The caller may pass NULL for this value, which indicates no 578* message done callback is necessary. 579* 580* context 581* [in] Client context value passed to the cl_pfn_msgdone_cb_t 582* function. 583* 584* RETURN VALUE 585* CL_SUCCESS if the message was successfully queued in the Dispatcher. 586* 587* NOTES 588* The caller must not modify the memory pointed to by p_data until 589* the Dispatcher call the pfn_callback function. 590* 591* SEE ALSO 592* Dispatcher 593*********/ 594 595/****f* Component Library: Dispatcher/cl_disp_get_queue_status 596* NAME 597* cl_disp_get_queue_status 598* 599* DESCRIPTION 600* This function posts a message to a Dispatcher object. 601* 602* SYNOPSIS 603*/ 604void 605cl_disp_get_queue_status(IN const cl_disp_reg_handle_t handle, 606 OUT uint32_t * p_num_queued_msgs, 607 OUT uint64_t * p_last_msg_queue_time_ms); 608/* 609* PARAMETERS 610* handle 611* [in] cl_disp_reg_handle_t value return by cl_disp_register. 612* 613* p_last_msg_queue_time_ms 614* [out] pointer to a variable to hold the time the last popped up message 615* spent in the queue 616* 617* p_num_queued_msgs 618* [out] number of messages in the queue 619* 620* RETURN VALUE 621* Thr time the last popped up message stayed in the queue, in msec 622* 623* NOTES 624* Extarnel Locking is not required. 625* 626* SEE ALSO 627* Dispatcher 628*********/ 629 630END_C_DECLS 631#endif /* !defined(_CL_DISPATCHER_H_) */ 632