1/*
2 * Copyright (c) 2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1.  Redistributions of source code must retain the above copyright
11 *     notice, this list of conditions and the following disclaimer.
12 * 2.  Redistributions in binary form must reproduce the above copyright
13 *     notice, this list of conditions and the following disclaimer in the
14 *     documentation and/or other materials provided with the distribution.
15 * 3.  Neither the name of Apple Inc. ("Apple") nor the names of its
16 *     contributors may be used to endorse or promote products derived from
17 *     this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 * Portions of this software have been released under the following terms:
31 *
32 * (c) Copyright 1989-1993 OPEN SOFTWARE FOUNDATION, INC.
33 * (c) Copyright 1989-1993 HEWLETT-PACKARD COMPANY
34 * (c) Copyright 1989-1993 DIGITAL EQUIPMENT CORPORATION
35 *
36 * To anyone who acknowledges that this file is provided "AS IS"
37 * without any express or implied warranty:
38 * permission to use, copy, modify, and distribute this file for any
39 * purpose is hereby granted without fee, provided that the above
40 * copyright notices and this notice appears in all source code copies,
41 * and that none of the names of Open Software Foundation, Inc., Hewlett-
42 * Packard Company or Digital Equipment Corporation be used
43 * in advertising or publicity pertaining to distribution of the software
44 * without specific, written prior permission.  Neither Open Software
45 * Foundation, Inc., Hewlett-Packard Company nor Digital
46 * Equipment Corporation makes any representations about the suitability
47 * of this software for any purpose.
48 *
49 * Copyright (c) 2007, Novell, Inc. All rights reserved.
50 * Redistribution and use in source and binary forms, with or without
51 * modification, are permitted provided that the following conditions
52 * are met:
53 *
54 * 1.  Redistributions of source code must retain the above copyright
55 *     notice, this list of conditions and the following disclaimer.
56 * 2.  Redistributions in binary form must reproduce the above copyright
57 *     notice, this list of conditions and the following disclaimer in the
58 *     documentation and/or other materials provided with the distribution.
59 * 3.  Neither the name of Novell Inc. nor the names of its contributors
60 *     may be used to endorse or promote products derived from this
61 *     this software without specific prior written permission.
62 *
63 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
64 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
65 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
66 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY
67 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
68 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
69 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
70 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
71 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
72 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
73 *
74 * @APPLE_LICENSE_HEADER_END@
75 */
76
77/*
78**
79**  NAME:
80**
81**      dgpkt.h
82**
83**  FACILITY:
84**
85**      Remote Procedure Call (RPC)
86**
87**  ABSTRACT:
88**
89**  Routines for managing the datagram packet pool.
90**
91**
92*/
93
94#ifndef _DGPKT_H
95#define _DGPKT_H
96
97/* ========================================================================= */
98/*
99 * Datagram Packet Rationing
100 *
101 * Every active call requires a packet pool "reservation".  The purpose
102 * of this reservation is to guarantee that the call can always make
103 * progress.
104 *
105 * "Progress" is defined as the ability to exchange at least one packet
106 * at a time over a conversation.
107 *
108 * "Rationing" is the condition where calls are only allowed to use
109 * their reserved packet.
110 *
111 * The condition that invokes rationing is: the number of available
112 * packets in the free pool is less than or equal to the number of
113 * packet reservations.
114 */
115
116/*
117 * Define the maximum number of packet's (used for both xqe's and rqe's)
118 * that we'll ever allocate.
119 */
120
121#ifndef RPC_C_DG_PKT_MAX
122#define RPC_C_DG_PKT_MAX    100000
123#endif
124
125/*
126 * Define the number of packets that should be allocated during
127 * initialization to seed the free list.
128 */
129
130#ifndef RPC_C_DG_PKT_INIT_CNT
131#define RPC_C_DG_PKT_INIT_CNT       48
132#endif
133
134/*
135 * One of the conditions on which the packet rationing scheme is based is
136 * that, when rationing is necessary, all RPC threads will be able to drain
137 * their queues and use only the packets they have reserved.  One possible
138 * scenario is which this wouldn't be the case is if in a group of N
139 * inter-communicating processes (where N can be 1, in the case of KRPC)
140 * all packets got queued up by clients before the rationing state was
141 * entered.  In such a case, there would be no reservations to grant to
142 * the server sides of the calls, and thus no way for the clients to drain.
143 *
144 * Avoiding this potential deadlock requires that we guarantee that at least
145 * one server call can be started for such a group of N processes.  Since N can
146 * be 1, it follows that every process must be guaranteed that it can always
147 * start at least one server call.
148 *
149 * We guarantee the capability of running server calls by setting aside
150 * reservations for that purpose at startup.  A single reservation would be
151 * sufficient to guarantee the correctness of the algorithm, but making a few
152 * reservations will increase the fairness in the worst of cases.
153 */
154#ifndef RPC_C_DG_PKT_INIT_SERVER_RESVS
155#define RPC_C_DG_PKT_INIT_SERVER_RESVS  5
156#endif
157
158/*
159 * The  following structure is used to build the free packet pool.  Since
160 * the pool contains both xqe's and rqe's, we need a third, neutral pointer
161 * which can be used to access either type of transmission element.
162 */
163
164typedef struct rpc_dg_pkt_pool_elt_t {
165    union {
166        struct rpc_dg_pkt_pool_elt_t *next; /* for building a free list */
167        struct {
168            rpc_dg_xmitq_elt_t xqe;
169            rpc_dg_pkt_body_t  pkt;
170        } xqe;                             /* for handling xqe's       */
171        struct {
172            rpc_dg_recvq_elt_t rqe;
173            rpc_dg_raw_pkt_t   pkt;
174            unsigned32         pad1;       /* Need some padding to get */
175            unsigned32         pad2;       /* 0 MOD 8 byte alignment   */
176            rpc_dg_sock_pool_elt_p_t sock_ref;
177        } rqe;                             /* for handling rqe's       */
178    } u;
179    boolean is_on_free_list;
180} rpc_dg_pkt_pool_elt_t, *rpc_dg_pkt_pool_elt_p_t;
181
182/*
183 * The packet pool is a global structure which contains:
184 *       - a mutex to lock when modifying this structure
185 *       - the max # of packets that can be allocated
186 *       - a counter of the number of packets remaining to be allocated
187 *       - number of packet reservations currently held
188 *       - counters of the number of currently active xqe's/rqe's
189 *         and failed rqe allocations (this should always be 0 if things
190 *         are working) and xqe allocation attempts that had to block.
191 *       - a counter of the number of packets on the free list
192 *       - a linked list of free packets
193 *       - a pointer to the tail of the free list
194 *       - pointers to the head and tail of a linked list of call
195 *         handles that are waiting to be signalled when a packet
196 *         becomes available.
197 *       - pointers to the head and tail of a linked list of call
198 *         handles that are waiting to be signalled when a packet
199 *         reservation becomes available.
200 */
201
202typedef struct rpc_dg_pkt_pool_t {
203    rpc_mutex_t             pkt_mutex;
204    unsigned32              max_pkt_count;
205    unsigned32              pkt_count;
206    unsigned32              reservations;
207    unsigned32              srv_resv_avail;
208    unsigned32              max_resv_pkt; /* # of pkts for the largest fragment */
209    unsigned32              active_rqes;
210    unsigned32              active_xqes;
211    unsigned32              failed_alloc_rqe;
212    unsigned32              blocked_alloc_xqe;
213    unsigned32              free_count;
214    unsigned                is_rationing: 1;    /* T => is rationing */
215    unsigned                low_on_pkts: 1;     /* T => pkt pool is low */
216    rpc_dg_pkt_pool_elt_p_t free_list;
217    rpc_dg_pkt_pool_elt_p_t free_list_tail;
218    rpc_dg_call_p_t pkt_waiters_head, pkt_waiters_tail;
219    rpc_dg_call_p_t rsv_waiters_head, rsv_waiters_tail;
220} rpc_dg_pkt_pool_t;
221
222EXTERNAL rpc_dg_pkt_pool_t rpc_g_dg_pkt_pool;
223
224/*
225 * Macros for locking/unlocking the packet pool's mutex.
226 */
227
228#define RPC_DG_PKT_POOL_LOCK(junk)    RPC_MUTEX_LOCK(rpc_g_dg_pkt_pool.pkt_mutex)
229#define RPC_DG_PKT_POOL_UNLOCK(junk)  RPC_MUTEX_UNLOCK(rpc_g_dg_pkt_pool.pkt_mutex)
230#define RPC_DG_PKT_POOL_LOCK_ASSERT(junk) \
231                          RPC_MUTEX_LOCK_ASSERT(rpc_g_dg_pkt_pool.pkt_mutex)
232
233/*
234 * Macro to determine if we are in the packet rationing state.
235 *
236 * This macro requires that the packet pool is locked.
237 */
238
239#define RPC_DG_PKT_RATIONING(junk) \
240        (rpc_g_dg_pkt_pool.free_count + rpc_g_dg_pkt_pool.pkt_count <= \
241                   rpc_g_dg_pkt_pool.reservations)
242
243/* ========================================================================= */
244
245PRIVATE void rpc__dg_pkt_pool_init    (void);
246
247PRIVATE void rpc__dg_pkt_pool_fork_handler    (
248        rpc_fork_stage_id_t  /*stage*/
249    );
250
251PRIVATE rpc_dg_xmitq_elt_p_t rpc__dg_pkt_alloc_xqe    (
252        rpc_dg_call_p_t  /*call*/,
253        unsigned32 * /*st*/
254    );
255
256PRIVATE rpc_dg_recvq_elt_p_t rpc__dg_pkt_alloc_rqe    (
257        rpc_dg_ccall_p_t  /*ccall*/
258    );
259
260PRIVATE void rpc__dg_pkt_free_xqe    (
261        rpc_dg_xmitq_elt_p_t  /*pkt*/,
262        rpc_dg_call_p_t  /*call*/
263    );
264
265PRIVATE void rpc__dg_pkt_free_rqe_for_stub    (
266        rpc_dg_recvq_elt_p_t  /*pkt*/
267    );
268
269PRIVATE void rpc__dg_pkt_free_rqe    (
270        rpc_dg_recvq_elt_p_t  /*pkt*/,
271        rpc_dg_call_p_t  /*call*/
272    );
273
274PRIVATE boolean32 rpc__dg_pkt_adjust_reservation    (
275        rpc_dg_call_p_t  /*call*/,
276        unsigned32 /*nreq*/,
277        boolean32  /*block*/
278    );
279
280PRIVATE void rpc__dg_pkt_cancel_reservation    (
281        rpc_dg_call_p_t  /*call*/
282    );
283
284PRIVATE boolean32 rpc__dg_pkt_is_rationing    (
285        boolean32 * /*low_on_pkts*/
286    );
287#endif /* _DGPKT_H */
288