1/*-
2 * Copyright (c) 2010 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Edward Tomasz Napierala under sponsorship
6 * from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: stable/11/sys/kern/kern_racct.c 335536 2018-06-22 09:18:38Z avg $
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: stable/11/sys/kern/kern_racct.c 335536 2018-06-22 09:18:38Z avg $");
34
35#include "opt_sched.h"
36
37#include <sys/param.h>
38#include <sys/buf.h>
39#include <sys/systm.h>
40#include <sys/eventhandler.h>
41#include <sys/jail.h>
42#include <sys/kernel.h>
43#include <sys/kthread.h>
44#include <sys/lock.h>
45#include <sys/loginclass.h>
46#include <sys/malloc.h>
47#include <sys/mutex.h>
48#include <sys/proc.h>
49#include <sys/racct.h>
50#include <sys/resourcevar.h>
51#include <sys/sbuf.h>
52#include <sys/sched.h>
53#include <sys/sdt.h>
54#include <sys/smp.h>
55#include <sys/sx.h>
56#include <sys/sysctl.h>
57#include <sys/sysent.h>
58#include <sys/sysproto.h>
59#include <sys/umtx.h>
60#include <machine/smp.h>
61
62#ifdef RCTL
63#include <sys/rctl.h>
64#endif
65
66#ifdef RACCT
67
68FEATURE(racct, "Resource Accounting");
69
70/*
71 * Do not block processes that have their %cpu usage <= pcpu_threshold.
72 */
73static int pcpu_threshold = 1;
74#ifdef RACCT_DEFAULT_TO_DISABLED
75int racct_enable = 0;
76#else
77int racct_enable = 1;
78#endif
79
80SYSCTL_NODE(_kern, OID_AUTO, racct, CTLFLAG_RW, 0, "Resource Accounting");
81SYSCTL_UINT(_kern_racct, OID_AUTO, enable, CTLFLAG_RDTUN, &racct_enable,
82    0, "Enable RACCT/RCTL");
83SYSCTL_UINT(_kern_racct, OID_AUTO, pcpu_threshold, CTLFLAG_RW, &pcpu_threshold,
84    0, "Processes with higher %cpu usage than this value can be throttled.");
85
86/*
87 * How many seconds it takes to use the scheduler %cpu calculations.  When a
88 * process starts, we compute its %cpu usage by dividing its runtime by the
89 * process wall clock time.  After RACCT_PCPU_SECS pass, we use the value
90 * provided by the scheduler.
91 */
92#define RACCT_PCPU_SECS		3
93
94struct mtx racct_lock;
95MTX_SYSINIT(racct_lock, &racct_lock, "racct lock", MTX_DEF);
96
97static uma_zone_t racct_zone;
98
99static void racct_sub_racct(struct racct *dest, const struct racct *src);
100static void racct_sub_cred_locked(struct ucred *cred, int resource,
101		uint64_t amount);
102static void racct_add_cred_locked(struct ucred *cred, int resource,
103		uint64_t amount);
104
105SDT_PROVIDER_DEFINE(racct);
106SDT_PROBE_DEFINE3(racct, , rusage, add,
107    "struct proc *", "int", "uint64_t");
108SDT_PROBE_DEFINE3(racct, , rusage, add__failure,
109    "struct proc *", "int", "uint64_t");
110SDT_PROBE_DEFINE3(racct, , rusage, add__buf,
111    "struct proc *", "const struct buf *", "int");
112SDT_PROBE_DEFINE3(racct, , rusage, add__cred,
113    "struct ucred *", "int", "uint64_t");
114SDT_PROBE_DEFINE3(racct, , rusage, add__force,
115    "struct proc *", "int", "uint64_t");
116SDT_PROBE_DEFINE3(racct, , rusage, set,
117    "struct proc *", "int", "uint64_t");
118SDT_PROBE_DEFINE3(racct, , rusage, set__failure,
119    "struct proc *", "int", "uint64_t");
120SDT_PROBE_DEFINE3(racct, , rusage, set__force,
121    "struct proc *", "int", "uint64_t");
122SDT_PROBE_DEFINE3(racct, , rusage, sub,
123    "struct proc *", "int", "uint64_t");
124SDT_PROBE_DEFINE3(racct, , rusage, sub__cred,
125    "struct ucred *", "int", "uint64_t");
126SDT_PROBE_DEFINE1(racct, , racct, create,
127    "struct racct *");
128SDT_PROBE_DEFINE1(racct, , racct, destroy,
129    "struct racct *");
130SDT_PROBE_DEFINE2(racct, , racct, join,
131    "struct racct *", "struct racct *");
132SDT_PROBE_DEFINE2(racct, , racct, join__failure,
133    "struct racct *", "struct racct *");
134SDT_PROBE_DEFINE2(racct, , racct, leave,
135    "struct racct *", "struct racct *");
136
137int racct_types[] = {
138	[RACCT_CPU] =
139		RACCT_IN_MILLIONS,
140	[RACCT_DATA] =
141		RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
142	[RACCT_STACK] =
143		RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
144	[RACCT_CORE] =
145		RACCT_DENIABLE,
146	[RACCT_RSS] =
147		RACCT_RECLAIMABLE,
148	[RACCT_MEMLOCK] =
149		RACCT_RECLAIMABLE | RACCT_DENIABLE,
150	[RACCT_NPROC] =
151		RACCT_RECLAIMABLE | RACCT_DENIABLE,
152	[RACCT_NOFILE] =
153		RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
154	[RACCT_VMEM] =
155		RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
156	[RACCT_NPTS] =
157		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
158	[RACCT_SWAP] =
159		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
160	[RACCT_NTHR] =
161		RACCT_RECLAIMABLE | RACCT_DENIABLE,
162	[RACCT_MSGQQUEUED] =
163		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
164	[RACCT_MSGQSIZE] =
165		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
166	[RACCT_NMSGQ] =
167		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
168	[RACCT_NSEM] =
169		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
170	[RACCT_NSEMOP] =
171		RACCT_RECLAIMABLE | RACCT_INHERITABLE | RACCT_DENIABLE,
172	[RACCT_NSHM] =
173		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
174	[RACCT_SHMSIZE] =
175		RACCT_RECLAIMABLE | RACCT_DENIABLE | RACCT_SLOPPY,
176	[RACCT_WALLCLOCK] =
177		RACCT_IN_MILLIONS,
178	[RACCT_PCTCPU] =
179		RACCT_DECAYING | RACCT_DENIABLE | RACCT_IN_MILLIONS,
180	[RACCT_READBPS] =
181		RACCT_DECAYING,
182	[RACCT_WRITEBPS] =
183		RACCT_DECAYING,
184	[RACCT_READIOPS] =
185		RACCT_DECAYING,
186	[RACCT_WRITEIOPS] =
187		RACCT_DECAYING };
188
189static const fixpt_t RACCT_DECAY_FACTOR = 0.3 * FSCALE;
190
191#ifdef SCHED_4BSD
192/*
193 * Contains intermediate values for %cpu calculations to avoid using floating
194 * point in the kernel.
195 * ccpu_exp[k] = FSCALE * (ccpu/FSCALE)^k = FSCALE * exp(-k/20)
196 * It is needed only for the 4BSD scheduler, because in ULE, the ccpu equals to
197 * zero so the calculations are more straightforward.
198 */
199fixpt_t ccpu_exp[] = {
200	[0] = FSCALE * 1,
201	[1] = FSCALE * 0.95122942450071400909,
202	[2] = FSCALE * 0.90483741803595957316,
203	[3] = FSCALE * 0.86070797642505780722,
204	[4] = FSCALE * 0.81873075307798185866,
205	[5] = FSCALE * 0.77880078307140486824,
206	[6] = FSCALE * 0.74081822068171786606,
207	[7] = FSCALE * 0.70468808971871343435,
208	[8] = FSCALE * 0.67032004603563930074,
209	[9] = FSCALE * 0.63762815162177329314,
210	[10] = FSCALE * 0.60653065971263342360,
211	[11] = FSCALE * 0.57694981038048669531,
212	[12] = FSCALE * 0.54881163609402643262,
213	[13] = FSCALE * 0.52204577676101604789,
214	[14] = FSCALE * 0.49658530379140951470,
215	[15] = FSCALE * 0.47236655274101470713,
216	[16] = FSCALE * 0.44932896411722159143,
217	[17] = FSCALE * 0.42741493194872666992,
218	[18] = FSCALE * 0.40656965974059911188,
219	[19] = FSCALE * 0.38674102345450120691,
220	[20] = FSCALE * 0.36787944117144232159,
221	[21] = FSCALE * 0.34993774911115535467,
222	[22] = FSCALE * 0.33287108369807955328,
223	[23] = FSCALE * 0.31663676937905321821,
224	[24] = FSCALE * 0.30119421191220209664,
225	[25] = FSCALE * 0.28650479686019010032,
226	[26] = FSCALE * 0.27253179303401260312,
227	[27] = FSCALE * 0.25924026064589150757,
228	[28] = FSCALE * 0.24659696394160647693,
229	[29] = FSCALE * 0.23457028809379765313,
230	[30] = FSCALE * 0.22313016014842982893,
231	[31] = FSCALE * 0.21224797382674305771,
232	[32] = FSCALE * 0.20189651799465540848,
233	[33] = FSCALE * 0.19204990862075411423,
234	[34] = FSCALE * 0.18268352405273465022,
235	[35] = FSCALE * 0.17377394345044512668,
236	[36] = FSCALE * 0.16529888822158653829,
237	[37] = FSCALE * 0.15723716631362761621,
238	[38] = FSCALE * 0.14956861922263505264,
239	[39] = FSCALE * 0.14227407158651357185,
240	[40] = FSCALE * 0.13533528323661269189,
241	[41] = FSCALE * 0.12873490358780421886,
242	[42] = FSCALE * 0.12245642825298191021,
243	[43] = FSCALE * 0.11648415777349695786,
244	[44] = FSCALE * 0.11080315836233388333,
245	[45] = FSCALE * 0.10539922456186433678,
246	[46] = FSCALE * 0.10025884372280373372,
247	[47] = FSCALE * 0.09536916221554961888,
248	[48] = FSCALE * 0.09071795328941250337,
249	[49] = FSCALE * 0.08629358649937051097,
250	[50] = FSCALE * 0.08208499862389879516,
251	[51] = FSCALE * 0.07808166600115315231,
252	[52] = FSCALE * 0.07427357821433388042,
253	[53] = FSCALE * 0.07065121306042958674,
254	[54] = FSCALE * 0.06720551273974976512,
255	[55] = FSCALE * 0.06392786120670757270,
256	[56] = FSCALE * 0.06081006262521796499,
257	[57] = FSCALE * 0.05784432087483846296,
258	[58] = FSCALE * 0.05502322005640722902,
259	[59] = FSCALE * 0.05233970594843239308,
260	[60] = FSCALE * 0.04978706836786394297,
261	[61] = FSCALE * 0.04735892439114092119,
262	[62] = FSCALE * 0.04504920239355780606,
263	[63] = FSCALE * 0.04285212686704017991,
264	[64] = FSCALE * 0.04076220397836621516,
265	[65] = FSCALE * 0.03877420783172200988,
266	[66] = FSCALE * 0.03688316740124000544,
267	[67] = FSCALE * 0.03508435410084502588,
268	[68] = FSCALE * 0.03337326996032607948,
269	[69] = FSCALE * 0.03174563637806794323,
270	[70] = FSCALE * 0.03019738342231850073,
271	[71] = FSCALE * 0.02872463965423942912,
272	[72] = FSCALE * 0.02732372244729256080,
273	[73] = FSCALE * 0.02599112877875534358,
274	[74] = FSCALE * 0.02472352647033939120,
275	[75] = FSCALE * 0.02351774585600910823,
276	[76] = FSCALE * 0.02237077185616559577,
277	[77] = FSCALE * 0.02127973643837716938,
278	[78] = FSCALE * 0.02024191144580438847,
279	[79] = FSCALE * 0.01925470177538692429,
280	[80] = FSCALE * 0.01831563888873418029,
281	[81] = FSCALE * 0.01742237463949351138,
282	[82] = FSCALE * 0.01657267540176124754,
283	[83] = FSCALE * 0.01576441648485449082,
284	[84] = FSCALE * 0.01499557682047770621,
285	[85] = FSCALE * 0.01426423390899925527,
286	[86] = FSCALE * 0.01356855901220093175,
287	[87] = FSCALE * 0.01290681258047986886,
288	[88] = FSCALE * 0.01227733990306844117,
289	[89] = FSCALE * 0.01167856697039544521,
290	[90] = FSCALE * 0.01110899653824230649,
291	[91] = FSCALE * 0.01056720438385265337,
292	[92] = FSCALE * 0.01005183574463358164,
293	[93] = FSCALE * 0.00956160193054350793,
294	[94] = FSCALE * 0.00909527710169581709,
295	[95] = FSCALE * 0.00865169520312063417,
296	[96] = FSCALE * 0.00822974704902002884,
297	[97] = FSCALE * 0.00782837754922577143,
298	[98] = FSCALE * 0.00744658307092434051,
299	[99] = FSCALE * 0.00708340892905212004,
300	[100] = FSCALE * 0.00673794699908546709,
301	[101] = FSCALE * 0.00640933344625638184,
302	[102] = FSCALE * 0.00609674656551563610,
303	[103] = FSCALE * 0.00579940472684214321,
304	[104] = FSCALE * 0.00551656442076077241,
305	[105] = FSCALE * 0.00524751839918138427,
306	[106] = FSCALE * 0.00499159390691021621,
307	[107] = FSCALE * 0.00474815099941147558,
308	[108] = FSCALE * 0.00451658094261266798,
309	[109] = FSCALE * 0.00429630469075234057,
310	[110] = FSCALE * 0.00408677143846406699,
311};
312#endif
313
314#define	CCPU_EXP_MAX	110
315
316/*
317 * This function is analogical to the getpcpu() function in the ps(1) command.
318 * They should both calculate in the same way so that the racct %cpu
319 * calculations are consistent with the values showed by the ps(1) tool.
320 * The calculations are more complex in the 4BSD scheduler because of the value
321 * of the ccpu variable.  In ULE it is defined to be zero which saves us some
322 * work.
323 */
324static uint64_t
325racct_getpcpu(struct proc *p, u_int pcpu)
326{
327	u_int swtime;
328#ifdef SCHED_4BSD
329	fixpt_t pctcpu, pctcpu_next;
330#endif
331#ifdef SMP
332	struct pcpu *pc;
333	int found;
334#endif
335	fixpt_t p_pctcpu;
336	struct thread *td;
337
338	ASSERT_RACCT_ENABLED();
339
340	/*
341	 * If the process is swapped out, we count its %cpu usage as zero.
342	 * This behaviour is consistent with the userland ps(1) tool.
343	 */
344	if ((p->p_flag & P_INMEM) == 0)
345		return (0);
346	swtime = (ticks - p->p_swtick) / hz;
347
348	/*
349	 * For short-lived processes, the sched_pctcpu() returns small
350	 * values even for cpu intensive processes.  Therefore we use
351	 * our own estimate in this case.
352	 */
353	if (swtime < RACCT_PCPU_SECS)
354		return (pcpu);
355
356	p_pctcpu = 0;
357	FOREACH_THREAD_IN_PROC(p, td) {
358		if (td == PCPU_GET(idlethread))
359			continue;
360#ifdef SMP
361		found = 0;
362		STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
363			if (td == pc->pc_idlethread) {
364				found = 1;
365				break;
366			}
367		}
368		if (found)
369			continue;
370#endif
371		thread_lock(td);
372#ifdef SCHED_4BSD
373		pctcpu = sched_pctcpu(td);
374		/* Count also the yet unfinished second. */
375		pctcpu_next = (pctcpu * ccpu_exp[1]) >> FSHIFT;
376		pctcpu_next += sched_pctcpu_delta(td);
377		p_pctcpu += max(pctcpu, pctcpu_next);
378#else
379		/*
380		 * In ULE the %cpu statistics are updated on every
381		 * sched_pctcpu() call.  So special calculations to
382		 * account for the latest (unfinished) second are
383		 * not needed.
384		 */
385		p_pctcpu += sched_pctcpu(td);
386#endif
387		thread_unlock(td);
388	}
389
390#ifdef SCHED_4BSD
391	if (swtime <= CCPU_EXP_MAX)
392		return ((100 * (uint64_t)p_pctcpu * 1000000) /
393		    (FSCALE - ccpu_exp[swtime]));
394#endif
395
396	return ((100 * (uint64_t)p_pctcpu * 1000000) / FSCALE);
397}
398
399static void
400racct_add_racct(struct racct *dest, const struct racct *src)
401{
402	int i;
403
404	ASSERT_RACCT_ENABLED();
405	RACCT_LOCK_ASSERT();
406
407	/*
408	 * Update resource usage in dest.
409	 */
410	for (i = 0; i <= RACCT_MAX; i++) {
411		KASSERT(dest->r_resources[i] >= 0,
412		    ("%s: resource %d propagation meltdown: dest < 0",
413		    __func__, i));
414		KASSERT(src->r_resources[i] >= 0,
415		    ("%s: resource %d propagation meltdown: src < 0",
416		    __func__, i));
417		dest->r_resources[i] += src->r_resources[i];
418	}
419}
420
421static void
422racct_sub_racct(struct racct *dest, const struct racct *src)
423{
424	int i;
425
426	ASSERT_RACCT_ENABLED();
427	RACCT_LOCK_ASSERT();
428
429	/*
430	 * Update resource usage in dest.
431	 */
432	for (i = 0; i <= RACCT_MAX; i++) {
433		if (!RACCT_IS_SLOPPY(i) && !RACCT_IS_DECAYING(i)) {
434			KASSERT(dest->r_resources[i] >= 0,
435			    ("%s: resource %d propagation meltdown: dest < 0",
436			    __func__, i));
437			KASSERT(src->r_resources[i] >= 0,
438			    ("%s: resource %d propagation meltdown: src < 0",
439			    __func__, i));
440			KASSERT(src->r_resources[i] <= dest->r_resources[i],
441			    ("%s: resource %d propagation meltdown: src > dest",
442			    __func__, i));
443		}
444		if (RACCT_CAN_DROP(i)) {
445			dest->r_resources[i] -= src->r_resources[i];
446			if (dest->r_resources[i] < 0) {
447				KASSERT(RACCT_IS_SLOPPY(i) ||
448				    RACCT_IS_DECAYING(i),
449				    ("%s: resource %d usage < 0", __func__, i));
450				dest->r_resources[i] = 0;
451			}
452		}
453	}
454}
455
456void
457racct_create(struct racct **racctp)
458{
459
460	if (!racct_enable)
461		return;
462
463	SDT_PROBE1(racct, , racct, create, racctp);
464
465	KASSERT(*racctp == NULL, ("racct already allocated"));
466
467	*racctp = uma_zalloc(racct_zone, M_WAITOK | M_ZERO);
468}
469
470static void
471racct_destroy_locked(struct racct **racctp)
472{
473	struct racct *racct;
474	int i;
475
476	ASSERT_RACCT_ENABLED();
477
478	SDT_PROBE1(racct, , racct, destroy, racctp);
479
480	RACCT_LOCK_ASSERT();
481	KASSERT(racctp != NULL, ("NULL racctp"));
482	KASSERT(*racctp != NULL, ("NULL racct"));
483
484	racct = *racctp;
485
486	for (i = 0; i <= RACCT_MAX; i++) {
487		if (RACCT_IS_SLOPPY(i))
488			continue;
489		if (!RACCT_IS_RECLAIMABLE(i))
490			continue;
491		KASSERT(racct->r_resources[i] == 0,
492		    ("destroying non-empty racct: "
493		    "%ju allocated for resource %d\n",
494		    racct->r_resources[i], i));
495	}
496	uma_zfree(racct_zone, racct);
497	*racctp = NULL;
498}
499
500void
501racct_destroy(struct racct **racct)
502{
503
504	if (!racct_enable)
505		return;
506
507	RACCT_LOCK();
508	racct_destroy_locked(racct);
509	RACCT_UNLOCK();
510}
511
512/*
513 * Increase consumption of 'resource' by 'amount' for 'racct',
514 * but not its parents.  Differently from other cases, 'amount' here
515 * may be less than zero.
516 */
517static void
518racct_adjust_resource(struct racct *racct, int resource,
519    int64_t amount)
520{
521
522	ASSERT_RACCT_ENABLED();
523	RACCT_LOCK_ASSERT();
524	KASSERT(racct != NULL, ("NULL racct"));
525
526	racct->r_resources[resource] += amount;
527	if (racct->r_resources[resource] < 0) {
528		KASSERT(RACCT_IS_SLOPPY(resource) || RACCT_IS_DECAYING(resource),
529		    ("%s: resource %d usage < 0", __func__, resource));
530		racct->r_resources[resource] = 0;
531	}
532
533	/*
534	 * There are some cases where the racct %cpu resource would grow
535	 * beyond 100% per core.  For example in racct_proc_exit() we add
536	 * the process %cpu usage to the ucred racct containers.  If too
537	 * many processes terminated in a short time span, the ucred %cpu
538	 * resource could grow too much.  Also, the 4BSD scheduler sometimes
539	 * returns for a thread more than 100% cpu usage. So we set a sane
540	 * boundary here to 100% * the maxumum number of CPUs.
541	 */
542	if ((resource == RACCT_PCTCPU) &&
543	    (racct->r_resources[RACCT_PCTCPU] > 100 * 1000000 * (int64_t)MAXCPU))
544		racct->r_resources[RACCT_PCTCPU] = 100 * 1000000 * (int64_t)MAXCPU;
545}
546
547static int
548racct_add_locked(struct proc *p, int resource, uint64_t amount, int force)
549{
550#ifdef RCTL
551	int error;
552#endif
553
554	ASSERT_RACCT_ENABLED();
555
556	/*
557	 * We need proc lock to dereference p->p_ucred.
558	 */
559	PROC_LOCK_ASSERT(p, MA_OWNED);
560
561#ifdef RCTL
562	error = rctl_enforce(p, resource, amount);
563	if (error && !force && RACCT_IS_DENIABLE(resource)) {
564		SDT_PROBE3(racct, , rusage, add__failure, p, resource, amount);
565		return (error);
566	}
567#endif
568	racct_adjust_resource(p->p_racct, resource, amount);
569	racct_add_cred_locked(p->p_ucred, resource, amount);
570
571	return (0);
572}
573
574/*
575 * Increase allocation of 'resource' by 'amount' for process 'p'.
576 * Return 0 if it's below limits, or errno, if it's not.
577 */
578int
579racct_add(struct proc *p, int resource, uint64_t amount)
580{
581	int error;
582
583	if (!racct_enable)
584		return (0);
585
586	SDT_PROBE3(racct, , rusage, add, p, resource, amount);
587
588	RACCT_LOCK();
589	error = racct_add_locked(p, resource, amount, 0);
590	RACCT_UNLOCK();
591	return (error);
592}
593
594/*
595 * Increase allocation of 'resource' by 'amount' for process 'p'.
596 * Doesn't check for limits and never fails.
597 */
598void
599racct_add_force(struct proc *p, int resource, uint64_t amount)
600{
601
602	if (!racct_enable)
603		return;
604
605	SDT_PROBE3(racct, , rusage, add__force, p, resource, amount);
606
607	RACCT_LOCK();
608	racct_add_locked(p, resource, amount, 1);
609	RACCT_UNLOCK();
610}
611
612static void
613racct_add_cred_locked(struct ucred *cred, int resource, uint64_t amount)
614{
615	struct prison *pr;
616
617	ASSERT_RACCT_ENABLED();
618
619	racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, amount);
620	for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent)
621		racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource,
622		    amount);
623	racct_adjust_resource(cred->cr_loginclass->lc_racct, resource, amount);
624}
625
626/*
627 * Increase allocation of 'resource' by 'amount' for credential 'cred'.
628 * Doesn't check for limits and never fails.
629 */
630void
631racct_add_cred(struct ucred *cred, int resource, uint64_t amount)
632{
633
634	if (!racct_enable)
635		return;
636
637	SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount);
638
639	RACCT_LOCK();
640	racct_add_cred_locked(cred, resource, amount);
641	RACCT_UNLOCK();
642}
643
644/*
645 * Account for disk IO resource consumption.  Checks for limits,
646 * but never fails, due to disk limits being undeniable.
647 */
648void
649racct_add_buf(struct proc *p, const struct buf *bp, int is_write)
650{
651
652	ASSERT_RACCT_ENABLED();
653	PROC_LOCK_ASSERT(p, MA_OWNED);
654
655	SDT_PROBE3(racct, , rusage, add__buf, p, bp, is_write);
656
657	RACCT_LOCK();
658	if (is_write) {
659		racct_add_locked(curproc, RACCT_WRITEBPS, bp->b_bcount, 1);
660		racct_add_locked(curproc, RACCT_WRITEIOPS, 1, 1);
661	} else {
662		racct_add_locked(curproc, RACCT_READBPS, bp->b_bcount, 1);
663		racct_add_locked(curproc, RACCT_READIOPS, 1, 1);
664	}
665	RACCT_UNLOCK();
666}
667
668static int
669racct_set_locked(struct proc *p, int resource, uint64_t amount, int force)
670{
671	int64_t old_amount, decayed_amount, diff_proc, diff_cred;
672#ifdef RCTL
673	int error;
674#endif
675
676	ASSERT_RACCT_ENABLED();
677
678	/*
679	 * We need proc lock to dereference p->p_ucred.
680	 */
681	PROC_LOCK_ASSERT(p, MA_OWNED);
682
683	old_amount = p->p_racct->r_resources[resource];
684	/*
685	 * The diffs may be negative.
686	 */
687	diff_proc = amount - old_amount;
688	if (resource == RACCT_PCTCPU) {
689		/*
690		 * Resources in per-credential racct containers may decay.
691		 * If this is the case, we need to calculate the difference
692		 * between the new amount and the proportional value of the
693		 * old amount that has decayed in the ucred racct containers.
694		 */
695		decayed_amount = old_amount * RACCT_DECAY_FACTOR / FSCALE;
696		diff_cred = amount - decayed_amount;
697	} else
698		diff_cred = diff_proc;
699#ifdef notyet
700	KASSERT(diff_proc >= 0 || RACCT_CAN_DROP(resource),
701	    ("%s: usage of non-droppable resource %d dropping", __func__,
702	     resource));
703#endif
704#ifdef RCTL
705	if (diff_proc > 0) {
706		error = rctl_enforce(p, resource, diff_proc);
707		if (error && !force && RACCT_IS_DENIABLE(resource)) {
708			SDT_PROBE3(racct, , rusage, set__failure, p, resource,
709			    amount);
710			return (error);
711		}
712	}
713#endif
714	racct_adjust_resource(p->p_racct, resource, diff_proc);
715	if (diff_cred > 0)
716		racct_add_cred_locked(p->p_ucred, resource, diff_cred);
717	else if (diff_cred < 0)
718		racct_sub_cred_locked(p->p_ucred, resource, -diff_cred);
719
720	return (0);
721}
722
723/*
724 * Set allocation of 'resource' to 'amount' for process 'p'.
725 * Return 0 if it's below limits, or errno, if it's not.
726 *
727 * Note that decreasing the allocation always returns 0,
728 * even if it's above the limit.
729 */
730int
731racct_set(struct proc *p, int resource, uint64_t amount)
732{
733	int error;
734
735	if (!racct_enable)
736		return (0);
737
738	SDT_PROBE3(racct, , rusage, set__force, p, resource, amount);
739
740	RACCT_LOCK();
741	error = racct_set_locked(p, resource, amount, 0);
742	RACCT_UNLOCK();
743	return (error);
744}
745
746void
747racct_set_force(struct proc *p, int resource, uint64_t amount)
748{
749
750	if (!racct_enable)
751		return;
752
753	SDT_PROBE3(racct, , rusage, set, p, resource, amount);
754
755	RACCT_LOCK();
756	racct_set_locked(p, resource, amount, 1);
757	RACCT_UNLOCK();
758}
759
760/*
761 * Returns amount of 'resource' the process 'p' can keep allocated.
762 * Allocating more than that would be denied, unless the resource
763 * is marked undeniable.  Amount of already allocated resource does
764 * not matter.
765 */
766uint64_t
767racct_get_limit(struct proc *p, int resource)
768{
769#ifdef RCTL
770	uint64_t available;
771
772	if (!racct_enable)
773		return (UINT64_MAX);
774
775	RACCT_LOCK();
776	available = rctl_get_limit(p, resource);
777	RACCT_UNLOCK();
778
779	return (available);
780#else
781
782	return (UINT64_MAX);
783#endif
784}
785
786/*
787 * Returns amount of 'resource' the process 'p' can keep allocated.
788 * Allocating more than that would be denied, unless the resource
789 * is marked undeniable.  Amount of already allocated resource does
790 * matter.
791 */
792uint64_t
793racct_get_available(struct proc *p, int resource)
794{
795#ifdef RCTL
796	uint64_t available;
797
798	if (!racct_enable)
799		return (UINT64_MAX);
800
801	RACCT_LOCK();
802	available = rctl_get_available(p, resource);
803	RACCT_UNLOCK();
804
805	return (available);
806#else
807
808	return (UINT64_MAX);
809#endif
810}
811
812/*
813 * Returns amount of the %cpu resource that process 'p' can add to its %cpu
814 * utilization.  Adding more than that would lead to the process being
815 * throttled.
816 */
817static int64_t
818racct_pcpu_available(struct proc *p)
819{
820#ifdef RCTL
821	uint64_t available;
822
823	ASSERT_RACCT_ENABLED();
824
825	RACCT_LOCK();
826	available = rctl_pcpu_available(p);
827	RACCT_UNLOCK();
828
829	return (available);
830#else
831
832	return (INT64_MAX);
833#endif
834}
835
836/*
837 * Decrease allocation of 'resource' by 'amount' for process 'p'.
838 */
839void
840racct_sub(struct proc *p, int resource, uint64_t amount)
841{
842
843	if (!racct_enable)
844		return;
845
846	SDT_PROBE3(racct, , rusage, sub, p, resource, amount);
847
848	/*
849	 * We need proc lock to dereference p->p_ucred.
850	 */
851	PROC_LOCK_ASSERT(p, MA_OWNED);
852	KASSERT(RACCT_CAN_DROP(resource),
853	    ("%s: called for non-droppable resource %d", __func__, resource));
854
855	RACCT_LOCK();
856	KASSERT(amount <= p->p_racct->r_resources[resource],
857	    ("%s: freeing %ju of resource %d, which is more "
858	     "than allocated %jd for %s (pid %d)", __func__, amount, resource,
859	    (intmax_t)p->p_racct->r_resources[resource], p->p_comm, p->p_pid));
860
861	racct_adjust_resource(p->p_racct, resource, -amount);
862	racct_sub_cred_locked(p->p_ucred, resource, amount);
863	RACCT_UNLOCK();
864}
865
866static void
867racct_sub_cred_locked(struct ucred *cred, int resource, uint64_t amount)
868{
869	struct prison *pr;
870
871	ASSERT_RACCT_ENABLED();
872
873	racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, -amount);
874	for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent)
875		racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource,
876		    -amount);
877	racct_adjust_resource(cred->cr_loginclass->lc_racct, resource, -amount);
878}
879
880/*
881 * Decrease allocation of 'resource' by 'amount' for credential 'cred'.
882 */
883void
884racct_sub_cred(struct ucred *cred, int resource, uint64_t amount)
885{
886
887	if (!racct_enable)
888		return;
889
890	SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount);
891
892#ifdef notyet
893	KASSERT(RACCT_CAN_DROP(resource),
894	    ("%s: called for resource %d which can not drop", __func__,
895	     resource));
896#endif
897
898	RACCT_LOCK();
899	racct_sub_cred_locked(cred, resource, amount);
900	RACCT_UNLOCK();
901}
902
903/*
904 * Inherit resource usage information from the parent process.
905 */
906int
907racct_proc_fork(struct proc *parent, struct proc *child)
908{
909	int i, error = 0;
910
911	if (!racct_enable)
912		return (0);
913
914	/*
915	 * Create racct for the child process.
916	 */
917	racct_create(&child->p_racct);
918
919	PROC_LOCK(parent);
920	PROC_LOCK(child);
921	RACCT_LOCK();
922
923#ifdef RCTL
924	error = rctl_proc_fork(parent, child);
925	if (error != 0)
926		goto out;
927#endif
928
929	/* Init process cpu time. */
930	child->p_prev_runtime = 0;
931	child->p_throttled = 0;
932
933	/*
934	 * Inherit resource usage.
935	 */
936	for (i = 0; i <= RACCT_MAX; i++) {
937		if (parent->p_racct->r_resources[i] == 0 ||
938		    !RACCT_IS_INHERITABLE(i))
939			continue;
940
941		error = racct_set_locked(child, i,
942		    parent->p_racct->r_resources[i], 0);
943		if (error != 0)
944			goto out;
945	}
946
947	error = racct_add_locked(child, RACCT_NPROC, 1, 0);
948	error += racct_add_locked(child, RACCT_NTHR, 1, 0);
949
950out:
951	RACCT_UNLOCK();
952	PROC_UNLOCK(child);
953	PROC_UNLOCK(parent);
954
955	if (error != 0)
956		racct_proc_exit(child);
957
958	return (error);
959}
960
961/*
962 * Called at the end of fork1(), to handle rules that require the process
963 * to be fully initialized.
964 */
965void
966racct_proc_fork_done(struct proc *child)
967{
968
969	if (!racct_enable)
970		return;
971
972	PROC_LOCK_ASSERT(child, MA_OWNED);
973
974#ifdef RCTL
975	RACCT_LOCK();
976	rctl_enforce(child, RACCT_NPROC, 0);
977	rctl_enforce(child, RACCT_NTHR, 0);
978	RACCT_UNLOCK();
979#endif
980}
981
982void
983racct_proc_exit(struct proc *p)
984{
985	struct timeval wallclock;
986	uint64_t pct_estimate, pct, runtime;
987	int i;
988
989	if (!racct_enable)
990		return;
991
992	PROC_LOCK(p);
993	/*
994	 * We don't need to calculate rux, proc_reap() has already done this.
995	 */
996	runtime = cputick2usec(p->p_rux.rux_runtime);
997#ifdef notyet
998	KASSERT(runtime >= p->p_prev_runtime, ("runtime < p_prev_runtime"));
999#else
1000	if (runtime < p->p_prev_runtime)
1001		runtime = p->p_prev_runtime;
1002#endif
1003	microuptime(&wallclock);
1004	timevalsub(&wallclock, &p->p_stats->p_start);
1005	if (wallclock.tv_sec > 0 || wallclock.tv_usec > 0) {
1006		pct_estimate = (1000000 * runtime * 100) /
1007		    ((uint64_t)wallclock.tv_sec * 1000000 +
1008		    wallclock.tv_usec);
1009	} else
1010		pct_estimate = 0;
1011	pct = racct_getpcpu(p, pct_estimate);
1012
1013	RACCT_LOCK();
1014	racct_set_locked(p, RACCT_CPU, runtime, 0);
1015	racct_add_cred_locked(p->p_ucred, RACCT_PCTCPU, pct);
1016
1017	KASSERT(p->p_racct->r_resources[RACCT_RSS] == 0,
1018	    ("process reaped with %ju allocated for RSS\n",
1019	    p->p_racct->r_resources[RACCT_RSS]));
1020	for (i = 0; i <= RACCT_MAX; i++) {
1021		if (p->p_racct->r_resources[i] == 0)
1022			continue;
1023		if (!RACCT_IS_RECLAIMABLE(i))
1024			continue;
1025		racct_set_locked(p, i, 0, 0);
1026	}
1027
1028#ifdef RCTL
1029	rctl_racct_release(p->p_racct);
1030#endif
1031	racct_destroy_locked(&p->p_racct);
1032	RACCT_UNLOCK();
1033	PROC_UNLOCK(p);
1034}
1035
1036/*
1037 * Called after credentials change, to move resource utilisation
1038 * between raccts.
1039 */
1040void
1041racct_proc_ucred_changed(struct proc *p, struct ucred *oldcred,
1042    struct ucred *newcred)
1043{
1044	struct uidinfo *olduip, *newuip;
1045	struct loginclass *oldlc, *newlc;
1046	struct prison *oldpr, *newpr, *pr;
1047
1048	if (!racct_enable)
1049		return;
1050
1051	PROC_LOCK_ASSERT(p, MA_OWNED);
1052
1053	newuip = newcred->cr_ruidinfo;
1054	olduip = oldcred->cr_ruidinfo;
1055	newlc = newcred->cr_loginclass;
1056	oldlc = oldcred->cr_loginclass;
1057	newpr = newcred->cr_prison;
1058	oldpr = oldcred->cr_prison;
1059
1060	RACCT_LOCK();
1061	if (newuip != olduip) {
1062		racct_sub_racct(olduip->ui_racct, p->p_racct);
1063		racct_add_racct(newuip->ui_racct, p->p_racct);
1064	}
1065	if (newlc != oldlc) {
1066		racct_sub_racct(oldlc->lc_racct, p->p_racct);
1067		racct_add_racct(newlc->lc_racct, p->p_racct);
1068	}
1069	if (newpr != oldpr) {
1070		for (pr = oldpr; pr != NULL; pr = pr->pr_parent)
1071			racct_sub_racct(pr->pr_prison_racct->prr_racct,
1072			    p->p_racct);
1073		for (pr = newpr; pr != NULL; pr = pr->pr_parent)
1074			racct_add_racct(pr->pr_prison_racct->prr_racct,
1075			    p->p_racct);
1076	}
1077	RACCT_UNLOCK();
1078}
1079
1080void
1081racct_move(struct racct *dest, struct racct *src)
1082{
1083
1084	ASSERT_RACCT_ENABLED();
1085
1086	RACCT_LOCK();
1087	racct_add_racct(dest, src);
1088	racct_sub_racct(src, src);
1089	RACCT_UNLOCK();
1090}
1091
1092/*
1093 * Make the process sleep in userret() for 'timeout' ticks.  Setting
1094 * timeout to -1 makes it sleep until woken up by racct_proc_wakeup().
1095 */
1096void
1097racct_proc_throttle(struct proc *p, int timeout)
1098{
1099	struct thread *td;
1100#ifdef SMP
1101	int cpuid;
1102#endif
1103
1104	KASSERT(timeout != 0, ("timeout %d", timeout));
1105	ASSERT_RACCT_ENABLED();
1106	PROC_LOCK_ASSERT(p, MA_OWNED);
1107
1108	/*
1109	 * Do not block kernel processes.  Also do not block processes with
1110	 * low %cpu utilization to improve interactivity.
1111	 */
1112	if ((p->p_flag & (P_SYSTEM | P_KPROC)) != 0)
1113		return;
1114
1115	if (p->p_throttled < 0 || (timeout > 0 && p->p_throttled > timeout))
1116		return;
1117
1118	p->p_throttled = timeout;
1119
1120	FOREACH_THREAD_IN_PROC(p, td) {
1121		thread_lock(td);
1122		switch (td->td_state) {
1123		case TDS_RUNQ:
1124			/*
1125			 * If the thread is on the scheduler run-queue, we can
1126			 * not just remove it from there.  So we set the flag
1127			 * TDF_NEEDRESCHED for the thread, so that once it is
1128			 * running, it is taken off the cpu as soon as possible.
1129			 */
1130			td->td_flags |= TDF_NEEDRESCHED;
1131			break;
1132		case TDS_RUNNING:
1133			/*
1134			 * If the thread is running, we request a context
1135			 * switch for it by setting the TDF_NEEDRESCHED flag.
1136			 */
1137			td->td_flags |= TDF_NEEDRESCHED;
1138#ifdef SMP
1139			cpuid = td->td_oncpu;
1140			if ((cpuid != NOCPU) && (td != curthread))
1141				ipi_cpu(cpuid, IPI_AST);
1142#endif
1143			break;
1144		default:
1145			break;
1146		}
1147		thread_unlock(td);
1148	}
1149}
1150
1151static void
1152racct_proc_wakeup(struct proc *p)
1153{
1154
1155	ASSERT_RACCT_ENABLED();
1156
1157	PROC_LOCK_ASSERT(p, MA_OWNED);
1158
1159	if (p->p_throttled != 0) {
1160		p->p_throttled = 0;
1161		wakeup(p->p_racct);
1162	}
1163}
1164
1165static void
1166racct_decay_callback(struct racct *racct, void *dummy1, void *dummy2)
1167{
1168	int64_t r_old, r_new;
1169
1170	ASSERT_RACCT_ENABLED();
1171	RACCT_LOCK_ASSERT();
1172
1173#ifdef RCTL
1174	rctl_throttle_decay(racct, RACCT_READBPS);
1175	rctl_throttle_decay(racct, RACCT_WRITEBPS);
1176	rctl_throttle_decay(racct, RACCT_READIOPS);
1177	rctl_throttle_decay(racct, RACCT_WRITEIOPS);
1178#endif
1179
1180	r_old = racct->r_resources[RACCT_PCTCPU];
1181
1182	/* If there is nothing to decay, just exit. */
1183	if (r_old <= 0)
1184		return;
1185
1186	r_new = r_old * RACCT_DECAY_FACTOR / FSCALE;
1187	racct->r_resources[RACCT_PCTCPU] = r_new;
1188}
1189
1190static void
1191racct_decay_pre(void)
1192{
1193
1194	RACCT_LOCK();
1195}
1196
1197static void
1198racct_decay_post(void)
1199{
1200
1201	RACCT_UNLOCK();
1202}
1203
1204static void
1205racct_decay(void)
1206{
1207
1208	ASSERT_RACCT_ENABLED();
1209
1210	ui_racct_foreach(racct_decay_callback, racct_decay_pre,
1211	    racct_decay_post, NULL, NULL);
1212	loginclass_racct_foreach(racct_decay_callback, racct_decay_pre,
1213	    racct_decay_post, NULL, NULL);
1214	prison_racct_foreach(racct_decay_callback, racct_decay_pre,
1215	    racct_decay_post, NULL, NULL);
1216}
1217
1218static void
1219racctd(void)
1220{
1221	struct thread *td;
1222	struct proc *p;
1223	struct timeval wallclock;
1224	uint64_t pct, pct_estimate, runtime;
1225
1226	ASSERT_RACCT_ENABLED();
1227
1228	for (;;) {
1229		racct_decay();
1230
1231		sx_slock(&allproc_lock);
1232
1233		LIST_FOREACH(p, &zombproc, p_list) {
1234			PROC_LOCK(p);
1235			racct_set(p, RACCT_PCTCPU, 0);
1236			PROC_UNLOCK(p);
1237		}
1238
1239		FOREACH_PROC_IN_SYSTEM(p) {
1240			PROC_LOCK(p);
1241			if (p->p_state != PRS_NORMAL) {
1242				PROC_UNLOCK(p);
1243				continue;
1244			}
1245
1246			microuptime(&wallclock);
1247			timevalsub(&wallclock, &p->p_stats->p_start);
1248			PROC_STATLOCK(p);
1249			FOREACH_THREAD_IN_PROC(p, td)
1250				ruxagg(p, td);
1251			runtime = cputick2usec(p->p_rux.rux_runtime);
1252			PROC_STATUNLOCK(p);
1253#ifdef notyet
1254			KASSERT(runtime >= p->p_prev_runtime,
1255			    ("runtime < p_prev_runtime"));
1256#else
1257			if (runtime < p->p_prev_runtime)
1258				runtime = p->p_prev_runtime;
1259#endif
1260			p->p_prev_runtime = runtime;
1261			if (wallclock.tv_sec > 0 || wallclock.tv_usec > 0) {
1262				pct_estimate = (1000000 * runtime * 100) /
1263				    ((uint64_t)wallclock.tv_sec * 1000000 +
1264				    wallclock.tv_usec);
1265			} else
1266				pct_estimate = 0;
1267			pct = racct_getpcpu(p, pct_estimate);
1268			RACCT_LOCK();
1269#ifdef RCTL
1270			rctl_throttle_decay(p->p_racct, RACCT_READBPS);
1271			rctl_throttle_decay(p->p_racct, RACCT_WRITEBPS);
1272			rctl_throttle_decay(p->p_racct, RACCT_READIOPS);
1273			rctl_throttle_decay(p->p_racct, RACCT_WRITEIOPS);
1274#endif
1275			racct_set_locked(p, RACCT_PCTCPU, pct, 1);
1276			racct_set_locked(p, RACCT_CPU, runtime, 0);
1277			racct_set_locked(p, RACCT_WALLCLOCK,
1278			    (uint64_t)wallclock.tv_sec * 1000000 +
1279			    wallclock.tv_usec, 0);
1280			RACCT_UNLOCK();
1281			PROC_UNLOCK(p);
1282		}
1283
1284		/*
1285		 * To ensure that processes are throttled in a fair way, we need
1286		 * to iterate over all processes again and check the limits
1287		 * for %cpu resource only after ucred racct containers have been
1288		 * properly filled.
1289		 */
1290		FOREACH_PROC_IN_SYSTEM(p) {
1291			PROC_LOCK(p);
1292			if (p->p_state != PRS_NORMAL) {
1293				PROC_UNLOCK(p);
1294				continue;
1295			}
1296
1297			if (racct_pcpu_available(p) <= 0) {
1298				if (p->p_racct->r_resources[RACCT_PCTCPU] >
1299				    pcpu_threshold)
1300					racct_proc_throttle(p, -1);
1301			} else if (p->p_throttled == -1) {
1302				racct_proc_wakeup(p);
1303			}
1304			PROC_UNLOCK(p);
1305		}
1306		sx_sunlock(&allproc_lock);
1307		pause("-", hz);
1308	}
1309}
1310
1311static struct kproc_desc racctd_kp = {
1312	"racctd",
1313	racctd,
1314	NULL
1315};
1316
1317static void
1318racctd_init(void)
1319{
1320	if (!racct_enable)
1321		return;
1322
1323	kproc_start(&racctd_kp);
1324}
1325SYSINIT(racctd, SI_SUB_RACCTD, SI_ORDER_FIRST, racctd_init, NULL);
1326
1327static void
1328racct_init(void)
1329{
1330	if (!racct_enable)
1331		return;
1332
1333	racct_zone = uma_zcreate("racct", sizeof(struct racct),
1334	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1335	/*
1336	 * XXX: Move this somewhere.
1337	 */
1338	prison0.pr_prison_racct = prison_racct_find("0");
1339}
1340SYSINIT(racct, SI_SUB_RACCT, SI_ORDER_FIRST, racct_init, NULL);
1341
1342#endif /* !RACCT */
1343