1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2001 Matthew Dillon.  All Rights Reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28/* Mutex pool routines.  These routines are designed to be used as short
29 * term leaf mutexes (e.g. the last mutex you might acquire other then
30 * calling msleep()).  They operate using a shared pool.  A mutex is chosen
31 * from the pool based on the supplied pointer (which may or may not be
32 * valid).
33 *
34 * Advantages:
35 *	- no structural overhead.  Mutexes can be associated with structures
36 *	  without adding bloat to the structures.
37 *	- mutexes can be obtained for invalid pointers, useful when uses
38 *	  mutexes to interlock destructor ops.
39 *	- no initialization/destructor overhead.
40 *	- can be used with msleep.
41 *
42 * Disadvantages:
43 *	- should generally only be used as leaf mutexes.
44 *	- pool/pool dependency ordering cannot be depended on.
45 *	- possible L1 cache mastersip contention between cpus.
46 */
47
48#include <sys/cdefs.h>
49__FBSDID("$FreeBSD$");
50
51#include <sys/param.h>
52#include <sys/proc.h>
53#include <sys/kernel.h>
54#include <sys/ktr.h>
55#include <sys/lock.h>
56#include <sys/malloc.h>
57#include <sys/mutex.h>
58#include <sys/systm.h>
59
60
61static MALLOC_DEFINE(M_MTXPOOL, "mtx_pool", "mutex pool");
62
63/* Pool sizes must be a power of two */
64#ifndef MTX_POOL_SLEEP_SIZE
65#define MTX_POOL_SLEEP_SIZE		1024
66#endif
67
68struct mtxpool_header {
69	int		mtxpool_size;
70	int		mtxpool_mask;
71	int		mtxpool_shift;
72	int		mtxpool_next __aligned(CACHE_LINE_SIZE);
73};
74
75struct mtx_pool {
76	struct mtxpool_header mtx_pool_header;
77	struct mtx	mtx_pool_ary[1];
78};
79
80#define mtx_pool_size	mtx_pool_header.mtxpool_size
81#define mtx_pool_mask	mtx_pool_header.mtxpool_mask
82#define mtx_pool_shift	mtx_pool_header.mtxpool_shift
83#define mtx_pool_next	mtx_pool_header.mtxpool_next
84
85struct mtx_pool __read_frequently *mtxpool_sleep;
86
87#if UINTPTR_MAX == UINT64_MAX	/* 64 bits */
88# define POINTER_BITS		64
89# define HASH_MULTIPLIER	11400714819323198485u /* (2^64)*(sqrt(5)-1)/2 */
90#else				/* assume 32 bits */
91# define POINTER_BITS		32
92# define HASH_MULTIPLIER	2654435769u	      /* (2^32)*(sqrt(5)-1)/2 */
93#endif
94
95/*
96 * Return the (shared) pool mutex associated with the specified address.
97 * The returned mutex is a leaf level mutex, meaning that if you obtain it
98 * you cannot obtain any other mutexes until you release it.  You can
99 * legally msleep() on the mutex.
100 */
101struct mtx *
102mtx_pool_find(struct mtx_pool *pool, void *ptr)
103{
104	int p;
105
106	KASSERT(pool != NULL, ("_mtx_pool_find(): null pool"));
107	/*
108	 * Fibonacci hash, see Knuth's
109	 * _Art of Computer Programming, Volume 3 / Sorting and Searching_
110	 */
111	p = ((HASH_MULTIPLIER * (uintptr_t)ptr) >> pool->mtx_pool_shift) &
112	    pool->mtx_pool_mask;
113	return (&pool->mtx_pool_ary[p]);
114}
115
116static void
117mtx_pool_initialize(struct mtx_pool *pool, const char *mtx_name, int pool_size,
118    int opts)
119{
120	int i, maskbits;
121
122	pool->mtx_pool_size = pool_size;
123	pool->mtx_pool_mask = pool_size - 1;
124	for (i = 1, maskbits = 0; (i & pool_size) == 0; i = i << 1)
125		maskbits++;
126	pool->mtx_pool_shift = POINTER_BITS - maskbits;
127	pool->mtx_pool_next = 0;
128	for (i = 0; i < pool_size; ++i)
129		mtx_init(&pool->mtx_pool_ary[i], mtx_name, NULL, opts);
130}
131
132struct mtx_pool *
133mtx_pool_create(const char *mtx_name, int pool_size, int opts)
134{
135	struct mtx_pool *pool;
136
137	if (pool_size <= 0 || !powerof2(pool_size)) {
138		printf("WARNING: %s pool size is not a power of 2.\n",
139		    mtx_name);
140		pool_size = 128;
141	}
142	pool = malloc(sizeof (struct mtx_pool) +
143	    ((pool_size - 1) * sizeof (struct mtx)),
144	    M_MTXPOOL, M_WAITOK | M_ZERO);
145	mtx_pool_initialize(pool, mtx_name, pool_size, opts);
146	return pool;
147}
148
149void
150mtx_pool_destroy(struct mtx_pool **poolp)
151{
152	int i;
153	struct mtx_pool *pool = *poolp;
154
155	for (i = pool->mtx_pool_size - 1; i >= 0; --i)
156		mtx_destroy(&pool->mtx_pool_ary[i]);
157	free(pool, M_MTXPOOL);
158	*poolp = NULL;
159}
160
161static void
162mtx_pool_setup_dynamic(void *dummy __unused)
163{
164	mtxpool_sleep = mtx_pool_create("sleep mtxpool",
165	    MTX_POOL_SLEEP_SIZE, MTX_DEF);
166}
167
168/*
169 * Obtain a (shared) mutex from the pool.  The returned mutex is a leaf
170 * level mutex, meaning that if you obtain it you cannot obtain any other
171 * mutexes until you release it.  You can legally msleep() on the mutex.
172 */
173struct mtx *
174mtx_pool_alloc(struct mtx_pool *pool)
175{
176	int i;
177
178	KASSERT(pool != NULL, ("mtx_pool_alloc(): null pool"));
179	/*
180	 * mtx_pool_next is unprotected against multiple accesses,
181	 * but simultaneous access by two CPUs should not be very
182	 * harmful.
183	 */
184	i = pool->mtx_pool_next;
185	pool->mtx_pool_next = (i + 1) & pool->mtx_pool_mask;
186	return (&pool->mtx_pool_ary[i]);
187}
188
189SYSINIT(mtxpooli2, SI_SUB_MTX_POOL_DYNAMIC, SI_ORDER_FIRST,
190    mtx_pool_setup_dynamic, NULL);
191