1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7#include "rxe.h"
8
9#define RXE_POOL_TIMEOUT	(200)
10#define RXE_POOL_ALIGN		(16)
11
12static const struct rxe_type_info {
13	const char *name;
14	size_t size;
15	size_t elem_offset;
16	void (*cleanup)(struct rxe_pool_elem *elem);
17	u32 min_index;
18	u32 max_index;
19	u32 max_elem;
20} rxe_type_info[RXE_NUM_TYPES] = {
21	[RXE_TYPE_UC] = {
22		.name		= "uc",
23		.size		= sizeof(struct rxe_ucontext),
24		.elem_offset	= offsetof(struct rxe_ucontext, elem),
25		.min_index	= 1,
26		.max_index	= RXE_MAX_UCONTEXT,
27		.max_elem	= RXE_MAX_UCONTEXT,
28	},
29	[RXE_TYPE_PD] = {
30		.name		= "pd",
31		.size		= sizeof(struct rxe_pd),
32		.elem_offset	= offsetof(struct rxe_pd, elem),
33		.min_index	= 1,
34		.max_index	= RXE_MAX_PD,
35		.max_elem	= RXE_MAX_PD,
36	},
37	[RXE_TYPE_AH] = {
38		.name		= "ah",
39		.size		= sizeof(struct rxe_ah),
40		.elem_offset	= offsetof(struct rxe_ah, elem),
41		.min_index	= RXE_MIN_AH_INDEX,
42		.max_index	= RXE_MAX_AH_INDEX,
43		.max_elem	= RXE_MAX_AH,
44	},
45	[RXE_TYPE_SRQ] = {
46		.name		= "srq",
47		.size		= sizeof(struct rxe_srq),
48		.elem_offset	= offsetof(struct rxe_srq, elem),
49		.cleanup	= rxe_srq_cleanup,
50		.min_index	= RXE_MIN_SRQ_INDEX,
51		.max_index	= RXE_MAX_SRQ_INDEX,
52		.max_elem	= RXE_MAX_SRQ,
53	},
54	[RXE_TYPE_QP] = {
55		.name		= "qp",
56		.size		= sizeof(struct rxe_qp),
57		.elem_offset	= offsetof(struct rxe_qp, elem),
58		.cleanup	= rxe_qp_cleanup,
59		.min_index	= RXE_MIN_QP_INDEX,
60		.max_index	= RXE_MAX_QP_INDEX,
61		.max_elem	= RXE_MAX_QP,
62	},
63	[RXE_TYPE_CQ] = {
64		.name		= "cq",
65		.size		= sizeof(struct rxe_cq),
66		.elem_offset	= offsetof(struct rxe_cq, elem),
67		.cleanup	= rxe_cq_cleanup,
68		.min_index	= 1,
69		.max_index	= RXE_MAX_CQ,
70		.max_elem	= RXE_MAX_CQ,
71	},
72	[RXE_TYPE_MR] = {
73		.name		= "mr",
74		.size		= sizeof(struct rxe_mr),
75		.elem_offset	= offsetof(struct rxe_mr, elem),
76		.cleanup	= rxe_mr_cleanup,
77		.min_index	= RXE_MIN_MR_INDEX,
78		.max_index	= RXE_MAX_MR_INDEX,
79		.max_elem	= RXE_MAX_MR,
80	},
81	[RXE_TYPE_MW] = {
82		.name		= "mw",
83		.size		= sizeof(struct rxe_mw),
84		.elem_offset	= offsetof(struct rxe_mw, elem),
85		.cleanup	= rxe_mw_cleanup,
86		.min_index	= RXE_MIN_MW_INDEX,
87		.max_index	= RXE_MAX_MW_INDEX,
88		.max_elem	= RXE_MAX_MW,
89	},
90};
91
92void rxe_pool_init(struct rxe_dev *rxe, struct rxe_pool *pool,
93		   enum rxe_elem_type type)
94{
95	const struct rxe_type_info *info = &rxe_type_info[type];
96
97	memset(pool, 0, sizeof(*pool));
98
99	pool->rxe		= rxe;
100	pool->name		= info->name;
101	pool->type		= type;
102	pool->max_elem		= info->max_elem;
103	pool->elem_size		= ALIGN(info->size, RXE_POOL_ALIGN);
104	pool->elem_offset	= info->elem_offset;
105	pool->cleanup		= info->cleanup;
106
107	atomic_set(&pool->num_elem, 0);
108
109	xa_init_flags(&pool->xa, XA_FLAGS_ALLOC);
110	pool->limit.min = info->min_index;
111	pool->limit.max = info->max_index;
112}
113
114void rxe_pool_cleanup(struct rxe_pool *pool)
115{
116	WARN_ON(!xa_empty(&pool->xa));
117}
118
119int __rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_elem *elem,
120				bool sleepable)
121{
122	int err;
123	gfp_t gfp_flags;
124
125	if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
126		goto err_cnt;
127
128	elem->pool = pool;
129	elem->obj = (u8 *)elem - pool->elem_offset;
130	kref_init(&elem->ref_cnt);
131	init_completion(&elem->complete);
132
133	/* AH objects are unique in that the create_ah verb
134	 * can be called in atomic context. If the create_ah
135	 * call is not sleepable use GFP_ATOMIC.
136	 */
137	gfp_flags = sleepable ? GFP_KERNEL : GFP_ATOMIC;
138
139	if (sleepable)
140		might_sleep();
141	err = xa_alloc_cyclic(&pool->xa, &elem->index, NULL, pool->limit,
142			      &pool->next, gfp_flags);
143	if (err < 0)
144		goto err_cnt;
145
146	return 0;
147
148err_cnt:
149	atomic_dec(&pool->num_elem);
150	return -EINVAL;
151}
152
153void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
154{
155	struct rxe_pool_elem *elem;
156	struct xarray *xa = &pool->xa;
157	void *obj;
158
159	rcu_read_lock();
160	elem = xa_load(xa, index);
161	if (elem && kref_get_unless_zero(&elem->ref_cnt))
162		obj = elem->obj;
163	else
164		obj = NULL;
165	rcu_read_unlock();
166
167	return obj;
168}
169
170static void rxe_elem_release(struct kref *kref)
171{
172	struct rxe_pool_elem *elem = container_of(kref, typeof(*elem), ref_cnt);
173
174	complete(&elem->complete);
175}
176
177int __rxe_cleanup(struct rxe_pool_elem *elem, bool sleepable)
178{
179	struct rxe_pool *pool = elem->pool;
180	struct xarray *xa = &pool->xa;
181	static int timeout = RXE_POOL_TIMEOUT;
182	int ret, err = 0;
183	void *xa_ret;
184
185	if (sleepable)
186		might_sleep();
187
188	/* erase xarray entry to prevent looking up
189	 * the pool elem from its index
190	 */
191	xa_ret = xa_erase(xa, elem->index);
192	WARN_ON(xa_err(xa_ret));
193
194	/* if this is the last call to rxe_put complete the
195	 * object. It is safe to touch obj->elem after this since
196	 * it is freed below
197	 */
198	__rxe_put(elem);
199
200	/* wait until all references to the object have been
201	 * dropped before final object specific cleanup and
202	 * return to rdma-core
203	 */
204	if (sleepable) {
205		if (!completion_done(&elem->complete) && timeout) {
206			ret = wait_for_completion_timeout(&elem->complete,
207					timeout);
208
209			/* Shouldn't happen. There are still references to
210			 * the object but, rather than deadlock, free the
211			 * object or pass back to rdma-core.
212			 */
213			if (WARN_ON(!ret))
214				err = -EINVAL;
215		}
216	} else {
217		unsigned long until = jiffies + timeout;
218
219		/* AH objects are unique in that the destroy_ah verb
220		 * can be called in atomic context. This delay
221		 * replaces the wait_for_completion call above
222		 * when the destroy_ah call is not sleepable
223		 */
224		while (!completion_done(&elem->complete) &&
225				time_before(jiffies, until))
226			mdelay(1);
227
228		if (WARN_ON(!completion_done(&elem->complete)))
229			err = -EINVAL;
230	}
231
232	if (pool->cleanup)
233		pool->cleanup(elem);
234
235	atomic_dec(&pool->num_elem);
236
237	return err;
238}
239
240int __rxe_get(struct rxe_pool_elem *elem)
241{
242	return kref_get_unless_zero(&elem->ref_cnt);
243}
244
245int __rxe_put(struct rxe_pool_elem *elem)
246{
247	return kref_put(&elem->ref_cnt, rxe_elem_release);
248}
249
250void __rxe_finalize(struct rxe_pool_elem *elem)
251{
252	void *xa_ret;
253
254	xa_ret = xa_store(&elem->pool->xa, elem->index, elem, GFP_KERNEL);
255	WARN_ON(xa_err(xa_ret));
256}
257