1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25/*
26 * sol_uverbs_hca.c
27 *
28 * Provides the Solaris OFED User Verbs thin common hca interface for
29 * sharing of IBT client handle, device list, and asynchronous event
30 * delivery.
31 */
32#include <sys/vfs.h>
33#ifdef VFS_OPS
34#include <sys/vfs_opreg.h>
35#include <sys/vnode.h>
36#endif
37#include <sys/errno.h>
38#include <sys/cred.h>
39#include <sys/uio.h>
40#include <sys/semaphore.h>
41#include <sys/ddi.h>
42#include <sys/sunddi.h>
43
44#include <sys/ib/ibtl/ibvti.h>
45#include <sys/ib/clients/of/ofa_solaris.h>
46#include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
47#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_hca.h>
48#include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
49
50extern char	*sol_uverbs_dbg_str;
51
52/*
53 * Globals for managing the list of HCA's and the registered clients.
54 */
55kmutex_t	sol_uverbs_hca_lock;
56llist_head_t	sol_uverbs_hca_list;
57llist_head_t	sol_uverbs_client_list;
58static uint32_t	sol_uverbs_common_hca_initialized = 0;
59
60typedef struct sol_uverbs_hca_client_data {
61	llist_head_t		list;
62	sol_uverbs_ib_client_t	*client;
63	void			*data;
64} sol_uverbs_hca_client_data_t;
65
66static
67int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
68	sol_uverbs_ib_client_t *client);
69
70/*
71 * Function:
72 *	sol_uverbs_ib_register_client
73 * Input:
74 *	client  	- Pointer to the client structure
75 * Output:
76 *	None
77 * Returns:
78 *	Zero on success, else error code.
79 * Description:
80 *	The Solaris User Verbs kernel agent provides a single
81 *	common view of the IBTF devices.  This function allows
82 *	Solaris OFA kernel implementations to share this view
83 *	by registerng a client callback for notification of HCA
84 *	addtion and removal.  Note that when this function is
85 *	called, the client will	get an "add" callback for all
86 *	existing devices.
87 */
88int
89sol_uverbs_ib_register_client(sol_uverbs_ib_client_t *client)
90{
91	llist_head_t		*entry;
92	sol_uverbs_hca_t	*hca;
93
94	ASSERT(client != NULL);
95	mutex_enter(&sol_uverbs_hca_lock);
96	llist_head_init(&client->list, client);
97	llist_add_tail(&client->list, &sol_uverbs_client_list);
98	list_for_each(entry, &sol_uverbs_hca_list) {
99		hca = (sol_uverbs_hca_t *)entry->ptr;
100
101		if (client->add &&
102		    !sol_uverbs_hca_add_client_context(hca, client)) {
103			client->add(hca);
104		}
105	}
106	mutex_exit(&sol_uverbs_hca_lock);
107
108	return (0);
109}
110
111/*
112 * Function:
113 *	sol_uverbs_ib_unregister_client
114 * Input:
115 *	client  - Pointer to the client structure
116 * Output:
117 *	None
118 * Returns:
119 *	None
120 * Description:
121 *	Removes a client registration previously created with
122 *	the sol_uverbs_ib_register_client() call.
123 */
124void
125sol_uverbs_ib_unregister_client(sol_uverbs_ib_client_t *client)
126{
127	llist_head_t			*entry, *centry, *tmp;
128	sol_uverbs_hca_t		*hca;
129	sol_uverbs_hca_client_data_t	*context;
130
131	ASSERT(client != NULL);
132	mutex_enter(&sol_uverbs_hca_lock);
133
134	list_for_each(entry, &sol_uverbs_hca_list) {
135		hca = (sol_uverbs_hca_t *)entry->ptr;
136
137		ASSERT(hca != NULL);
138
139		if (client->remove) {
140			client->remove(hca);
141		}
142		mutex_enter(&hca->client_data_lock);
143		centry = hca->client_data_list.nxt;
144		tmp = centry->nxt;
145
146		while (centry != &hca->client_data_list) {
147			ASSERT(centry);
148			context = (sol_uverbs_hca_client_data_t *)centry->ptr;
149			ASSERT(context != NULL);
150
151			if (context->client == client) {
152				llist_del(centry);
153				kmem_free(context, sizeof (*context));
154			}
155			centry = tmp;
156			tmp = centry->nxt;
157		}
158		mutex_exit(&hca->client_data_lock);
159	}
160	llist_del(&client->list);
161	mutex_exit(&sol_uverbs_hca_lock);
162}
163
164/*
165 * Function:
166 *	sol_uverbs_ib_get_client_data
167 * Input:
168 *	hca    	- Pointer to HCA struct passed in the client
169 *                add function callback.
170 *     client   - A pointer to the client structure.
171 * Output:
172 *	None
173 * Returns:
174 *	The client data, or NULL on error.
175 * Description:
176 *	Returns the client data associated with the given
177 *      HCA. The data is set/specified via the
178 *	sol_uverbs_ib_set_client_data() function.
179 */
180void *
181sol_uverbs_ib_get_client_data(sol_uverbs_hca_t *hca,
182					sol_uverbs_ib_client_t *client)
183{
184	llist_head_t			*entry;
185	sol_uverbs_hca_client_data_t	*context;
186	void				*data = NULL;
187
188	ASSERT(hca != NULL);
189	ASSERT(client != NULL);
190
191	mutex_enter(&hca->client_data_lock);
192
193	list_for_each(entry, &hca->client_data_list) {
194		context = (sol_uverbs_hca_client_data_t *)entry->ptr;
195
196		ASSERT(context != NULL);
197
198		if (context->client == client) {
199			data = context->data;
200			break;
201		}
202	}
203	mutex_exit(&hca->client_data_lock);
204	return (data);
205}
206
207/*
208 * Function:
209 *	sol_uverbs_ib_set_client_data
210 * Input:
211 *	hca    	- Pointer to HCA struct passed in the client
212 *                add function.
213 *     client   - A pointer to the client structure.
214 *     data     - The client data to associate with the HCA.
215 * Output:
216 *	None
217 * Returns:
218 *	None
219 * Description:
220 *	Sets the client data associated with the given
221 *      HCA.
222 */
223void
224sol_uverbs_ib_set_client_data(sol_uverbs_hca_t *hca,
225	sol_uverbs_ib_client_t *client, void *data)
226{
227	llist_head_t			*entry;
228	sol_uverbs_hca_client_data_t	*context;
229
230	ASSERT(hca != NULL);
231	ASSERT(client != NULL);
232
233	mutex_enter(&hca->client_data_lock);
234
235	list_for_each(entry, &hca->client_data_list) {
236		context = (sol_uverbs_hca_client_data_t *)entry->ptr;
237
238		ASSERT(context != NULL);
239
240		if (context->client == client) {
241			context->data = data;
242			goto out;
243		}
244	}
245	SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
246	    "HCA SET CLIENT DATA: No client found for %s\n",
247	    client->name != NULL ? client->name : "NULL Client Name");
248
249out:
250	mutex_exit(&hca->client_data_lock);
251}
252
253/*
254 * Function:
255 *	sol_uverbs_ib_register_event_handler
256 * Input:
257 *	handler  - Pointer to handler structure
258 * Output:
259 *	None
260 * Returns:
261 *	Zero
262 * Description:
263 *	Register to receive ansynchronous notifications
264 *	for the HCA defined in the handler struct.  The notifications
265 *	are delivered via the callback function defined in the handler
266 *	struct.
267 */
268int
269sol_uverbs_ib_register_event_handler(sol_uverbs_ib_event_handler_t *handler)
270{
271	ASSERT(handler != NULL);
272	ASSERT(handler->hca != NULL);
273
274	mutex_enter(&handler->hca->event_handler_lock);
275	llist_head_init(&handler->list, handler);
276	llist_add_tail(&handler->list, &handler->hca->event_handler_list);
277	mutex_exit(&handler->hca->event_handler_lock);
278	return (0);
279}
280
281/*
282 * Function:
283 *	sol_uverbs_ib_unregister_event_handler
284 * Input:
285 *	handler  - Pointer to handler structure
286 * Output:
287 *	None
288 * Returns:
289 *	Zero
290 * Description:
291 *	Unregister a ansynchronous notification handler previously
292 *	registered via the osl_uverbs_ib_register_event_handler() call.
293 */
294int
295sol_uverbs_ib_unregister_event_handler(sol_uverbs_ib_event_handler_t *handler)
296{
297	ASSERT(handler != NULL);
298	ASSERT(handler->hca != NULL);
299
300	mutex_enter(&handler->hca->event_handler_lock);
301	llist_del(&handler->list);
302	mutex_exit(&handler->hca->event_handler_lock);
303	return (0);
304}
305
306/*
307 * Function:
308 *	sol_uverbs_common_hca_init
309 * Input:
310 *	None
311 * Output:
312 *	None
313 * Returns:
314 *	Zero
315 * Description:
316 *	Perform initialization required by the common hca client API.
317 */
318int
319sol_uverbs_common_hca_init()
320{
321	llist_head_init(&sol_uverbs_hca_list, NULL);
322	llist_head_init(&sol_uverbs_client_list, NULL);
323	mutex_init(&sol_uverbs_hca_lock, NULL, MUTEX_DRIVER, NULL);
324	sol_uverbs_common_hca_initialized = 1;
325	return (0);
326}
327
328/*
329 * Function:
330 *	sol_uverbs_common_hca_fini
331 * Input:
332 *	None
333 * Output:
334 *	None
335 * Returns:
336 *	None
337 * Description:
338 *	Perform cleanup required by the common hca client API.
339 */
340void
341sol_uverbs_common_hca_fini()
342{
343	ASSERT(llist_empty(&sol_uverbs_client_list));
344	sol_uverbs_common_hca_initialized = 0;
345	mutex_destroy(&sol_uverbs_hca_lock);
346}
347
348/*
349 * Helpers for internal use only
350 */
351/*
352 * Function:
353 *	sol_uverbs_hca_add_client_context
354 * Input:
355 *	hca	- Pointer to the hca struct to add a client context.
356 *	client  - Pointer to the client.
357 * Output:
358 *	None
359 * Returns:
360 *	0 on success, else the error.
361 * Description:
362 *	Create a context for the specified client and attach it to
363 *	the specified hca.
364 */
365static
366int sol_uverbs_hca_add_client_context(sol_uverbs_hca_t *hca,
367    sol_uverbs_ib_client_t *client)
368{
369	sol_uverbs_hca_client_data_t   *context;
370
371	context = kmem_zalloc(sizeof (*context), KM_NOSLEEP);
372
373	if (!context) {
374		SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
375		    "HCA: Couldn't allocate client context for %s",
376		    client->name ? client->name : "Name is NULL");
377		return (ENOMEM);
378	}
379
380	context->client = client;
381	context->data   = NULL;
382	llist_head_init(&context->list, context);
383
384	mutex_enter(&hca->client_data_lock);
385	llist_add(&context->list, &hca->client_data_list);
386	mutex_exit(&hca->client_data_lock);
387	return (0);
388}
389
390/*
391 * Function:
392 *	sol_uverbs_ibt_hdl_to_hca
393 * Input:
394 *	hca_hdl - IBT handle to an HCA.
395 * Output:
396 *	None
397 * Returns:
398 *	A pointer to the sol_uverbs HCA structure associated with the handle,
399 *	or NULL if no associated HCA is found.
400 * Description:
401 *	Given an IBT hca handle, return the user verbs HCA structure associated
402 *	with that handle.
403 */
404sol_uverbs_hca_t *
405sol_uverbs_ibt_hdl_to_hca(ibt_hca_hdl_t hca_hdl)
406{
407	llist_head_t		*entry;
408	sol_uverbs_hca_t	*hca;
409	sol_uverbs_hca_t	*ret = NULL;
410
411	mutex_enter(&sol_uverbs_hca_lock);
412	list_for_each(entry, &sol_uverbs_hca_list) {
413		hca = (sol_uverbs_hca_t *)entry->ptr;
414
415		if (hca->hdl == hca_hdl) {
416			ret = hca;
417			break;
418		}
419	}
420	mutex_exit(&sol_uverbs_hca_lock);
421
422	return (ret);
423}
424