1/*-
2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: stable/11/sys/dev/ntb/ntb.c 355152 2019-11-28 00:41:42Z mav $");
29
30#include <sys/param.h>
31#include <sys/kernel.h>
32#include <sys/systm.h>
33#include <sys/bus.h>
34#include <sys/rmlock.h>
35#include <sys/malloc.h>
36#include <sys/module.h>
37#include <sys/sysctl.h>
38
39#include "ntb.h"
40
41devclass_t ntb_hw_devclass;
42SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls");
43
44struct ntb_child {
45	device_t	dev;
46	int		function;
47	int		enabled;
48	int		mwoff;
49	int		mwcnt;
50	int		spadoff;
51	int		spadcnt;
52	int		dboff;
53	int		dbcnt;
54	uint64_t	dbmask;
55	void		*ctx;
56	const struct ntb_ctx_ops *ctx_ops;
57	struct rmlock	ctx_lock;
58	struct ntb_child *next;
59};
60
61int
62ntb_register_device(device_t dev)
63{
64	struct ntb_child **cpp = device_get_softc(dev);
65	struct ntb_child *nc;
66	int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt;
67	char cfg[128] = "";
68	char buf[32];
69	char *n, *np, *c, *p, *name;
70
71	mwu = 0;
72	mwt = NTB_MW_COUNT(dev);
73	spadu = 0;
74	spadt = NTB_SPAD_COUNT(dev);
75	dbu = 0;
76	dbt = flsll(NTB_DB_VALID_MASK(dev));
77
78	device_printf(dev, "%d memory windows, %d scratchpads, "
79	    "%d doorbells\n", mwt, spadt, dbt);
80
81	snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev),
82	    device_get_unit(dev));
83	TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg));
84	n = cfg;
85	i = 0;
86	while ((c = strsep(&n, ",")) != NULL) {
87		np = c;
88		name = strsep(&np, ":");
89		if (name != NULL && name[0] == 0)
90			name = NULL;
91		p = strsep(&np, ":");
92		mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu;
93		p = strsep(&np, ":");
94		spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu;
95		db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu;
96
97		if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) {
98			device_printf(dev, "Not enough resources for config\n");
99			break;
100		}
101
102		nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO);
103		nc->function = i;
104		nc->mwoff = mwu;
105		nc->mwcnt = mw;
106		nc->spadoff = spadu;
107		nc->spadcnt = spad;
108		nc->dboff = dbu;
109		nc->dbcnt = db;
110		nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db));
111		rm_init(&nc->ctx_lock, "ntb ctx");
112		nc->dev = device_add_child(dev, name, -1);
113		if (nc->dev == NULL) {
114			ntb_unregister_device(dev);
115			return (ENOMEM);
116		}
117		device_set_ivars(nc->dev, nc);
118		*cpp = nc;
119		cpp = &nc->next;
120
121		if (bootverbose) {
122			device_printf(dev, "%d \"%s\":", i, name);
123			if (mw > 0) {
124				printf(" memory windows %d", mwu);
125				if (mw > 1)
126					printf("-%d", mwu + mw - 1);
127			}
128			if (spad > 0) {
129				printf(" scratchpads %d", spadu);
130				if (spad > 1)
131					printf("-%d", spadu + spad - 1);
132			}
133			if (db > 0) {
134				printf(" doorbells %d", dbu);
135				if (db > 1)
136					printf("-%d", dbu + db - 1);
137			}
138			printf("\n");
139		}
140
141		mwu += mw;
142		spadu += spad;
143		dbu += db;
144		i++;
145	}
146
147	bus_generic_attach(dev);
148	return (0);
149}
150
151int
152ntb_unregister_device(device_t dev)
153{
154	struct ntb_child **cpp = device_get_softc(dev);
155	struct ntb_child *nc;
156	int error = 0;
157
158	while ((nc = *cpp) != NULL) {
159		*cpp = (*cpp)->next;
160		error = device_delete_child(dev, nc->dev);
161		if (error)
162			break;
163		rm_destroy(&nc->ctx_lock);
164		free(nc, M_DEVBUF);
165	}
166	return (error);
167}
168
169int
170ntb_child_location_str(device_t dev, device_t child, char *buf,
171    size_t buflen)
172{
173	struct ntb_child *nc = device_get_ivars(child);
174
175	snprintf(buf, buflen, "function=%d", nc->function);
176	return (0);
177}
178
179int
180ntb_print_child(device_t dev, device_t child)
181{
182	struct ntb_child *nc = device_get_ivars(child);
183	int retval;
184
185	retval = bus_print_child_header(dev, child);
186	if (nc->mwcnt > 0) {
187		printf(" mw %d", nc->mwoff);
188		if (nc->mwcnt > 1)
189			printf("-%d", nc->mwoff + nc->mwcnt - 1);
190	}
191	if (nc->spadcnt > 0) {
192		printf(" spad %d", nc->spadoff);
193		if (nc->spadcnt > 1)
194			printf("-%d", nc->spadoff + nc->spadcnt - 1);
195	}
196	if (nc->dbcnt > 0) {
197		printf(" db %d", nc->dboff);
198		if (nc->dbcnt > 1)
199			printf("-%d", nc->dboff + nc->dbcnt - 1);
200	}
201	retval += printf(" at function %d", nc->function);
202	retval += bus_print_child_domain(dev, child);
203	retval += bus_print_child_footer(dev, child);
204
205	return (retval);
206}
207
208bus_dma_tag_t
209ntb_get_dma_tag(device_t bus, device_t child)
210{
211
212	return (bus_get_dma_tag(bus));
213}
214
215void
216ntb_link_event(device_t dev)
217{
218	struct ntb_child **cpp = device_get_softc(dev);
219	struct ntb_child *nc;
220	struct rm_priotracker ctx_tracker;
221	enum ntb_speed speed;
222	enum ntb_width width;
223
224	if (NTB_LINK_IS_UP(dev, &speed, &width)) {
225		device_printf(dev, "Link is up (PCIe %d.x / x%d)\n",
226		    (int)speed, (int)width);
227	} else {
228		device_printf(dev, "Link is down\n");
229	}
230	for (nc = *cpp; nc != NULL; nc = nc->next) {
231		rm_rlock(&nc->ctx_lock, &ctx_tracker);
232		if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL)
233			nc->ctx_ops->link_event(nc->ctx);
234		rm_runlock(&nc->ctx_lock, &ctx_tracker);
235	}
236}
237
238void
239ntb_db_event(device_t dev, uint32_t vec)
240{
241	struct ntb_child **cpp = device_get_softc(dev);
242	struct ntb_child *nc;
243	struct rm_priotracker ctx_tracker;
244
245	for (nc = *cpp; nc != NULL; nc = nc->next) {
246		rm_rlock(&nc->ctx_lock, &ctx_tracker);
247		if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL)
248			nc->ctx_ops->db_event(nc->ctx, vec);
249		rm_runlock(&nc->ctx_lock, &ctx_tracker);
250	}
251}
252
253bool
254ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width)
255{
256
257	return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width));
258}
259
260int
261ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width)
262{
263	struct ntb_child *nc = device_get_ivars(ntb);
264	struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
265	struct ntb_child *nc1;
266
267	for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
268		if (nc1->enabled) {
269			nc->enabled = 1;
270			return (0);
271		}
272	}
273	nc->enabled = 1;
274	return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width));
275}
276
277int
278ntb_link_disable(device_t ntb)
279{
280	struct ntb_child *nc = device_get_ivars(ntb);
281	struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
282	struct ntb_child *nc1;
283
284	if (!nc->enabled)
285		return (0);
286	nc->enabled = 0;
287	for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
288		if (nc1->enabled)
289			return (0);
290	}
291	return (NTB_LINK_DISABLE(device_get_parent(ntb)));
292}
293
294bool
295ntb_link_enabled(device_t ntb)
296{
297	struct ntb_child *nc = device_get_ivars(ntb);
298
299	return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb)));
300}
301
302int
303ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops)
304{
305	struct ntb_child *nc = device_get_ivars(ntb);
306
307	if (ctx == NULL || ctx_ops == NULL)
308		return (EINVAL);
309
310	rm_wlock(&nc->ctx_lock);
311	if (nc->ctx_ops != NULL) {
312		rm_wunlock(&nc->ctx_lock);
313		return (EINVAL);
314	}
315	nc->ctx = ctx;
316	nc->ctx_ops = ctx_ops;
317
318	/*
319	 * If applicaiton driver asks for link events, generate fake one now
320	 * to let it update link state without races while we hold the lock.
321	 */
322	if (ctx_ops->link_event != NULL)
323		ctx_ops->link_event(ctx);
324	rm_wunlock(&nc->ctx_lock);
325
326	return (0);
327}
328
329void *
330ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops)
331{
332	struct ntb_child *nc = device_get_ivars(ntb);
333
334	KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus"));
335	if (ctx_ops != NULL)
336		*ctx_ops = nc->ctx_ops;
337	return (nc->ctx);
338}
339
340void
341ntb_clear_ctx(device_t ntb)
342{
343	struct ntb_child *nc = device_get_ivars(ntb);
344
345	rm_wlock(&nc->ctx_lock);
346	nc->ctx = NULL;
347	nc->ctx_ops = NULL;
348	rm_wunlock(&nc->ctx_lock);
349}
350
351uint8_t
352ntb_mw_count(device_t ntb)
353{
354	struct ntb_child *nc = device_get_ivars(ntb);
355
356	return (nc->mwcnt);
357}
358
359int
360ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base,
361    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
362    bus_addr_t *plimit)
363{
364	struct ntb_child *nc = device_get_ivars(ntb);
365
366	return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff,
367	    base, vbase, size, align, align_size, plimit));
368}
369
370int
371ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size)
372{
373	struct ntb_child *nc = device_get_ivars(ntb);
374
375	return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff,
376	    addr, size));
377}
378
379int
380ntb_mw_clear_trans(device_t ntb, unsigned mw_idx)
381{
382	struct ntb_child *nc = device_get_ivars(ntb);
383
384	return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff));
385}
386
387int
388ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode)
389{
390	struct ntb_child *nc = device_get_ivars(ntb);
391
392	return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
393}
394
395int
396ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode)
397{
398	struct ntb_child *nc = device_get_ivars(ntb);
399
400	return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
401}
402
403uint8_t
404ntb_spad_count(device_t ntb)
405{
406	struct ntb_child *nc = device_get_ivars(ntb);
407
408	return (nc->spadcnt);
409}
410
411void
412ntb_spad_clear(device_t ntb)
413{
414	struct ntb_child *nc = device_get_ivars(ntb);
415	unsigned i;
416
417	for (i = 0; i < nc->spadcnt; i++)
418		NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0);
419}
420
421int
422ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val)
423{
424	struct ntb_child *nc = device_get_ivars(ntb);
425
426	return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val));
427}
428
429int
430ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
431{
432	struct ntb_child *nc = device_get_ivars(ntb);
433
434	return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val));
435}
436
437int
438ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val)
439{
440	struct ntb_child *nc = device_get_ivars(ntb);
441
442	return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff,
443	    val));
444}
445
446int
447ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
448{
449	struct ntb_child *nc = device_get_ivars(ntb);
450
451	return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff,
452	    val));
453}
454
455uint64_t
456ntb_db_valid_mask(device_t ntb)
457{
458	struct ntb_child *nc = device_get_ivars(ntb);
459
460	return (nc->dbmask);
461}
462
463int
464ntb_db_vector_count(device_t ntb)
465{
466
467	return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb)));
468}
469
470uint64_t
471ntb_db_vector_mask(device_t ntb, uint32_t vector)
472{
473	struct ntb_child *nc = device_get_ivars(ntb);
474
475	return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector)
476	    >> nc->dboff) & nc->dbmask);
477}
478
479int
480ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size)
481{
482
483	return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size));
484}
485
486void
487ntb_db_clear(device_t ntb, uint64_t bits)
488{
489	struct ntb_child *nc = device_get_ivars(ntb);
490
491	return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff));
492}
493
494void
495ntb_db_clear_mask(device_t ntb, uint64_t bits)
496{
497	struct ntb_child *nc = device_get_ivars(ntb);
498
499	return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff));
500}
501
502uint64_t
503ntb_db_read(device_t ntb)
504{
505	struct ntb_child *nc = device_get_ivars(ntb);
506
507	return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff)
508	    & nc->dbmask);
509}
510
511void
512ntb_db_set_mask(device_t ntb, uint64_t bits)
513{
514	struct ntb_child *nc = device_get_ivars(ntb);
515
516	return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff));
517}
518
519void
520ntb_peer_db_set(device_t ntb, uint64_t bits)
521{
522	struct ntb_child *nc = device_get_ivars(ntb);
523
524	return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff));
525}
526
527MODULE_VERSION(ntb, 1);
528