1/*-
2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#include <sys/param.h>
31#include <sys/kernel.h>
32#include <sys/systm.h>
33#include <sys/bus.h>
34#include <sys/rmlock.h>
35#include <sys/malloc.h>
36#include <sys/module.h>
37#include <sys/sysctl.h>
38
39#include "ntb.h"
40
41devclass_t ntb_hw_devclass;
42SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
43    "NTB sysctls");
44
45struct ntb_child {
46	device_t	dev;
47	int		function;
48	int		enabled;
49	int		mwoff;
50	int		mwcnt;
51	int		spadoff;
52	int		spadcnt;
53	int		dboff;
54	int		dbcnt;
55	uint64_t	dbmask;
56	void		*ctx;
57	const struct ntb_ctx_ops *ctx_ops;
58	struct rmlock	ctx_lock;
59	struct ntb_child *next;
60};
61
62int
63ntb_register_device(device_t dev)
64{
65	struct ntb_child **cpp = device_get_softc(dev);
66	struct ntb_child *nc;
67	int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt;
68	char cfg[128] = "";
69	char buf[32];
70	char *n, *np, *c, *p, *name;
71
72	mwu = 0;
73	mwt = NTB_MW_COUNT(dev);
74	spadu = 0;
75	spadt = NTB_SPAD_COUNT(dev);
76	dbu = 0;
77	dbt = flsll(NTB_DB_VALID_MASK(dev));
78
79	device_printf(dev, "%d memory windows, %d scratchpads, "
80	    "%d doorbells\n", mwt, spadt, dbt);
81
82	snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev),
83	    device_get_unit(dev));
84	TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg));
85	n = cfg;
86	i = 0;
87	while ((c = strsep(&n, ",")) != NULL) {
88		np = c;
89		name = strsep(&np, ":");
90		if (name != NULL && name[0] == 0)
91			name = NULL;
92		p = strsep(&np, ":");
93		mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu;
94		p = strsep(&np, ":");
95		spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu;
96		db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu;
97
98		if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) {
99			device_printf(dev, "Not enough resources for config\n");
100			break;
101		}
102
103		nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO);
104		nc->function = i;
105		nc->mwoff = mwu;
106		nc->mwcnt = mw;
107		nc->spadoff = spadu;
108		nc->spadcnt = spad;
109		nc->dboff = dbu;
110		nc->dbcnt = db;
111		nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db));
112		rm_init(&nc->ctx_lock, "ntb ctx");
113		nc->dev = device_add_child(dev, name, -1);
114		if (nc->dev == NULL) {
115			ntb_unregister_device(dev);
116			return (ENOMEM);
117		}
118		device_set_ivars(nc->dev, nc);
119		*cpp = nc;
120		cpp = &nc->next;
121
122		if (bootverbose) {
123			device_printf(dev, "%d \"%s\":", i, name);
124			if (mw > 0) {
125				printf(" memory windows %d", mwu);
126				if (mw > 1)
127					printf("-%d", mwu + mw - 1);
128			}
129			if (spad > 0) {
130				printf(" scratchpads %d", spadu);
131				if (spad > 1)
132					printf("-%d", spadu + spad - 1);
133			}
134			if (db > 0) {
135				printf(" doorbells %d", dbu);
136				if (db > 1)
137					printf("-%d", dbu + db - 1);
138			}
139			printf("\n");
140		}
141
142		mwu += mw;
143		spadu += spad;
144		dbu += db;
145		i++;
146	}
147
148	bus_generic_attach(dev);
149	return (0);
150}
151
152int
153ntb_unregister_device(device_t dev)
154{
155	struct ntb_child **cpp = device_get_softc(dev);
156	struct ntb_child *nc;
157	int error = 0;
158
159	while ((nc = *cpp) != NULL) {
160		*cpp = (*cpp)->next;
161		error = device_delete_child(dev, nc->dev);
162		if (error)
163			break;
164		rm_destroy(&nc->ctx_lock);
165		free(nc, M_DEVBUF);
166	}
167	return (error);
168}
169
170int
171ntb_child_location_str(device_t dev, device_t child, char *buf,
172    size_t buflen)
173{
174	struct ntb_child *nc = device_get_ivars(child);
175
176	snprintf(buf, buflen, "function=%d", nc->function);
177	return (0);
178}
179
180int
181ntb_print_child(device_t dev, device_t child)
182{
183	struct ntb_child *nc = device_get_ivars(child);
184	int retval;
185
186	retval = bus_print_child_header(dev, child);
187	if (nc->mwcnt > 0) {
188		printf(" mw %d", nc->mwoff);
189		if (nc->mwcnt > 1)
190			printf("-%d", nc->mwoff + nc->mwcnt - 1);
191	}
192	if (nc->spadcnt > 0) {
193		printf(" spad %d", nc->spadoff);
194		if (nc->spadcnt > 1)
195			printf("-%d", nc->spadoff + nc->spadcnt - 1);
196	}
197	if (nc->dbcnt > 0) {
198		printf(" db %d", nc->dboff);
199		if (nc->dbcnt > 1)
200			printf("-%d", nc->dboff + nc->dbcnt - 1);
201	}
202	retval += printf(" at function %d", nc->function);
203	retval += bus_print_child_domain(dev, child);
204	retval += bus_print_child_footer(dev, child);
205
206	return (retval);
207}
208
209bus_dma_tag_t
210ntb_get_dma_tag(device_t bus, device_t child)
211{
212
213	return (bus_get_dma_tag(bus));
214}
215
216void
217ntb_link_event(device_t dev)
218{
219	struct ntb_child **cpp = device_get_softc(dev);
220	struct ntb_child *nc;
221	struct rm_priotracker ctx_tracker;
222	enum ntb_speed speed;
223	enum ntb_width width;
224
225	if (NTB_LINK_IS_UP(dev, &speed, &width)) {
226		device_printf(dev, "Link is up (PCIe %d.x / x%d)\n",
227		    (int)speed, (int)width);
228	} else {
229		device_printf(dev, "Link is down\n");
230	}
231	for (nc = *cpp; nc != NULL; nc = nc->next) {
232		rm_rlock(&nc->ctx_lock, &ctx_tracker);
233		if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL)
234			nc->ctx_ops->link_event(nc->ctx);
235		rm_runlock(&nc->ctx_lock, &ctx_tracker);
236	}
237}
238
239void
240ntb_db_event(device_t dev, uint32_t vec)
241{
242	struct ntb_child **cpp = device_get_softc(dev);
243	struct ntb_child *nc;
244	struct rm_priotracker ctx_tracker;
245
246	for (nc = *cpp; nc != NULL; nc = nc->next) {
247		rm_rlock(&nc->ctx_lock, &ctx_tracker);
248		if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL)
249			nc->ctx_ops->db_event(nc->ctx, vec);
250		rm_runlock(&nc->ctx_lock, &ctx_tracker);
251	}
252}
253
254int
255ntb_port_number(device_t ntb)
256{
257	return (NTB_PORT_NUMBER(device_get_parent(ntb)));
258}
259
260int
261ntb_peer_port_count(device_t ntb)
262{
263	return (NTB_PEER_PORT_COUNT(device_get_parent(ntb)));
264}
265
266int
267ntb_peer_port_number(device_t ntb, int pidx)
268{
269	return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx));
270}
271
272int
273ntb_peer_port_idx(device_t ntb, int port)
274{
275	return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port));
276}
277
278bool
279ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width)
280{
281
282	return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width));
283}
284
285int
286ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width)
287{
288	struct ntb_child *nc = device_get_ivars(ntb);
289	struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
290	struct ntb_child *nc1;
291
292	for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
293		if (nc1->enabled) {
294			nc->enabled = 1;
295			return (0);
296		}
297	}
298	nc->enabled = 1;
299	return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width));
300}
301
302int
303ntb_link_disable(device_t ntb)
304{
305	struct ntb_child *nc = device_get_ivars(ntb);
306	struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
307	struct ntb_child *nc1;
308
309	if (!nc->enabled)
310		return (0);
311	nc->enabled = 0;
312	for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
313		if (nc1->enabled)
314			return (0);
315	}
316	return (NTB_LINK_DISABLE(device_get_parent(ntb)));
317}
318
319bool
320ntb_link_enabled(device_t ntb)
321{
322	struct ntb_child *nc = device_get_ivars(ntb);
323
324	return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb)));
325}
326
327int
328ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops)
329{
330	struct ntb_child *nc = device_get_ivars(ntb);
331
332	if (ctx == NULL || ctx_ops == NULL)
333		return (EINVAL);
334
335	rm_wlock(&nc->ctx_lock);
336	if (nc->ctx_ops != NULL) {
337		rm_wunlock(&nc->ctx_lock);
338		return (EINVAL);
339	}
340	nc->ctx = ctx;
341	nc->ctx_ops = ctx_ops;
342
343	/*
344	 * If applicaiton driver asks for link events, generate fake one now
345	 * to let it update link state without races while we hold the lock.
346	 */
347	if (ctx_ops->link_event != NULL)
348		ctx_ops->link_event(ctx);
349	rm_wunlock(&nc->ctx_lock);
350
351	return (0);
352}
353
354void *
355ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops)
356{
357	struct ntb_child *nc = device_get_ivars(ntb);
358
359	KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus"));
360	if (ctx_ops != NULL)
361		*ctx_ops = nc->ctx_ops;
362	return (nc->ctx);
363}
364
365void
366ntb_clear_ctx(device_t ntb)
367{
368	struct ntb_child *nc = device_get_ivars(ntb);
369
370	rm_wlock(&nc->ctx_lock);
371	nc->ctx = NULL;
372	nc->ctx_ops = NULL;
373	rm_wunlock(&nc->ctx_lock);
374}
375
376uint8_t
377ntb_mw_count(device_t ntb)
378{
379	struct ntb_child *nc = device_get_ivars(ntb);
380
381	return (nc->mwcnt);
382}
383
384int
385ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base,
386    caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
387    bus_addr_t *plimit)
388{
389	struct ntb_child *nc = device_get_ivars(ntb);
390
391	return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff,
392	    base, vbase, size, align, align_size, plimit));
393}
394
395int
396ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size)
397{
398	struct ntb_child *nc = device_get_ivars(ntb);
399
400	return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff,
401	    addr, size));
402}
403
404int
405ntb_mw_clear_trans(device_t ntb, unsigned mw_idx)
406{
407	struct ntb_child *nc = device_get_ivars(ntb);
408
409	return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff));
410}
411
412int
413ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode)
414{
415	struct ntb_child *nc = device_get_ivars(ntb);
416
417	return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
418}
419
420int
421ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode)
422{
423	struct ntb_child *nc = device_get_ivars(ntb);
424
425	return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
426}
427
428uint8_t
429ntb_spad_count(device_t ntb)
430{
431	struct ntb_child *nc = device_get_ivars(ntb);
432
433	return (nc->spadcnt);
434}
435
436void
437ntb_spad_clear(device_t ntb)
438{
439	struct ntb_child *nc = device_get_ivars(ntb);
440	unsigned i;
441
442	for (i = 0; i < nc->spadcnt; i++)
443		NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0);
444}
445
446int
447ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val)
448{
449	struct ntb_child *nc = device_get_ivars(ntb);
450
451	return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val));
452}
453
454int
455ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
456{
457	struct ntb_child *nc = device_get_ivars(ntb);
458
459	return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val));
460}
461
462int
463ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val)
464{
465	struct ntb_child *nc = device_get_ivars(ntb);
466
467	return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff,
468	    val));
469}
470
471int
472ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
473{
474	struct ntb_child *nc = device_get_ivars(ntb);
475
476	return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff,
477	    val));
478}
479
480uint64_t
481ntb_db_valid_mask(device_t ntb)
482{
483	struct ntb_child *nc = device_get_ivars(ntb);
484
485	return (nc->dbmask);
486}
487
488int
489ntb_db_vector_count(device_t ntb)
490{
491
492	return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb)));
493}
494
495uint64_t
496ntb_db_vector_mask(device_t ntb, uint32_t vector)
497{
498	struct ntb_child *nc = device_get_ivars(ntb);
499
500	return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector)
501	    >> nc->dboff) & nc->dbmask);
502}
503
504int
505ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size)
506{
507
508	return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size));
509}
510
511void
512ntb_db_clear(device_t ntb, uint64_t bits)
513{
514	struct ntb_child *nc = device_get_ivars(ntb);
515
516	return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff));
517}
518
519void
520ntb_db_clear_mask(device_t ntb, uint64_t bits)
521{
522	struct ntb_child *nc = device_get_ivars(ntb);
523
524	return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff));
525}
526
527uint64_t
528ntb_db_read(device_t ntb)
529{
530	struct ntb_child *nc = device_get_ivars(ntb);
531
532	return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff)
533	    & nc->dbmask);
534}
535
536void
537ntb_db_set_mask(device_t ntb, uint64_t bits)
538{
539	struct ntb_child *nc = device_get_ivars(ntb);
540
541	return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff));
542}
543
544void
545ntb_peer_db_set(device_t ntb, uint64_t bits)
546{
547	struct ntb_child *nc = device_get_ivars(ntb);
548
549	return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff));
550}
551
552MODULE_VERSION(ntb, 1);
553