1/*-
2 * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by SRI International and the University of
6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
7 * ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31/* ARM PrimeCell DMA Controller (PL330) driver. */
32
33#include <sys/cdefs.h>
34#include "opt_platform.h"
35#include <sys/param.h>
36#include <sys/endian.h>
37#include <sys/systm.h>
38#include <sys/conf.h>
39#include <sys/bus.h>
40#include <sys/kernel.h>
41#include <sys/kthread.h>
42#include <sys/sglist.h>
43#include <sys/module.h>
44#include <sys/lock.h>
45#include <sys/resource.h>
46#include <sys/rman.h>
47
48#include <vm/vm.h>
49#include <vm/vm_extern.h>
50#include <vm/vm_kern.h>
51#include <vm/pmap.h>
52
53#include <machine/bus.h>
54
55#ifdef FDT
56#include <dev/fdt/fdt_common.h>
57#include <dev/ofw/ofw_bus.h>
58#include <dev/ofw/ofw_bus_subr.h>
59#endif
60
61#include <dev/xdma/xdma.h>
62#include <dev/xdma/controller/pl330.h>
63
64#include "xdma_if.h"
65
66#define PL330_DEBUG
67#undef PL330_DEBUG
68
69#ifdef PL330_DEBUG
70#define dprintf(fmt, ...)  printf(fmt, ##__VA_ARGS__)
71#else
72#define dprintf(fmt, ...)
73#endif
74
75#define	READ4(_sc, _reg)	\
76	bus_read_4(_sc->res[0], _reg)
77#define	WRITE4(_sc, _reg, _val)	\
78	bus_write_4(_sc->res[0], _reg, _val)
79
80#define	PL330_NCHANNELS	32
81#define	PL330_MAXLOAD	2048
82
83struct pl330_channel {
84	struct pl330_softc	*sc;
85	xdma_channel_t		*xchan;
86	int			used;
87	int			index;
88	uint8_t			*ibuf;
89	bus_addr_t		ibuf_phys;
90	uint32_t		enqueued;
91	uint32_t		capacity;
92};
93
94struct pl330_fdt_data {
95	uint32_t periph_id;
96};
97
98struct pl330_softc {
99	device_t		dev;
100	struct resource		*res[PL330_NCHANNELS + 1];
101	void			*ih[PL330_NCHANNELS];
102	struct pl330_channel	channels[PL330_NCHANNELS];
103};
104
105static struct resource_spec pl330_spec[] = {
106	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
107	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
108	{ SYS_RES_IRQ,		1,	RF_ACTIVE | RF_OPTIONAL },
109	{ SYS_RES_IRQ,		2,	RF_ACTIVE | RF_OPTIONAL },
110	{ SYS_RES_IRQ,		3,	RF_ACTIVE | RF_OPTIONAL },
111	{ SYS_RES_IRQ,		4,	RF_ACTIVE | RF_OPTIONAL },
112	{ SYS_RES_IRQ,		5,	RF_ACTIVE | RF_OPTIONAL },
113	{ SYS_RES_IRQ,		6,	RF_ACTIVE | RF_OPTIONAL },
114	{ SYS_RES_IRQ,		7,	RF_ACTIVE | RF_OPTIONAL },
115	{ SYS_RES_IRQ,		8,	RF_ACTIVE | RF_OPTIONAL },
116	{ SYS_RES_IRQ,		9,	RF_ACTIVE | RF_OPTIONAL },
117	{ SYS_RES_IRQ,		10,	RF_ACTIVE | RF_OPTIONAL },
118	{ SYS_RES_IRQ,		11,	RF_ACTIVE | RF_OPTIONAL },
119	{ SYS_RES_IRQ,		12,	RF_ACTIVE | RF_OPTIONAL },
120	{ SYS_RES_IRQ,		13,	RF_ACTIVE | RF_OPTIONAL },
121	{ SYS_RES_IRQ,		14,	RF_ACTIVE | RF_OPTIONAL },
122	{ SYS_RES_IRQ,		15,	RF_ACTIVE | RF_OPTIONAL },
123	{ SYS_RES_IRQ,		16,	RF_ACTIVE | RF_OPTIONAL },
124	{ SYS_RES_IRQ,		17,	RF_ACTIVE | RF_OPTIONAL },
125	{ SYS_RES_IRQ,		18,	RF_ACTIVE | RF_OPTIONAL },
126	{ SYS_RES_IRQ,		19,	RF_ACTIVE | RF_OPTIONAL },
127	{ SYS_RES_IRQ,		20,	RF_ACTIVE | RF_OPTIONAL },
128	{ SYS_RES_IRQ,		21,	RF_ACTIVE | RF_OPTIONAL },
129	{ SYS_RES_IRQ,		22,	RF_ACTIVE | RF_OPTIONAL },
130	{ SYS_RES_IRQ,		23,	RF_ACTIVE | RF_OPTIONAL },
131	{ SYS_RES_IRQ,		24,	RF_ACTIVE | RF_OPTIONAL },
132	{ SYS_RES_IRQ,		25,	RF_ACTIVE | RF_OPTIONAL },
133	{ SYS_RES_IRQ,		26,	RF_ACTIVE | RF_OPTIONAL },
134	{ SYS_RES_IRQ,		27,	RF_ACTIVE | RF_OPTIONAL },
135	{ SYS_RES_IRQ,		28,	RF_ACTIVE | RF_OPTIONAL },
136	{ SYS_RES_IRQ,		29,	RF_ACTIVE | RF_OPTIONAL },
137	{ SYS_RES_IRQ,		30,	RF_ACTIVE | RF_OPTIONAL },
138	{ SYS_RES_IRQ,		31,	RF_ACTIVE | RF_OPTIONAL },
139	{ -1, 0 }
140};
141
142#define	HWTYPE_NONE	0
143#define	HWTYPE_STD	1
144
145static struct ofw_compat_data compat_data[] = {
146	{ "arm,pl330",		HWTYPE_STD },
147	{ NULL,			HWTYPE_NONE },
148};
149
150static void
151pl330_intr(void *arg)
152{
153	xdma_transfer_status_t status;
154	struct xdma_transfer_status st;
155	struct pl330_channel *chan;
156	struct xdma_channel *xchan;
157	struct pl330_softc *sc;
158	uint32_t pending;
159	int i;
160	int c;
161
162	sc = arg;
163
164	pending = READ4(sc, INTMIS);
165
166	dprintf("%s: 0x%x, LC0 %x, SAR %x DAR %x\n",
167	    __func__, pending, READ4(sc, LC0(0)),
168	    READ4(sc, SAR(0)), READ4(sc, DAR(0)));
169
170	WRITE4(sc, INTCLR, pending);
171
172	for (c = 0; c < PL330_NCHANNELS; c++) {
173		if ((pending & (1 << c)) == 0) {
174			continue;
175		}
176		chan = &sc->channels[c];
177		xchan = chan->xchan;
178		st.error = 0;
179		st.transferred = 0;
180		for (i = 0; i < chan->enqueued; i++) {
181			xchan_seg_done(xchan, &st);
182		}
183
184		/* Accept new requests. */
185		chan->capacity = PL330_MAXLOAD;
186
187		/* Finish operation */
188		status.error = 0;
189		status.transferred = 0;
190		xdma_callback(chan->xchan, &status);
191	}
192}
193
194static uint32_t
195emit_mov(uint8_t *buf, uint32_t reg, uint32_t val)
196{
197
198	buf[0] = DMAMOV;
199	buf[1] = reg;
200	buf[2] = val;
201	buf[3] = val >> 8;
202	buf[4] = val >> 16;
203	buf[5] = val >> 24;
204
205	return (6);
206}
207
208static uint32_t
209emit_lp(uint8_t *buf, uint8_t idx, uint32_t iter)
210{
211
212	if (idx > 1)
213		return (0); /* We have two loops only. */
214
215	buf[0] = DMALP;
216	buf[0] |= (idx << 1);
217	buf[1] = (iter - 1) & 0xff;
218
219	return (2);
220}
221
222static uint32_t
223emit_lpend(uint8_t *buf, uint8_t idx,
224    uint8_t burst, uint8_t jump_addr_relative)
225{
226
227	buf[0] = DMALPEND;
228	buf[0] |= DMALPEND_NF;
229	buf[0] |= (idx << 2);
230	if (burst)
231		buf[0] |= (1 << 1) | (1 << 0);
232	else
233		buf[0] |= (0 << 1) | (1 << 0);
234	buf[1] = jump_addr_relative;
235
236	return (2);
237}
238
239static uint32_t
240emit_ld(uint8_t *buf, uint8_t burst)
241{
242
243	buf[0] = DMALD;
244	if (burst)
245		buf[0] |= (1 << 1) | (1 << 0);
246	else
247		buf[0] |= (0 << 1) | (1 << 0);
248
249	return (1);
250}
251
252static uint32_t
253emit_st(uint8_t *buf, uint8_t burst)
254{
255
256	buf[0] = DMAST;
257	if (burst)
258		buf[0] |= (1 << 1) | (1 << 0);
259	else
260		buf[0] |= (0 << 1) | (1 << 0);
261
262	return (1);
263}
264
265static uint32_t
266emit_end(uint8_t *buf)
267{
268
269	buf[0] = DMAEND;
270
271	return (1);
272}
273
274static uint32_t
275emit_sev(uint8_t *buf, uint32_t ev)
276{
277
278	buf[0] = DMASEV;
279	buf[1] = (ev << 3);
280
281	return (2);
282}
283
284static uint32_t
285emit_wfp(uint8_t *buf, uint32_t p_id)
286{
287
288	buf[0] = DMAWFP;
289	buf[0] |= (1 << 0);
290	buf[1] = (p_id << 3);
291
292	return (2);
293}
294
295static uint32_t
296emit_go(uint8_t *buf, uint32_t chan_id,
297    uint32_t addr, uint8_t non_secure)
298{
299
300	buf[0] = DMAGO;
301	buf[0] |= (non_secure << 1);
302
303	buf[1] = chan_id;
304	buf[2] = addr;
305	buf[3] = addr >> 8;
306	buf[4] = addr >> 16;
307	buf[5] = addr >> 24;
308
309	return (6);
310}
311
312static int
313pl330_probe(device_t dev)
314{
315	int hwtype;
316
317	if (!ofw_bus_status_okay(dev))
318		return (ENXIO);
319
320	hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
321	if (hwtype == HWTYPE_NONE)
322		return (ENXIO);
323
324	device_set_desc(dev, "ARM PrimeCell DMA Controller (PL330)");
325
326	return (BUS_PROBE_DEFAULT);
327}
328
329static int
330pl330_attach(device_t dev)
331{
332	struct pl330_softc *sc;
333	phandle_t xref, node;
334	int err;
335	int i;
336
337	sc = device_get_softc(dev);
338	sc->dev = dev;
339
340	if (bus_alloc_resources(dev, pl330_spec, sc->res)) {
341		device_printf(dev, "could not allocate resources for device\n");
342		return (ENXIO);
343	}
344
345	/* Setup interrupt handler */
346	for (i = 0; i < PL330_NCHANNELS; i++) {
347		if (sc->res[i + 1] == NULL)
348			break;
349		err = bus_setup_intr(dev, sc->res[i + 1], INTR_TYPE_MISC | INTR_MPSAFE,
350		    NULL, pl330_intr, sc, sc->ih[i]);
351		if (err) {
352			device_printf(dev, "Unable to alloc interrupt resource.\n");
353			return (ENXIO);
354		}
355	}
356
357	node = ofw_bus_get_node(dev);
358	xref = OF_xref_from_node(node);
359	OF_device_register_xref(xref, dev);
360
361	return (0);
362}
363
364static int
365pl330_detach(device_t dev)
366{
367	return (0);
368}
369
370static int
371pl330_channel_alloc(device_t dev, struct xdma_channel *xchan)
372{
373	struct pl330_channel *chan;
374	struct pl330_softc *sc;
375	int i;
376
377	sc = device_get_softc(dev);
378
379	for (i = 0; i < PL330_NCHANNELS; i++) {
380		chan = &sc->channels[i];
381		if (chan->used == 0) {
382			chan->xchan = xchan;
383			xchan->chan = (void *)chan;
384			xchan->caps |= XCHAN_CAP_BUSDMA;
385			chan->index = i;
386			chan->sc = sc;
387			chan->used = 1;
388
389			chan->ibuf = (void *)kmem_alloc_contig(PAGE_SIZE * 8,
390			    M_ZERO, 0, ~0, PAGE_SIZE, 0,
391			    VM_MEMATTR_UNCACHEABLE);
392			chan->ibuf_phys = vtophys(chan->ibuf);
393
394			return (0);
395		}
396	}
397
398	return (-1);
399}
400
401static int
402pl330_channel_free(device_t dev, struct xdma_channel *xchan)
403{
404	struct pl330_channel *chan;
405
406	chan = (struct pl330_channel *)xchan->chan;
407	chan->used = 0;
408
409	return (0);
410}
411
412static int
413pl330_channel_capacity(device_t dev, xdma_channel_t *xchan,
414    uint32_t *capacity)
415{
416	struct pl330_channel *chan;
417
418	chan = (struct pl330_channel *)xchan->chan;
419
420	*capacity = chan->capacity;
421
422	return (0);
423}
424
425static int
426pl330_ccr_port_width(struct xdma_sglist *sg, uint32_t *addr)
427{
428	uint32_t reg;
429
430	reg = 0;
431
432	switch (sg->src_width) {
433	case 1:
434		reg |= CCR_SRC_BURST_SIZE_1;
435		break;
436	case 2:
437		reg |= CCR_SRC_BURST_SIZE_2;
438		break;
439	case 4:
440		reg |= CCR_SRC_BURST_SIZE_4;
441		break;
442	default:
443		return (-1);
444	}
445
446	switch (sg->dst_width) {
447	case 1:
448		reg |= CCR_DST_BURST_SIZE_1;
449		break;
450	case 2:
451		reg |= CCR_DST_BURST_SIZE_2;
452		break;
453	case 4:
454		reg |= CCR_DST_BURST_SIZE_4;
455		break;
456	default:
457		return (-1);
458	}
459
460	*addr |= reg;
461
462	return (0);
463}
464
465static int
466pl330_channel_submit_sg(device_t dev, struct xdma_channel *xchan,
467    struct xdma_sglist *sg, uint32_t sg_n)
468{
469	struct pl330_fdt_data *data;
470	xdma_controller_t *xdma;
471	struct pl330_channel *chan;
472	struct pl330_softc *sc;
473	uint32_t src_addr_lo;
474	uint32_t dst_addr_lo;
475	uint32_t len;
476	uint32_t reg;
477	uint32_t offs;
478	uint32_t cnt;
479	uint8_t *ibuf;
480	uint8_t dbuf[6];
481	uint8_t offs0, offs1;
482	int err;
483	int i;
484
485	sc = device_get_softc(dev);
486
487	xdma = xchan->xdma;
488	data = (struct pl330_fdt_data *)xdma->data;
489
490	chan = (struct pl330_channel *)xchan->chan;
491	ibuf = chan->ibuf;
492
493	dprintf("%s: chan->index %d\n", __func__, chan->index);
494
495	offs = 0;
496
497	for (i = 0; i < sg_n; i++) {
498		if (sg[i].direction == XDMA_DEV_TO_MEM)
499			reg = CCR_DST_INC;
500		else {
501			reg = CCR_SRC_INC;
502			reg |= (CCR_DST_PROT_PRIV);
503		}
504
505		err = pl330_ccr_port_width(&sg[i], &reg);
506		if (err != 0)
507			return (err);
508
509		offs += emit_mov(&chan->ibuf[offs], R_CCR, reg);
510
511		src_addr_lo = (uint32_t)sg[i].src_addr;
512		dst_addr_lo = (uint32_t)sg[i].dst_addr;
513		len = (uint32_t)sg[i].len;
514
515		dprintf("%s: src %x dst %x len %d periph_id %d\n", __func__,
516		    src_addr_lo, dst_addr_lo, len, data->periph_id);
517
518		offs += emit_mov(&ibuf[offs], R_SAR, src_addr_lo);
519		offs += emit_mov(&ibuf[offs], R_DAR, dst_addr_lo);
520
521		if (sg[i].src_width != sg[i].dst_width)
522			return (-1); /* Not supported. */
523
524		cnt = (len / sg[i].src_width);
525		if (cnt > 128) {
526			offs += emit_lp(&ibuf[offs], 0, cnt / 128);
527			offs0 = offs;
528			offs += emit_lp(&ibuf[offs], 1, 128);
529			offs1 = offs;
530		} else {
531			offs += emit_lp(&ibuf[offs], 0, cnt);
532			offs0 = offs;
533		}
534		offs += emit_wfp(&ibuf[offs], data->periph_id);
535		offs += emit_ld(&ibuf[offs], 1);
536		offs += emit_st(&ibuf[offs], 1);
537
538		if (cnt > 128)
539			offs += emit_lpend(&ibuf[offs], 1, 1, (offs - offs1));
540
541		offs += emit_lpend(&ibuf[offs], 0, 1, (offs - offs0));
542	}
543
544	offs += emit_sev(&ibuf[offs], chan->index);
545	offs += emit_end(&ibuf[offs]);
546
547	emit_go(dbuf, chan->index, chan->ibuf_phys, 0);
548
549	reg = (dbuf[1] << 24) | (dbuf[0] << 16);
550	WRITE4(sc, DBGINST0, reg);
551	reg = (dbuf[5] << 24) | (dbuf[4] << 16) | (dbuf[3] << 8) | dbuf[2];
552	WRITE4(sc, DBGINST1, reg);
553
554	WRITE4(sc, INTCLR, 0xffffffff);
555	WRITE4(sc, INTEN, (1 << chan->index));
556
557	chan->enqueued = sg_n;
558	chan->capacity = 0;
559
560	/* Start operation */
561	WRITE4(sc, DBGCMD, 0);
562
563	return (0);
564}
565
566static int
567pl330_channel_prep_sg(device_t dev, struct xdma_channel *xchan)
568{
569	struct pl330_channel *chan;
570
571	dprintf("%s(%d)\n", __func__, device_get_unit(dev));
572
573	chan = (struct pl330_channel *)xchan->chan;
574	chan->capacity = PL330_MAXLOAD;
575
576	return (0);
577}
578
579static int
580pl330_channel_control(device_t dev, xdma_channel_t *xchan, int cmd)
581{
582	switch (cmd) {
583	case XDMA_CMD_BEGIN:
584	case XDMA_CMD_TERMINATE:
585	case XDMA_CMD_PAUSE:
586		/* TODO: implement me */
587		return (-1);
588	}
589
590	return (0);
591}
592
593#ifdef FDT
594static int
595pl330_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr)
596{
597	struct pl330_fdt_data *data;
598
599	if (ncells != 1)
600		return (-1);
601
602	data = malloc(sizeof(struct pl330_fdt_data),
603	    M_DEVBUF, (M_WAITOK | M_ZERO));
604	data->periph_id = cells[0];
605
606	*ptr = data;
607
608	return (0);
609}
610#endif
611
612static device_method_t pl330_methods[] = {
613	/* Device interface */
614	DEVMETHOD(device_probe,			pl330_probe),
615	DEVMETHOD(device_attach,		pl330_attach),
616	DEVMETHOD(device_detach,		pl330_detach),
617
618	/* xDMA Interface */
619	DEVMETHOD(xdma_channel_alloc,		pl330_channel_alloc),
620	DEVMETHOD(xdma_channel_free,		pl330_channel_free),
621	DEVMETHOD(xdma_channel_control,		pl330_channel_control),
622
623	/* xDMA SG Interface */
624	DEVMETHOD(xdma_channel_capacity,	pl330_channel_capacity),
625	DEVMETHOD(xdma_channel_prep_sg,		pl330_channel_prep_sg),
626	DEVMETHOD(xdma_channel_submit_sg,	pl330_channel_submit_sg),
627
628#ifdef FDT
629	DEVMETHOD(xdma_ofw_md_data,		pl330_ofw_md_data),
630#endif
631
632	DEVMETHOD_END
633};
634
635static driver_t pl330_driver = {
636	"pl330",
637	pl330_methods,
638	sizeof(struct pl330_softc),
639};
640
641EARLY_DRIVER_MODULE(pl330, simplebus, pl330_driver, 0, 0,
642    BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE);
643