1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * offload engine driver for the Marvell XOR engine
4 * Copyright (C) 2007, 2008, Marvell International Ltd.
5 */
6
7#include <linux/init.h>
8#include <linux/slab.h>
9#include <linux/delay.h>
10#include <linux/dma-mapping.h>
11#include <linux/spinlock.h>
12#include <linux/interrupt.h>
13#include <linux/of_device.h>
14#include <linux/platform_device.h>
15#include <linux/memory.h>
16#include <linux/clk.h>
17#include <linux/of.h>
18#include <linux/of_irq.h>
19#include <linux/irqdomain.h>
20#include <linux/cpumask.h>
21#include <linux/platform_data/dma-mv_xor.h>
22
23#include "dmaengine.h"
24#include "mv_xor.h"
25
26enum mv_xor_type {
27	XOR_ORION,
28	XOR_ARMADA_38X,
29	XOR_ARMADA_37XX,
30};
31
32enum mv_xor_mode {
33	XOR_MODE_IN_REG,
34	XOR_MODE_IN_DESC,
35};
36
37static void mv_xor_issue_pending(struct dma_chan *chan);
38
39#define to_mv_xor_chan(chan)		\
40	container_of(chan, struct mv_xor_chan, dmachan)
41
42#define to_mv_xor_slot(tx)		\
43	container_of(tx, struct mv_xor_desc_slot, async_tx)
44
45#define mv_chan_to_devp(chan)           \
46	((chan)->dmadev.dev)
47
48static void mv_desc_init(struct mv_xor_desc_slot *desc,
49			 dma_addr_t addr, u32 byte_count,
50			 enum dma_ctrl_flags flags)
51{
52	struct mv_xor_desc *hw_desc = desc->hw_desc;
53
54	hw_desc->status = XOR_DESC_DMA_OWNED;
55	hw_desc->phy_next_desc = 0;
56	/* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
57	hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
58				XOR_DESC_EOD_INT_EN : 0;
59	hw_desc->phy_dest_addr = addr;
60	hw_desc->byte_count = byte_count;
61}
62
63static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
64{
65	struct mv_xor_desc *hw_desc = desc->hw_desc;
66
67	switch (desc->type) {
68	case DMA_XOR:
69	case DMA_INTERRUPT:
70		hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
71		break;
72	case DMA_MEMCPY:
73		hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
74		break;
75	default:
76		BUG();
77		return;
78	}
79}
80
81static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
82				  u32 next_desc_addr)
83{
84	struct mv_xor_desc *hw_desc = desc->hw_desc;
85	BUG_ON(hw_desc->phy_next_desc);
86	hw_desc->phy_next_desc = next_desc_addr;
87}
88
89static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
90				 int index, dma_addr_t addr)
91{
92	struct mv_xor_desc *hw_desc = desc->hw_desc;
93	hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
94	if (desc->type == DMA_XOR)
95		hw_desc->desc_command |= (1 << index);
96}
97
98static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
99{
100	return readl_relaxed(XOR_CURR_DESC(chan));
101}
102
103static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
104					u32 next_desc_addr)
105{
106	writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
107}
108
109static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
110{
111	u32 val = readl_relaxed(XOR_INTR_MASK(chan));
112	val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
113	writel_relaxed(val, XOR_INTR_MASK(chan));
114}
115
116static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
117{
118	u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
119	intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
120	return intr_cause;
121}
122
123static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
124{
125	u32 val;
126
127	val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
128	val = ~(val << (chan->idx * 16));
129	dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
130	writel_relaxed(val, XOR_INTR_CAUSE(chan));
131}
132
133static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
134{
135	u32 val = 0xFFFF0000 >> (chan->idx * 16);
136	writel_relaxed(val, XOR_INTR_CAUSE(chan));
137}
138
139static void mv_chan_set_mode(struct mv_xor_chan *chan,
140			     u32 op_mode)
141{
142	u32 config = readl_relaxed(XOR_CONFIG(chan));
143
144	config &= ~0x7;
145	config |= op_mode;
146
147#if defined(__BIG_ENDIAN)
148	config |= XOR_DESCRIPTOR_SWAP;
149#else
150	config &= ~XOR_DESCRIPTOR_SWAP;
151#endif
152
153	writel_relaxed(config, XOR_CONFIG(chan));
154}
155
156static void mv_chan_activate(struct mv_xor_chan *chan)
157{
158	dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
159
160	/* writel ensures all descriptors are flushed before activation */
161	writel(BIT(0), XOR_ACTIVATION(chan));
162}
163
164static char mv_chan_is_busy(struct mv_xor_chan *chan)
165{
166	u32 state = readl_relaxed(XOR_ACTIVATION(chan));
167
168	state = (state >> 4) & 0x3;
169
170	return (state == 1) ? 1 : 0;
171}
172
173/*
174 * mv_chan_start_new_chain - program the engine to operate on new
175 * chain headed by sw_desc
176 * Caller must hold &mv_chan->lock while calling this function
177 */
178static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
179				    struct mv_xor_desc_slot *sw_desc)
180{
181	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
182		__func__, __LINE__, sw_desc);
183
184	/* set the hardware chain */
185	mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
186
187	mv_chan->pending++;
188	mv_xor_issue_pending(&mv_chan->dmachan);
189}
190
191static dma_cookie_t
192mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
193				struct mv_xor_chan *mv_chan,
194				dma_cookie_t cookie)
195{
196	BUG_ON(desc->async_tx.cookie < 0);
197
198	if (desc->async_tx.cookie > 0) {
199		cookie = desc->async_tx.cookie;
200
201		dma_descriptor_unmap(&desc->async_tx);
202		/* call the callback (must not sleep or submit new
203		 * operations to this channel)
204		 */
205		dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
206	}
207
208	/* run dependent operations */
209	dma_run_dependencies(&desc->async_tx);
210
211	return cookie;
212}
213
214static int
215mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
216{
217	struct mv_xor_desc_slot *iter, *_iter;
218
219	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
220	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
221				 node) {
222
223		if (async_tx_test_ack(&iter->async_tx)) {
224			list_move_tail(&iter->node, &mv_chan->free_slots);
225			if (!list_empty(&iter->sg_tx_list)) {
226				list_splice_tail_init(&iter->sg_tx_list,
227							&mv_chan->free_slots);
228			}
229		}
230	}
231	return 0;
232}
233
234static int
235mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
236		   struct mv_xor_chan *mv_chan)
237{
238	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
239		__func__, __LINE__, desc, desc->async_tx.flags);
240
241	/* the client is allowed to attach dependent operations
242	 * until 'ack' is set
243	 */
244	if (!async_tx_test_ack(&desc->async_tx)) {
245		/* move this slot to the completed_slots */
246		list_move_tail(&desc->node, &mv_chan->completed_slots);
247		if (!list_empty(&desc->sg_tx_list)) {
248			list_splice_tail_init(&desc->sg_tx_list,
249					      &mv_chan->completed_slots);
250		}
251	} else {
252		list_move_tail(&desc->node, &mv_chan->free_slots);
253		if (!list_empty(&desc->sg_tx_list)) {
254			list_splice_tail_init(&desc->sg_tx_list,
255					      &mv_chan->free_slots);
256		}
257	}
258
259	return 0;
260}
261
262/* This function must be called with the mv_xor_chan spinlock held */
263static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
264{
265	struct mv_xor_desc_slot *iter, *_iter;
266	dma_cookie_t cookie = 0;
267	int busy = mv_chan_is_busy(mv_chan);
268	u32 current_desc = mv_chan_get_current_desc(mv_chan);
269	int current_cleaned = 0;
270	struct mv_xor_desc *hw_desc;
271
272	dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
273	dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
274	mv_chan_clean_completed_slots(mv_chan);
275
276	/* free completed slots from the chain starting with
277	 * the oldest descriptor
278	 */
279
280	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
281				 node) {
282
283		/* clean finished descriptors */
284		hw_desc = iter->hw_desc;
285		if (hw_desc->status & XOR_DESC_SUCCESS) {
286			cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
287								 cookie);
288
289			/* done processing desc, clean slot */
290			mv_desc_clean_slot(iter, mv_chan);
291
292			/* break if we did cleaned the current */
293			if (iter->async_tx.phys == current_desc) {
294				current_cleaned = 1;
295				break;
296			}
297		} else {
298			if (iter->async_tx.phys == current_desc) {
299				current_cleaned = 0;
300				break;
301			}
302		}
303	}
304
305	if ((busy == 0) && !list_empty(&mv_chan->chain)) {
306		if (current_cleaned) {
307			/*
308			 * current descriptor cleaned and removed, run
309			 * from list head
310			 */
311			iter = list_entry(mv_chan->chain.next,
312					  struct mv_xor_desc_slot,
313					  node);
314			mv_chan_start_new_chain(mv_chan, iter);
315		} else {
316			if (!list_is_last(&iter->node, &mv_chan->chain)) {
317				/*
318				 * descriptors are still waiting after
319				 * current, trigger them
320				 */
321				iter = list_entry(iter->node.next,
322						  struct mv_xor_desc_slot,
323						  node);
324				mv_chan_start_new_chain(mv_chan, iter);
325			} else {
326				/*
327				 * some descriptors are still waiting
328				 * to be cleaned
329				 */
330				tasklet_schedule(&mv_chan->irq_tasklet);
331			}
332		}
333	}
334
335	if (cookie > 0)
336		mv_chan->dmachan.completed_cookie = cookie;
337}
338
339static void mv_xor_tasklet(struct tasklet_struct *t)
340{
341	struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet);
342
343	spin_lock(&chan->lock);
344	mv_chan_slot_cleanup(chan);
345	spin_unlock(&chan->lock);
346}
347
348static struct mv_xor_desc_slot *
349mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
350{
351	struct mv_xor_desc_slot *iter;
352
353	spin_lock_bh(&mv_chan->lock);
354
355	if (!list_empty(&mv_chan->free_slots)) {
356		iter = list_first_entry(&mv_chan->free_slots,
357					struct mv_xor_desc_slot,
358					node);
359
360		list_move_tail(&iter->node, &mv_chan->allocated_slots);
361
362		spin_unlock_bh(&mv_chan->lock);
363
364		/* pre-ack descriptor */
365		async_tx_ack(&iter->async_tx);
366		iter->async_tx.cookie = -EBUSY;
367
368		return iter;
369
370	}
371
372	spin_unlock_bh(&mv_chan->lock);
373
374	/* try to free some slots if the allocation fails */
375	tasklet_schedule(&mv_chan->irq_tasklet);
376
377	return NULL;
378}
379
380/************************ DMA engine API functions ****************************/
381static dma_cookie_t
382mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
383{
384	struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
385	struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
386	struct mv_xor_desc_slot *old_chain_tail;
387	dma_cookie_t cookie;
388	int new_hw_chain = 1;
389
390	dev_dbg(mv_chan_to_devp(mv_chan),
391		"%s sw_desc %p: async_tx %p\n",
392		__func__, sw_desc, &sw_desc->async_tx);
393
394	spin_lock_bh(&mv_chan->lock);
395	cookie = dma_cookie_assign(tx);
396
397	if (list_empty(&mv_chan->chain))
398		list_move_tail(&sw_desc->node, &mv_chan->chain);
399	else {
400		new_hw_chain = 0;
401
402		old_chain_tail = list_entry(mv_chan->chain.prev,
403					    struct mv_xor_desc_slot,
404					    node);
405		list_move_tail(&sw_desc->node, &mv_chan->chain);
406
407		dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
408			&old_chain_tail->async_tx.phys);
409
410		/* fix up the hardware chain */
411		mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
412
413		/* if the channel is not busy */
414		if (!mv_chan_is_busy(mv_chan)) {
415			u32 current_desc = mv_chan_get_current_desc(mv_chan);
416			/*
417			 * and the curren desc is the end of the chain before
418			 * the append, then we need to start the channel
419			 */
420			if (current_desc == old_chain_tail->async_tx.phys)
421				new_hw_chain = 1;
422		}
423	}
424
425	if (new_hw_chain)
426		mv_chan_start_new_chain(mv_chan, sw_desc);
427
428	spin_unlock_bh(&mv_chan->lock);
429
430	return cookie;
431}
432
433/* returns the number of allocated descriptors */
434static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
435{
436	void *virt_desc;
437	dma_addr_t dma_desc;
438	int idx;
439	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
440	struct mv_xor_desc_slot *slot = NULL;
441	int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
442
443	/* Allocate descriptor slots */
444	idx = mv_chan->slots_allocated;
445	while (idx < num_descs_in_pool) {
446		slot = kzalloc(sizeof(*slot), GFP_KERNEL);
447		if (!slot) {
448			dev_info(mv_chan_to_devp(mv_chan),
449				 "channel only initialized %d descriptor slots",
450				 idx);
451			break;
452		}
453		virt_desc = mv_chan->dma_desc_pool_virt;
454		slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
455
456		dma_async_tx_descriptor_init(&slot->async_tx, chan);
457		slot->async_tx.tx_submit = mv_xor_tx_submit;
458		INIT_LIST_HEAD(&slot->node);
459		INIT_LIST_HEAD(&slot->sg_tx_list);
460		dma_desc = mv_chan->dma_desc_pool;
461		slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
462		slot->idx = idx++;
463
464		spin_lock_bh(&mv_chan->lock);
465		mv_chan->slots_allocated = idx;
466		list_add_tail(&slot->node, &mv_chan->free_slots);
467		spin_unlock_bh(&mv_chan->lock);
468	}
469
470	dev_dbg(mv_chan_to_devp(mv_chan),
471		"allocated %d descriptor slots\n",
472		mv_chan->slots_allocated);
473
474	return mv_chan->slots_allocated ? : -ENOMEM;
475}
476
477/*
478 * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
479 * a new MBus window if necessary. Use a cache for these check so that
480 * the MMIO mapped registers don't have to be accessed for this check
481 * to speed up this process.
482 */
483static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
484{
485	struct mv_xor_device *xordev = mv_chan->xordev;
486	void __iomem *base = mv_chan->mmr_high_base;
487	u32 win_enable;
488	u32 size;
489	u8 target, attr;
490	int ret;
491	int i;
492
493	/* Nothing needs to get done for the Armada 3700 */
494	if (xordev->xor_type == XOR_ARMADA_37XX)
495		return 0;
496
497	/*
498	 * Loop over the cached windows to check, if the requested area
499	 * is already mapped. If this the case, nothing needs to be done
500	 * and we can return.
501	 */
502	for (i = 0; i < WINDOW_COUNT; i++) {
503		if (addr >= xordev->win_start[i] &&
504		    addr <= xordev->win_end[i]) {
505			/* Window is already mapped */
506			return 0;
507		}
508	}
509
510	/*
511	 * The window is not mapped, so we need to create the new mapping
512	 */
513
514	/* If no IO window is found that addr has to be located in SDRAM */
515	ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
516	if (ret < 0)
517		return 0;
518
519	/*
520	 * Mask the base addr 'addr' according to 'size' read back from the
521	 * MBus window. Otherwise we might end up with an address located
522	 * somewhere in the middle of this area here.
523	 */
524	size -= 1;
525	addr &= ~size;
526
527	/*
528	 * Reading one of both enabled register is enough, as they are always
529	 * programmed to the identical values
530	 */
531	win_enable = readl(base + WINDOW_BAR_ENABLE(0));
532
533	/* Set 'i' to the first free window to write the new values to */
534	i = ffs(~win_enable) - 1;
535	if (i >= WINDOW_COUNT)
536		return -ENOMEM;
537
538	writel((addr & 0xffff0000) | (attr << 8) | target,
539	       base + WINDOW_BASE(i));
540	writel(size & 0xffff0000, base + WINDOW_SIZE(i));
541
542	/* Fill the caching variables for later use */
543	xordev->win_start[i] = addr;
544	xordev->win_end[i] = addr + size;
545
546	win_enable |= (1 << i);
547	win_enable |= 3 << (16 + (2 * i));
548	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
549	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
550
551	return 0;
552}
553
554static struct dma_async_tx_descriptor *
555mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
556		    unsigned int src_cnt, size_t len, unsigned long flags)
557{
558	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
559	struct mv_xor_desc_slot *sw_desc;
560	int ret;
561
562	if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
563		return NULL;
564
565	BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
566
567	dev_dbg(mv_chan_to_devp(mv_chan),
568		"%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
569		__func__, src_cnt, len, &dest, flags);
570
571	/* Check if a new window needs to get added for 'dest' */
572	ret = mv_xor_add_io_win(mv_chan, dest);
573	if (ret)
574		return NULL;
575
576	sw_desc = mv_chan_alloc_slot(mv_chan);
577	if (sw_desc) {
578		sw_desc->type = DMA_XOR;
579		sw_desc->async_tx.flags = flags;
580		mv_desc_init(sw_desc, dest, len, flags);
581		if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
582			mv_desc_set_mode(sw_desc);
583		while (src_cnt--) {
584			/* Check if a new window needs to get added for 'src' */
585			ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
586			if (ret)
587				return NULL;
588			mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
589		}
590	}
591
592	dev_dbg(mv_chan_to_devp(mv_chan),
593		"%s sw_desc %p async_tx %p \n",
594		__func__, sw_desc, &sw_desc->async_tx);
595	return sw_desc ? &sw_desc->async_tx : NULL;
596}
597
598static struct dma_async_tx_descriptor *
599mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
600		size_t len, unsigned long flags)
601{
602	/*
603	 * A MEMCPY operation is identical to an XOR operation with only
604	 * a single source address.
605	 */
606	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
607}
608
609static struct dma_async_tx_descriptor *
610mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
611{
612	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
613	dma_addr_t src, dest;
614	size_t len;
615
616	src = mv_chan->dummy_src_addr;
617	dest = mv_chan->dummy_dst_addr;
618	len = MV_XOR_MIN_BYTE_COUNT;
619
620	/*
621	 * We implement the DMA_INTERRUPT operation as a minimum sized
622	 * XOR operation with a single dummy source address.
623	 */
624	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
625}
626
627static void mv_xor_free_chan_resources(struct dma_chan *chan)
628{
629	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
630	struct mv_xor_desc_slot *iter, *_iter;
631	int in_use_descs = 0;
632
633	spin_lock_bh(&mv_chan->lock);
634
635	mv_chan_slot_cleanup(mv_chan);
636
637	list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
638					node) {
639		in_use_descs++;
640		list_move_tail(&iter->node, &mv_chan->free_slots);
641	}
642	list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
643				 node) {
644		in_use_descs++;
645		list_move_tail(&iter->node, &mv_chan->free_slots);
646	}
647	list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
648				 node) {
649		in_use_descs++;
650		list_move_tail(&iter->node, &mv_chan->free_slots);
651	}
652	list_for_each_entry_safe_reverse(
653		iter, _iter, &mv_chan->free_slots, node) {
654		list_del(&iter->node);
655		kfree(iter);
656		mv_chan->slots_allocated--;
657	}
658
659	dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
660		__func__, mv_chan->slots_allocated);
661	spin_unlock_bh(&mv_chan->lock);
662
663	if (in_use_descs)
664		dev_err(mv_chan_to_devp(mv_chan),
665			"freeing %d in use descriptors!\n", in_use_descs);
666}
667
668/**
669 * mv_xor_status - poll the status of an XOR transaction
670 * @chan: XOR channel handle
671 * @cookie: XOR transaction identifier
672 * @txstate: XOR transactions state holder (or NULL)
673 */
674static enum dma_status mv_xor_status(struct dma_chan *chan,
675					  dma_cookie_t cookie,
676					  struct dma_tx_state *txstate)
677{
678	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
679	enum dma_status ret;
680
681	ret = dma_cookie_status(chan, cookie, txstate);
682	if (ret == DMA_COMPLETE)
683		return ret;
684
685	spin_lock_bh(&mv_chan->lock);
686	mv_chan_slot_cleanup(mv_chan);
687	spin_unlock_bh(&mv_chan->lock);
688
689	return dma_cookie_status(chan, cookie, txstate);
690}
691
692static void mv_chan_dump_regs(struct mv_xor_chan *chan)
693{
694	u32 val;
695
696	val = readl_relaxed(XOR_CONFIG(chan));
697	dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
698
699	val = readl_relaxed(XOR_ACTIVATION(chan));
700	dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
701
702	val = readl_relaxed(XOR_INTR_CAUSE(chan));
703	dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
704
705	val = readl_relaxed(XOR_INTR_MASK(chan));
706	dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
707
708	val = readl_relaxed(XOR_ERROR_CAUSE(chan));
709	dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
710
711	val = readl_relaxed(XOR_ERROR_ADDR(chan));
712	dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
713}
714
715static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
716					  u32 intr_cause)
717{
718	if (intr_cause & XOR_INT_ERR_DECODE) {
719		dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
720		return;
721	}
722
723	dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
724		chan->idx, intr_cause);
725
726	mv_chan_dump_regs(chan);
727	WARN_ON(1);
728}
729
730static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
731{
732	struct mv_xor_chan *chan = data;
733	u32 intr_cause = mv_chan_get_intr_cause(chan);
734
735	dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
736
737	if (intr_cause & XOR_INTR_ERRORS)
738		mv_chan_err_interrupt_handler(chan, intr_cause);
739
740	tasklet_schedule(&chan->irq_tasklet);
741
742	mv_chan_clear_eoc_cause(chan);
743
744	return IRQ_HANDLED;
745}
746
747static void mv_xor_issue_pending(struct dma_chan *chan)
748{
749	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
750
751	if (mv_chan->pending >= MV_XOR_THRESHOLD) {
752		mv_chan->pending = 0;
753		mv_chan_activate(mv_chan);
754	}
755}
756
757/*
758 * Perform a transaction to verify the HW works.
759 */
760
761static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
762{
763	int i, ret;
764	void *src, *dest;
765	dma_addr_t src_dma, dest_dma;
766	struct dma_chan *dma_chan;
767	dma_cookie_t cookie;
768	struct dma_async_tx_descriptor *tx;
769	struct dmaengine_unmap_data *unmap;
770	int err = 0;
771
772	src = kmalloc(PAGE_SIZE, GFP_KERNEL);
773	if (!src)
774		return -ENOMEM;
775
776	dest = kzalloc(PAGE_SIZE, GFP_KERNEL);
777	if (!dest) {
778		kfree(src);
779		return -ENOMEM;
780	}
781
782	/* Fill in src buffer */
783	for (i = 0; i < PAGE_SIZE; i++)
784		((u8 *) src)[i] = (u8)i;
785
786	dma_chan = &mv_chan->dmachan;
787	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
788		err = -ENODEV;
789		goto out;
790	}
791
792	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
793	if (!unmap) {
794		err = -ENOMEM;
795		goto free_resources;
796	}
797
798	src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
799			       offset_in_page(src), PAGE_SIZE,
800			       DMA_TO_DEVICE);
801	unmap->addr[0] = src_dma;
802
803	ret = dma_mapping_error(dma_chan->device->dev, src_dma);
804	if (ret) {
805		err = -ENOMEM;
806		goto free_resources;
807	}
808	unmap->to_cnt = 1;
809
810	dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
811				offset_in_page(dest), PAGE_SIZE,
812				DMA_FROM_DEVICE);
813	unmap->addr[1] = dest_dma;
814
815	ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
816	if (ret) {
817		err = -ENOMEM;
818		goto free_resources;
819	}
820	unmap->from_cnt = 1;
821	unmap->len = PAGE_SIZE;
822
823	tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
824				    PAGE_SIZE, 0);
825	if (!tx) {
826		dev_err(dma_chan->device->dev,
827			"Self-test cannot prepare operation, disabling\n");
828		err = -ENODEV;
829		goto free_resources;
830	}
831
832	cookie = mv_xor_tx_submit(tx);
833	if (dma_submit_error(cookie)) {
834		dev_err(dma_chan->device->dev,
835			"Self-test submit error, disabling\n");
836		err = -ENODEV;
837		goto free_resources;
838	}
839
840	mv_xor_issue_pending(dma_chan);
841	async_tx_ack(tx);
842	msleep(1);
843
844	if (mv_xor_status(dma_chan, cookie, NULL) !=
845	    DMA_COMPLETE) {
846		dev_err(dma_chan->device->dev,
847			"Self-test copy timed out, disabling\n");
848		err = -ENODEV;
849		goto free_resources;
850	}
851
852	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
853				PAGE_SIZE, DMA_FROM_DEVICE);
854	if (memcmp(src, dest, PAGE_SIZE)) {
855		dev_err(dma_chan->device->dev,
856			"Self-test copy failed compare, disabling\n");
857		err = -ENODEV;
858		goto free_resources;
859	}
860
861free_resources:
862	dmaengine_unmap_put(unmap);
863	mv_xor_free_chan_resources(dma_chan);
864out:
865	kfree(src);
866	kfree(dest);
867	return err;
868}
869
870#define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
871static int
872mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
873{
874	int i, src_idx, ret;
875	struct page *dest;
876	struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
877	dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
878	dma_addr_t dest_dma;
879	struct dma_async_tx_descriptor *tx;
880	struct dmaengine_unmap_data *unmap;
881	struct dma_chan *dma_chan;
882	dma_cookie_t cookie;
883	u8 cmp_byte = 0;
884	u32 cmp_word;
885	int err = 0;
886	int src_count = MV_XOR_NUM_SRC_TEST;
887
888	for (src_idx = 0; src_idx < src_count; src_idx++) {
889		xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
890		if (!xor_srcs[src_idx]) {
891			while (src_idx--)
892				__free_page(xor_srcs[src_idx]);
893			return -ENOMEM;
894		}
895	}
896
897	dest = alloc_page(GFP_KERNEL);
898	if (!dest) {
899		while (src_idx--)
900			__free_page(xor_srcs[src_idx]);
901		return -ENOMEM;
902	}
903
904	/* Fill in src buffers */
905	for (src_idx = 0; src_idx < src_count; src_idx++) {
906		u8 *ptr = page_address(xor_srcs[src_idx]);
907		for (i = 0; i < PAGE_SIZE; i++)
908			ptr[i] = (1 << src_idx);
909	}
910
911	for (src_idx = 0; src_idx < src_count; src_idx++)
912		cmp_byte ^= (u8) (1 << src_idx);
913
914	cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
915		(cmp_byte << 8) | cmp_byte;
916
917	memset(page_address(dest), 0, PAGE_SIZE);
918
919	dma_chan = &mv_chan->dmachan;
920	if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
921		err = -ENODEV;
922		goto out;
923	}
924
925	unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
926					 GFP_KERNEL);
927	if (!unmap) {
928		err = -ENOMEM;
929		goto free_resources;
930	}
931
932	/* test xor */
933	for (i = 0; i < src_count; i++) {
934		unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
935					      0, PAGE_SIZE, DMA_TO_DEVICE);
936		dma_srcs[i] = unmap->addr[i];
937		ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
938		if (ret) {
939			err = -ENOMEM;
940			goto free_resources;
941		}
942		unmap->to_cnt++;
943	}
944
945	unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
946				      DMA_FROM_DEVICE);
947	dest_dma = unmap->addr[src_count];
948	ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
949	if (ret) {
950		err = -ENOMEM;
951		goto free_resources;
952	}
953	unmap->from_cnt = 1;
954	unmap->len = PAGE_SIZE;
955
956	tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
957				 src_count, PAGE_SIZE, 0);
958	if (!tx) {
959		dev_err(dma_chan->device->dev,
960			"Self-test cannot prepare operation, disabling\n");
961		err = -ENODEV;
962		goto free_resources;
963	}
964
965	cookie = mv_xor_tx_submit(tx);
966	if (dma_submit_error(cookie)) {
967		dev_err(dma_chan->device->dev,
968			"Self-test submit error, disabling\n");
969		err = -ENODEV;
970		goto free_resources;
971	}
972
973	mv_xor_issue_pending(dma_chan);
974	async_tx_ack(tx);
975	msleep(8);
976
977	if (mv_xor_status(dma_chan, cookie, NULL) !=
978	    DMA_COMPLETE) {
979		dev_err(dma_chan->device->dev,
980			"Self-test xor timed out, disabling\n");
981		err = -ENODEV;
982		goto free_resources;
983	}
984
985	dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
986				PAGE_SIZE, DMA_FROM_DEVICE);
987	for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
988		u32 *ptr = page_address(dest);
989		if (ptr[i] != cmp_word) {
990			dev_err(dma_chan->device->dev,
991				"Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
992				i, ptr[i], cmp_word);
993			err = -ENODEV;
994			goto free_resources;
995		}
996	}
997
998free_resources:
999	dmaengine_unmap_put(unmap);
1000	mv_xor_free_chan_resources(dma_chan);
1001out:
1002	src_idx = src_count;
1003	while (src_idx--)
1004		__free_page(xor_srcs[src_idx]);
1005	__free_page(dest);
1006	return err;
1007}
1008
1009static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
1010{
1011	struct dma_chan *chan, *_chan;
1012	struct device *dev = mv_chan->dmadev.dev;
1013
1014	dma_async_device_unregister(&mv_chan->dmadev);
1015
1016	dma_free_coherent(dev, MV_XOR_POOL_SIZE,
1017			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1018	dma_unmap_single(dev, mv_chan->dummy_src_addr,
1019			 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1020	dma_unmap_single(dev, mv_chan->dummy_dst_addr,
1021			 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1022
1023	list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
1024				 device_node) {
1025		list_del(&chan->device_node);
1026	}
1027
1028	free_irq(mv_chan->irq, mv_chan);
1029
1030	return 0;
1031}
1032
1033static struct mv_xor_chan *
1034mv_xor_channel_add(struct mv_xor_device *xordev,
1035		   struct platform_device *pdev,
1036		   int idx, dma_cap_mask_t cap_mask, int irq)
1037{
1038	int ret = 0;
1039	struct mv_xor_chan *mv_chan;
1040	struct dma_device *dma_dev;
1041
1042	mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
1043	if (!mv_chan)
1044		return ERR_PTR(-ENOMEM);
1045
1046	mv_chan->idx = idx;
1047	mv_chan->irq = irq;
1048	if (xordev->xor_type == XOR_ORION)
1049		mv_chan->op_in_desc = XOR_MODE_IN_REG;
1050	else
1051		mv_chan->op_in_desc = XOR_MODE_IN_DESC;
1052
1053	dma_dev = &mv_chan->dmadev;
1054	dma_dev->dev = &pdev->dev;
1055	mv_chan->xordev = xordev;
1056
1057	/*
1058	 * These source and destination dummy buffers are used to implement
1059	 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
1060	 * Hence, we only need to map the buffers at initialization-time.
1061	 */
1062	mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
1063		mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1064	mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
1065		mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1066
1067	/* allocate coherent memory for hardware descriptors
1068	 * note: writecombine gives slightly better performance, but
1069	 * requires that we explicitly flush the writes
1070	 */
1071	mv_chan->dma_desc_pool_virt =
1072	  dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
1073		       GFP_KERNEL);
1074	if (!mv_chan->dma_desc_pool_virt)
1075		return ERR_PTR(-ENOMEM);
1076
1077	/* discover transaction capabilites from the platform data */
1078	dma_dev->cap_mask = cap_mask;
1079
1080	INIT_LIST_HEAD(&dma_dev->channels);
1081
1082	/* set base routines */
1083	dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1084	dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1085	dma_dev->device_tx_status = mv_xor_status;
1086	dma_dev->device_issue_pending = mv_xor_issue_pending;
1087
1088	/* set prep routines based on capability */
1089	if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1090		dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1091	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1092		dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1093	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1094		dma_dev->max_xor = 8;
1095		dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1096	}
1097
1098	mv_chan->mmr_base = xordev->xor_base;
1099	mv_chan->mmr_high_base = xordev->xor_high_base;
1100	tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet);
1101
1102	/* clear errors before enabling interrupts */
1103	mv_chan_clear_err_status(mv_chan);
1104
1105	ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1106			  0, dev_name(&pdev->dev), mv_chan);
1107	if (ret)
1108		goto err_free_dma;
1109
1110	mv_chan_unmask_interrupts(mv_chan);
1111
1112	if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1113		mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1114	else
1115		mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1116
1117	spin_lock_init(&mv_chan->lock);
1118	INIT_LIST_HEAD(&mv_chan->chain);
1119	INIT_LIST_HEAD(&mv_chan->completed_slots);
1120	INIT_LIST_HEAD(&mv_chan->free_slots);
1121	INIT_LIST_HEAD(&mv_chan->allocated_slots);
1122	mv_chan->dmachan.device = dma_dev;
1123	dma_cookie_init(&mv_chan->dmachan);
1124
1125	list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1126
1127	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1128		ret = mv_chan_memcpy_self_test(mv_chan);
1129		dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1130		if (ret)
1131			goto err_free_irq;
1132	}
1133
1134	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1135		ret = mv_chan_xor_self_test(mv_chan);
1136		dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1137		if (ret)
1138			goto err_free_irq;
1139	}
1140
1141	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1142		 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1143		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1144		 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1145		 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1146
1147	ret = dma_async_device_register(dma_dev);
1148	if (ret)
1149		goto err_free_irq;
1150
1151	return mv_chan;
1152
1153err_free_irq:
1154	free_irq(mv_chan->irq, mv_chan);
1155err_free_dma:
1156	dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1157			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1158	return ERR_PTR(ret);
1159}
1160
1161static void
1162mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1163			 const struct mbus_dram_target_info *dram)
1164{
1165	void __iomem *base = xordev->xor_high_base;
1166	u32 win_enable = 0;
1167	int i;
1168
1169	for (i = 0; i < 8; i++) {
1170		writel(0, base + WINDOW_BASE(i));
1171		writel(0, base + WINDOW_SIZE(i));
1172		if (i < 4)
1173			writel(0, base + WINDOW_REMAP_HIGH(i));
1174	}
1175
1176	for (i = 0; i < dram->num_cs; i++) {
1177		const struct mbus_dram_window *cs = dram->cs + i;
1178
1179		writel((cs->base & 0xffff0000) |
1180		       (cs->mbus_attr << 8) |
1181		       dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1182		writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1183
1184		/* Fill the caching variables for later use */
1185		xordev->win_start[i] = cs->base;
1186		xordev->win_end[i] = cs->base + cs->size - 1;
1187
1188		win_enable |= (1 << i);
1189		win_enable |= 3 << (16 + (2 * i));
1190	}
1191
1192	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1193	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1194	writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1195	writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1196}
1197
1198static void
1199mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1200{
1201	void __iomem *base = xordev->xor_high_base;
1202	u32 win_enable = 0;
1203	int i;
1204
1205	for (i = 0; i < 8; i++) {
1206		writel(0, base + WINDOW_BASE(i));
1207		writel(0, base + WINDOW_SIZE(i));
1208		if (i < 4)
1209			writel(0, base + WINDOW_REMAP_HIGH(i));
1210	}
1211	/*
1212	 * For Armada3700 open default 4GB Mbus window. The dram
1213	 * related configuration are done at AXIS level.
1214	 */
1215	writel(0xffff0000, base + WINDOW_SIZE(0));
1216	win_enable |= 1;
1217	win_enable |= 3 << 16;
1218
1219	writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1220	writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1221	writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1222	writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1223}
1224
1225/*
1226 * Since this XOR driver is basically used only for RAID5, we don't
1227 * need to care about synchronizing ->suspend with DMA activity,
1228 * because the DMA engine will naturally be quiet due to the block
1229 * devices being suspended.
1230 */
1231static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1232{
1233	struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1234	int i;
1235
1236	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1237		struct mv_xor_chan *mv_chan = xordev->channels[i];
1238
1239		if (!mv_chan)
1240			continue;
1241
1242		mv_chan->saved_config_reg =
1243			readl_relaxed(XOR_CONFIG(mv_chan));
1244		mv_chan->saved_int_mask_reg =
1245			readl_relaxed(XOR_INTR_MASK(mv_chan));
1246	}
1247
1248	return 0;
1249}
1250
1251static int mv_xor_resume(struct platform_device *dev)
1252{
1253	struct mv_xor_device *xordev = platform_get_drvdata(dev);
1254	const struct mbus_dram_target_info *dram;
1255	int i;
1256
1257	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1258		struct mv_xor_chan *mv_chan = xordev->channels[i];
1259
1260		if (!mv_chan)
1261			continue;
1262
1263		writel_relaxed(mv_chan->saved_config_reg,
1264			       XOR_CONFIG(mv_chan));
1265		writel_relaxed(mv_chan->saved_int_mask_reg,
1266			       XOR_INTR_MASK(mv_chan));
1267	}
1268
1269	if (xordev->xor_type == XOR_ARMADA_37XX) {
1270		mv_xor_conf_mbus_windows_a3700(xordev);
1271		return 0;
1272	}
1273
1274	dram = mv_mbus_dram_info();
1275	if (dram)
1276		mv_xor_conf_mbus_windows(xordev, dram);
1277
1278	return 0;
1279}
1280
1281static const struct of_device_id mv_xor_dt_ids[] = {
1282	{ .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1283	{ .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1284	{ .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1285	{},
1286};
1287
1288static unsigned int mv_xor_engine_count;
1289
1290static int mv_xor_probe(struct platform_device *pdev)
1291{
1292	const struct mbus_dram_target_info *dram;
1293	struct mv_xor_device *xordev;
1294	struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1295	struct resource *res;
1296	unsigned int max_engines, max_channels;
1297	int i, ret;
1298
1299	dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1300
1301	xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1302	if (!xordev)
1303		return -ENOMEM;
1304
1305	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1306	if (!res)
1307		return -ENODEV;
1308
1309	xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1310					resource_size(res));
1311	if (!xordev->xor_base)
1312		return -EBUSY;
1313
1314	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1315	if (!res)
1316		return -ENODEV;
1317
1318	xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1319					     resource_size(res));
1320	if (!xordev->xor_high_base)
1321		return -EBUSY;
1322
1323	platform_set_drvdata(pdev, xordev);
1324
1325
1326	/*
1327	 * We need to know which type of XOR device we use before
1328	 * setting up. In non-dt case it can only be the legacy one.
1329	 */
1330	xordev->xor_type = XOR_ORION;
1331	if (pdev->dev.of_node) {
1332		const struct of_device_id *of_id =
1333			of_match_device(mv_xor_dt_ids,
1334					&pdev->dev);
1335
1336		xordev->xor_type = (uintptr_t)of_id->data;
1337	}
1338
1339	/*
1340	 * (Re-)program MBUS remapping windows if we are asked to.
1341	 */
1342	if (xordev->xor_type == XOR_ARMADA_37XX) {
1343		mv_xor_conf_mbus_windows_a3700(xordev);
1344	} else {
1345		dram = mv_mbus_dram_info();
1346		if (dram)
1347			mv_xor_conf_mbus_windows(xordev, dram);
1348	}
1349
1350	/* Not all platforms can gate the clock, so it is not
1351	 * an error if the clock does not exists.
1352	 */
1353	xordev->clk = clk_get(&pdev->dev, NULL);
1354	if (!IS_ERR(xordev->clk))
1355		clk_prepare_enable(xordev->clk);
1356
1357	/*
1358	 * We don't want to have more than one channel per CPU in
1359	 * order for async_tx to perform well. So we limit the number
1360	 * of engines and channels so that we take into account this
1361	 * constraint. Note that we also want to use channels from
1362	 * separate engines when possible.  For dual-CPU Armada 3700
1363	 * SoC with single XOR engine allow using its both channels.
1364	 */
1365	max_engines = num_present_cpus();
1366	if (xordev->xor_type == XOR_ARMADA_37XX)
1367		max_channels =	num_present_cpus();
1368	else
1369		max_channels = min_t(unsigned int,
1370				     MV_XOR_MAX_CHANNELS,
1371				     DIV_ROUND_UP(num_present_cpus(), 2));
1372
1373	if (mv_xor_engine_count >= max_engines)
1374		return 0;
1375
1376	if (pdev->dev.of_node) {
1377		struct device_node *np;
1378		int i = 0;
1379
1380		for_each_child_of_node(pdev->dev.of_node, np) {
1381			struct mv_xor_chan *chan;
1382			dma_cap_mask_t cap_mask;
1383			int irq;
1384
1385			if (i >= max_channels)
1386				continue;
1387
1388			dma_cap_zero(cap_mask);
1389			dma_cap_set(DMA_MEMCPY, cap_mask);
1390			dma_cap_set(DMA_XOR, cap_mask);
1391			dma_cap_set(DMA_INTERRUPT, cap_mask);
1392
1393			irq = irq_of_parse_and_map(np, 0);
1394			if (!irq) {
1395				ret = -ENODEV;
1396				goto err_channel_add;
1397			}
1398
1399			chan = mv_xor_channel_add(xordev, pdev, i,
1400						  cap_mask, irq);
1401			if (IS_ERR(chan)) {
1402				ret = PTR_ERR(chan);
1403				irq_dispose_mapping(irq);
1404				goto err_channel_add;
1405			}
1406
1407			xordev->channels[i] = chan;
1408			i++;
1409		}
1410	} else if (pdata && pdata->channels) {
1411		for (i = 0; i < max_channels; i++) {
1412			struct mv_xor_channel_data *cd;
1413			struct mv_xor_chan *chan;
1414			int irq;
1415
1416			cd = &pdata->channels[i];
1417			irq = platform_get_irq(pdev, i);
1418			if (irq < 0) {
1419				ret = irq;
1420				goto err_channel_add;
1421			}
1422
1423			chan = mv_xor_channel_add(xordev, pdev, i,
1424						  cd->cap_mask, irq);
1425			if (IS_ERR(chan)) {
1426				ret = PTR_ERR(chan);
1427				goto err_channel_add;
1428			}
1429
1430			xordev->channels[i] = chan;
1431		}
1432	}
1433
1434	return 0;
1435
1436err_channel_add:
1437	for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1438		if (xordev->channels[i]) {
1439			mv_xor_channel_remove(xordev->channels[i]);
1440			if (pdev->dev.of_node)
1441				irq_dispose_mapping(xordev->channels[i]->irq);
1442		}
1443
1444	if (!IS_ERR(xordev->clk)) {
1445		clk_disable_unprepare(xordev->clk);
1446		clk_put(xordev->clk);
1447	}
1448
1449	return ret;
1450}
1451
1452static struct platform_driver mv_xor_driver = {
1453	.probe		= mv_xor_probe,
1454	.suspend        = mv_xor_suspend,
1455	.resume         = mv_xor_resume,
1456	.driver		= {
1457		.name	        = MV_XOR_NAME,
1458		.of_match_table = mv_xor_dt_ids,
1459	},
1460};
1461
1462builtin_platform_driver(mv_xor_driver);
1463
1464/*
1465MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1466MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1467MODULE_LICENSE("GPL");
1468*/
1469