dma/ppc4xx/adma.c

1897 	struct ppc440spe_adma_desc_slot *sw_desc;
1904 	sw_desc = tx_to_ppc440spe_adma_slot(tx);
1906 	group_start = sw_desc->group_head;
1915 		list_splice_init(&sw_desc->group_list, &chan->chain);
1922 		list_splice_init(&sw_desc->group_list,
1936 		sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1948 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1959 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1961 	if (sw_desc) {
1962 		group_start = sw_desc->group_head;
1965 		sw_desc->async_tx.flags = flags;
1969 	return sw_desc ? &sw_desc->async_tx : NULL;
1980 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1997 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1999 	if (sw_desc) {
2000 		group_start = sw_desc->group_head;
2005 		sw_desc->unmap_len = len;
2006 		sw_desc->async_tx.flags = flags;
2010 	return sw_desc ? &sw_desc->async_tx : NULL;
2022 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2040 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2042 	if (sw_desc) {
2043 		group_start = sw_desc->group_head;
2050 		sw_desc->unmap_len = len;
2051 		sw_desc->async_tx.flags = flags;
2055 	return sw_desc ? &sw_desc->async_tx : NULL;
2088 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2098 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2099 	if (sw_desc) {
2104 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105 		set_bits(op, &sw_desc->flags);
2106 		sw_desc->src_cnt = src_cnt;
2107 		sw_desc->dst_cnt = dst_cnt;
2111 		iter = list_first_entry(&sw_desc->group_list,
2156 		sw_desc->async_tx.flags = flags;
2161 	return sw_desc;
2174 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2184 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2185 	if (sw_desc) {
2190 		chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191 		set_bits(op, &sw_desc->flags);
2192 		sw_desc->src_cnt = src_cnt;
2193 		sw_desc->dst_cnt = 1;
2195 		iter = list_first_entry(&sw_desc->group_list,
2265 		sw_desc->async_tx.flags = flags;
2270 	return sw_desc;
2279 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2379 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2380 	if (sw_desc) {
2381 		ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2387 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2389 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2400 			ppc440spe_adma_pq_set_src_mult(sw_desc,
2405 		sw_desc->async_tx.flags = flags;
2406 		list_for_each_entry(iter, &sw_desc->group_list,
2415 	return sw_desc;
2424 	struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2442 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2443 	if (sw_desc) {
2445 		sw_desc->async_tx.flags = flags;
2446 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2458 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2464 						 &sw_desc->group_list))) {
2478 		sw_desc->dst_cnt = dst_cnt;
2480 			set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2482 			set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2485 		ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2491 			ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2495 			ppc440spe_adma_pq_set_src_mult(sw_desc,
2501 	return sw_desc;
2513 	struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2531 		sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2533 		return sw_desc ? &sw_desc->async_tx : NULL;
2537 		sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2539 		return sw_desc ? &sw_desc->async_tx : NULL;
2564 		sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2570 		sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2576 	return sw_desc ? &sw_desc->async_tx : NULL;
2589 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2620 	sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2622 	if (sw_desc) {
2623 		ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2626 		sw_desc->async_tx.flags = flags;
2627 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2637 			iter = sw_desc->group_head;
2660 			iter = list_first_entry(&sw_desc->group_list,
2690 		ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2694 		list_for_each_entry_reverse(iter, &sw_desc->group_list,
2733 		list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2755 	return sw_desc ? &sw_desc->async_tx : NULL;
2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2788 	BUG_ON(index >= sw_desc->dst_cnt);
2790 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2798 		ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2802 		sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2803 		ppc440spe_desc_set_dest_addr(sw_desc,
2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2839 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2860 		if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2862 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2864 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2867 			iter = ppc440spe_get_group_entry(sw_desc, index);
2871 					&sw_desc->group_list, chain_node)
2877 					&sw_desc->group_list, chain_node) {
2891 						&sw_desc->flags)) {
2893 							sw_desc, index++);
2899 						&sw_desc->flags)) {
2901 							sw_desc, index++);
2914 			ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2918 			qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2924 			iter = ppc440spe_get_group_entry(sw_desc, index++);
2930 				iter = ppc440spe_get_group_entry(sw_desc,
2936 			if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2940 				iter = ppc440spe_get_group_entry(sw_desc,
2945 					    &sw_desc->group_list,
2955 					    &sw_desc->group_list,
2978 		ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2983 		qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2988 		iter = ppc440spe_get_group_entry(sw_desc, 0);
2989 		for (i = 0; i < sw_desc->descs_per_op; i++) {
3000 			iter = ppc440spe_get_group_entry(sw_desc,
3001 				sw_desc->descs_per_op);
3002 			for (i = 0; i < sw_desc->descs_per_op; i++) {
3020 		struct ppc440spe_adma_desc_slot *sw_desc,
3028 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3035 	list_for_each_entry_reverse(end, &sw_desc->group_list,
3042 	iter = ppc440spe_get_group_entry(sw_desc, idx);
3046 		list_for_each_entry_from(iter, &sw_desc->group_list,
3058 		list_for_each_entry_from(iter, &sw_desc->group_list,
3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3103 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3110 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3113 				&sw_desc->flags) ?  2 : 3;
3121 						&sw_desc->flags))
3125 				    &sw_desc->flags))
3129 				    &sw_desc->flags))
3133 				    &sw_desc->flags))
3139 				iter = ppc440spe_get_group_entry(sw_desc, 0);
3151 				iter = ppc440spe_get_group_entry(sw_desc,
3152 				    index - iskip + sw_desc->dst_cnt);
3160 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3162 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3166 			iter = ppc440spe_get_group_entry(sw_desc,
3174 			    test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175 			    sw_desc->dst_cnt == 2) {
3179 				iter = ppc440spe_get_group_entry(sw_desc, 1);
3188 		iter = sw_desc->group_head;
3194 			iter = ppc440spe_get_group_entry(sw_desc,
3195 				sw_desc->descs_per_op);
3206 		struct ppc440spe_adma_desc_slot *sw_desc,
3211 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212 	sw_desc = sw_desc->group_head;
3214 	if (likely(sw_desc))
3215 		ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3447 		struct ppc440spe_adma_desc_slot *sw_desc,
3454 	chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3459 		if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3461 					&sw_desc->flags) ? 2 : 3;
3465 				iter = ppc440spe_get_group_entry(sw_desc,
3466 					sw_desc->dst_cnt - 1);
3467 				if (sw_desc->dst_cnt == 2)
3469 							sw_desc, 0);
3475 				iter = ppc440spe_get_group_entry(sw_desc,
3477 							sw_desc->dst_cnt);
3489 			if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3491 			if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3494 			iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3515 		iter = sw_desc->group_head;
3516 		if (sw_desc->dst_cnt == 2) {
3521 			iter = ppc440spe_get_group_entry(sw_desc,
3522 			       sw_desc->descs_per_op);
3655 	struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3664 	sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3665 	if (sw_desc) {
3666 		group_start = sw_desc->group_head;
3667 		list_splice_init(&sw_desc->group_list, &chan->chain);
3668 		async_tx_ack(&sw_desc->async_tx);
3671 		cookie = dma_cookie_assign(&sw_desc->async_tx);
3682 		ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3701 	struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3715 	sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3716 	if (sw_desc) {
3718 		ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3719 		list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3736 	ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3737 	ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3740 	ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3742 	async_tx_ack(&sw_desc->async_tx);
3743 	sw_desc->async_tx.callback = ppc440spe_test_callback;
3744 	sw_desc->async_tx.callback_param = NULL;
3748 	ppc440spe_adma_tx_submit(&sw_desc->async_tx);