1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Optimized xor_block operation for RAID4/5
4 *
5 * Copyright IBM Corp. 2016
6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
7 */
8
9#include <linux/types.h>
10#include <linux/export.h>
11#include <linux/raid/xor.h>
12#include <asm/xor.h>
13
14static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
15		     const unsigned long * __restrict p2)
16{
17	asm volatile(
18		"	larl	1,2f\n"
19		"	aghi	%0,-1\n"
20		"	jm	3f\n"
21		"	srlg	0,%0,8\n"
22		"	ltgr	0,0\n"
23		"	jz	1f\n"
24		"0:	xc	0(256,%1),0(%2)\n"
25		"	la	%1,256(%1)\n"
26		"	la	%2,256(%2)\n"
27		"	brctg	0,0b\n"
28		"1:	ex	%0,0(1)\n"
29		"	j	3f\n"
30		"2:	xc	0(1,%1),0(%2)\n"
31		"3:\n"
32		: : "d" (bytes), "a" (p1), "a" (p2)
33		: "0", "1", "cc", "memory");
34}
35
36static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
37		     const unsigned long * __restrict p2,
38		     const unsigned long * __restrict p3)
39{
40	asm volatile(
41		"	larl	1,2f\n"
42		"	aghi	%0,-1\n"
43		"	jm	3f\n"
44		"	srlg	0,%0,8\n"
45		"	ltgr	0,0\n"
46		"	jz	1f\n"
47		"0:	xc	0(256,%1),0(%2)\n"
48		"	xc	0(256,%1),0(%3)\n"
49		"	la	%1,256(%1)\n"
50		"	la	%2,256(%2)\n"
51		"	la	%3,256(%3)\n"
52		"	brctg	0,0b\n"
53		"1:	ex	%0,0(1)\n"
54		"	ex	%0,6(1)\n"
55		"	j	3f\n"
56		"2:	xc	0(1,%1),0(%2)\n"
57		"	xc	0(1,%1),0(%3)\n"
58		"3:\n"
59		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
60		: : "0", "1", "cc", "memory");
61}
62
63static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
64		     const unsigned long * __restrict p2,
65		     const unsigned long * __restrict p3,
66		     const unsigned long * __restrict p4)
67{
68	asm volatile(
69		"	larl	1,2f\n"
70		"	aghi	%0,-1\n"
71		"	jm	3f\n"
72		"	srlg	0,%0,8\n"
73		"	ltgr	0,0\n"
74		"	jz	1f\n"
75		"0:	xc	0(256,%1),0(%2)\n"
76		"	xc	0(256,%1),0(%3)\n"
77		"	xc	0(256,%1),0(%4)\n"
78		"	la	%1,256(%1)\n"
79		"	la	%2,256(%2)\n"
80		"	la	%3,256(%3)\n"
81		"	la	%4,256(%4)\n"
82		"	brctg	0,0b\n"
83		"1:	ex	%0,0(1)\n"
84		"	ex	%0,6(1)\n"
85		"	ex	%0,12(1)\n"
86		"	j	3f\n"
87		"2:	xc	0(1,%1),0(%2)\n"
88		"	xc	0(1,%1),0(%3)\n"
89		"	xc	0(1,%1),0(%4)\n"
90		"3:\n"
91		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
92		: : "0", "1", "cc", "memory");
93}
94
95static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
96		     const unsigned long * __restrict p2,
97		     const unsigned long * __restrict p3,
98		     const unsigned long * __restrict p4,
99		     const unsigned long * __restrict p5)
100{
101	asm volatile(
102		"	larl	1,2f\n"
103		"	aghi	%0,-1\n"
104		"	jm	3f\n"
105		"	srlg	0,%0,8\n"
106		"	ltgr	0,0\n"
107		"	jz	1f\n"
108		"0:	xc	0(256,%1),0(%2)\n"
109		"	xc	0(256,%1),0(%3)\n"
110		"	xc	0(256,%1),0(%4)\n"
111		"	xc	0(256,%1),0(%5)\n"
112		"	la	%1,256(%1)\n"
113		"	la	%2,256(%2)\n"
114		"	la	%3,256(%3)\n"
115		"	la	%4,256(%4)\n"
116		"	la	%5,256(%5)\n"
117		"	brctg	0,0b\n"
118		"1:	ex	%0,0(1)\n"
119		"	ex	%0,6(1)\n"
120		"	ex	%0,12(1)\n"
121		"	ex	%0,18(1)\n"
122		"	j	3f\n"
123		"2:	xc	0(1,%1),0(%2)\n"
124		"	xc	0(1,%1),0(%3)\n"
125		"	xc	0(1,%1),0(%4)\n"
126		"	xc	0(1,%1),0(%5)\n"
127		"3:\n"
128		: "+d" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
129		  "+a" (p5)
130		: : "0", "1", "cc", "memory");
131}
132
133struct xor_block_template xor_block_xc = {
134	.name = "xc",
135	.do_2 = xor_xc_2,
136	.do_3 = xor_xc_3,
137	.do_4 = xor_xc_4,
138	.do_5 = xor_xc_5,
139};
140EXPORT_SYMBOL(xor_block_xc);
141