1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (C) 2016 Gvozden Ne��kovi��. All rights reserved.
23 */
24#include <sys/isa_defs.h>
25
26#if defined(__x86_64) && defined(HAVE_AVX2)
27
28#include <sys/types.h>
29#include <sys/simd.h>
30
31#ifdef __linux__
32#define	__asm __asm__ __volatile__
33#endif
34
35#define	_REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
36#define	REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
37
38#define	VR0_(REG, ...) "ymm"#REG
39#define	VR1_(_1, REG, ...) "ymm"#REG
40#define	VR2_(_1, _2, REG, ...) "ymm"#REG
41#define	VR3_(_1, _2, _3, REG, ...) "ymm"#REG
42#define	VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
43#define	VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
44#define	VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
45#define	VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
46
47#define	VR0(r...) VR0_(r)
48#define	VR1(r...) VR1_(r)
49#define	VR2(r...) VR2_(r, 1)
50#define	VR3(r...) VR3_(r, 1, 2)
51#define	VR4(r...) VR4_(r, 1, 2)
52#define	VR5(r...) VR5_(r, 1, 2, 3)
53#define	VR6(r...) VR6_(r, 1, 2, 3, 4)
54#define	VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
55
56#define	R_01(REG1, REG2, ...) REG1, REG2
57#define	_R_23(_0, _1, REG2, REG3, ...) REG2, REG3
58#define	R_23(REG...) _R_23(REG, 1, 2, 3)
59
60#define	ZFS_ASM_BUG()	ASSERT(0)
61
62extern const uint8_t gf_clmul_mod_lt[4*256][16];
63
64#define	ELEM_SIZE 32
65
66typedef struct v {
67	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
68} v_t;
69
70
71#define	XOR_ACC(src, r...)						\
72{									\
73	switch (REG_CNT(r)) {						\
74	case 4:								\
75		__asm(							\
76		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
77		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
78		    "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"	\
79		    "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"	\
80		    : : [SRC] "r" (src));				\
81		break;							\
82	case 2:								\
83		__asm(							\
84		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
85		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
86		    : : [SRC] "r" (src));				\
87		break;							\
88	default:							\
89		ZFS_ASM_BUG();						\
90	}								\
91}
92
93#define	XOR(r...)							\
94{									\
95	switch (REG_CNT(r)) {						\
96	case 8:								\
97		__asm(							\
98		    "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"	\
99		    "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"	\
100		    "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"	\
101		    "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r));		\
102		break;							\
103	case 4:								\
104		__asm(							\
105		    "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"	\
106		    "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r));		\
107		break;							\
108	default:							\
109		ZFS_ASM_BUG();						\
110	}								\
111}
112
113#define	ZERO(r...)	XOR(r, r)
114
115#define	COPY(r...) 							\
116{									\
117	switch (REG_CNT(r)) {						\
118	case 8:								\
119		__asm(							\
120		    "vmovdqa %" VR0(r) ", %" VR4(r) "\n"		\
121		    "vmovdqa %" VR1(r) ", %" VR5(r) "\n"		\
122		    "vmovdqa %" VR2(r) ", %" VR6(r) "\n"		\
123		    "vmovdqa %" VR3(r) ", %" VR7(r));			\
124		break;							\
125	case 4:								\
126		__asm(							\
127		    "vmovdqa %" VR0(r) ", %" VR2(r) "\n"		\
128		    "vmovdqa %" VR1(r) ", %" VR3(r));			\
129		break;							\
130	default:							\
131		ZFS_ASM_BUG();						\
132	}								\
133}
134
135#define	LOAD(src, r...) 						\
136{									\
137	switch (REG_CNT(r)) {						\
138	case 4:								\
139		__asm(							\
140		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
141		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
142		    "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n"		\
143		    "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n"		\
144		    : : [SRC] "r" (src));				\
145		break;							\
146	case 2:								\
147		__asm(							\
148		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
149		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
150		    : : [SRC] "r" (src));				\
151		break;							\
152	default:							\
153		ZFS_ASM_BUG();						\
154	}								\
155}
156
157#define	STORE(dst, r...)   						\
158{									\
159	switch (REG_CNT(r)) {						\
160	case 4:								\
161		__asm(							\
162		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
163		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
164		    "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n"		\
165		    "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n"		\
166		    : : [DST] "r" (dst));				\
167		break;							\
168	case 2:								\
169		__asm(							\
170		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
171		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
172		    : : [DST] "r" (dst));				\
173		break;							\
174	default:							\
175		ZFS_ASM_BUG();						\
176	}								\
177}
178
179#define	FLUSH()								\
180{									\
181	__asm("vzeroupper");						\
182}
183
184#define	MUL2_SETUP() 							\
185{   									\
186	__asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));	\
187	__asm("vpbroadcastq %xmm14, %ymm14");				\
188	__asm("vpxor        %ymm15, %ymm15 ,%ymm15");			\
189}
190
191#define	_MUL2(r...) 							\
192{									\
193	switch	(REG_CNT(r)) {						\
194	case 2:								\
195		__asm(							\
196		    "vpcmpgtb %" VR0(r)", %ymm15,     %ymm12\n"		\
197		    "vpcmpgtb %" VR1(r)", %ymm15,     %ymm13\n"		\
198		    "vpaddb   %" VR0(r)", %" VR0(r)", %" VR0(r) "\n"	\
199		    "vpaddb   %" VR1(r)", %" VR1(r)", %" VR1(r) "\n"	\
200		    "vpand    %ymm14,     %ymm12,     %ymm12\n"		\
201		    "vpand    %ymm14,     %ymm13,     %ymm13\n"		\
202		    "vpxor    %ymm12,     %" VR0(r)", %" VR0(r) "\n"	\
203		    "vpxor    %ymm13,     %" VR1(r)", %" VR1(r));	\
204		break;							\
205	default:							\
206		ZFS_ASM_BUG();						\
207	}								\
208}
209
210#define	MUL2(r...)							\
211{									\
212	switch (REG_CNT(r)) {						\
213	case 4:								\
214	    _MUL2(R_01(r));						\
215	    _MUL2(R_23(r));						\
216	    break;							\
217	case 2:								\
218	    _MUL2(r);							\
219	    break;							\
220	default:							\
221		ZFS_ASM_BUG();						\
222	}								\
223}
224
225#define	MUL4(r...)							\
226{									\
227	MUL2(r);							\
228	MUL2(r);							\
229}
230
231#define	_0f		"ymm15"
232#define	_as		"ymm14"
233#define	_bs		"ymm13"
234#define	_ltmod		"ymm12"
235#define	_ltmul		"ymm11"
236#define	_ta		"ymm10"
237#define	_tb		"ymm15"
238
239static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
240
241#define	_MULx2(c, r...)							\
242{									\
243	switch (REG_CNT(r)) {						\
244	case 2:								\
245		__asm(							\
246		    "vpbroadcastb (%[mask]), %%" _0f "\n"		\
247		    /* upper bits */					\
248		    "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n"	\
249		    "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n"	\
250									\
251		    "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"		\
252		    "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"		\
253		    "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"	\
254		    "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"	\
255		    "vpand %%" _0f ", %%" _as ", %%" _as "\n"		\
256		    "vpand %%" _0f ", %%" _bs ", %%" _bs "\n"		\
257									\
258		    "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"	\
259		    "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"	\
260		    "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"	\
261		    "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"	\
262		    /* lower bits */					\
263		    "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n"	\
264		    "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n"	\
265									\
266		    "vpxor %%" _ta ", %%" _as ", %%" _as "\n"		\
267		    "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n"		\
268									\
269		    "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"	\
270		    "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"	\
271		    "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
272		    "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
273									\
274		    "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"	\
275		    "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"	\
276		    "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"	\
277		    "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"	\
278		    : : [mask] "r" (&_mul_mask),			\
279		    [lt] "r" (gf_clmul_mod_lt[4*(c)]));			\
280		break;							\
281	default:							\
282		ZFS_ASM_BUG();						\
283	}								\
284}
285
286#define	MUL(c, r...)							\
287{									\
288	switch (REG_CNT(r)) {						\
289	case 4:								\
290		_MULx2(c, R_01(r));					\
291		_MULx2(c, R_23(r));					\
292		break;							\
293	case 2:								\
294		_MULx2(c, R_01(r));					\
295		break;							\
296	default:							\
297		ZFS_ASM_BUG();						\
298	}								\
299}
300
301#define	raidz_math_begin()	kfpu_begin()
302#define	raidz_math_end()						\
303{									\
304	FLUSH();							\
305	kfpu_end();							\
306}
307
308
309#define	SYN_STRIDE		4
310
311#define	ZERO_STRIDE		4
312#define	ZERO_DEFINE()		{}
313#define	ZERO_D			0, 1, 2, 3
314
315#define	COPY_STRIDE		4
316#define	COPY_DEFINE()		{}
317#define	COPY_D			0, 1, 2, 3
318
319#define	ADD_STRIDE		4
320#define	ADD_DEFINE()		{}
321#define	ADD_D 			0, 1, 2, 3
322
323#define	MUL_STRIDE		4
324#define	MUL_DEFINE() 		{}
325#define	MUL_D			0, 1, 2, 3
326
327#define	GEN_P_STRIDE		4
328#define	GEN_P_DEFINE()		{}
329#define	GEN_P_P			0, 1, 2, 3
330
331#define	GEN_PQ_STRIDE		4
332#define	GEN_PQ_DEFINE() 	{}
333#define	GEN_PQ_D		0, 1, 2, 3
334#define	GEN_PQ_C		4, 5, 6, 7
335
336#define	GEN_PQR_STRIDE		4
337#define	GEN_PQR_DEFINE() 	{}
338#define	GEN_PQR_D		0, 1, 2, 3
339#define	GEN_PQR_C		4, 5, 6, 7
340
341#define	SYN_Q_DEFINE()		{}
342#define	SYN_Q_D			0, 1, 2, 3
343#define	SYN_Q_X			4, 5, 6, 7
344
345#define	SYN_R_DEFINE()		{}
346#define	SYN_R_D			0, 1, 2, 3
347#define	SYN_R_X			4, 5, 6, 7
348
349#define	SYN_PQ_DEFINE() 	{}
350#define	SYN_PQ_D		0, 1, 2, 3
351#define	SYN_PQ_X		4, 5, 6, 7
352
353#define	REC_PQ_STRIDE		2
354#define	REC_PQ_DEFINE() 	{}
355#define	REC_PQ_X		0, 1
356#define	REC_PQ_Y		2, 3
357#define	REC_PQ_T		4, 5
358
359#define	SYN_PR_DEFINE() 	{}
360#define	SYN_PR_D		0, 1, 2, 3
361#define	SYN_PR_X		4, 5, 6, 7
362
363#define	REC_PR_STRIDE		2
364#define	REC_PR_DEFINE() 	{}
365#define	REC_PR_X		0, 1
366#define	REC_PR_Y		2, 3
367#define	REC_PR_T		4, 5
368
369#define	SYN_QR_DEFINE() 	{}
370#define	SYN_QR_D		0, 1, 2, 3
371#define	SYN_QR_X		4, 5, 6, 7
372
373#define	REC_QR_STRIDE		2
374#define	REC_QR_DEFINE() 	{}
375#define	REC_QR_X		0, 1
376#define	REC_QR_Y		2, 3
377#define	REC_QR_T		4, 5
378
379#define	SYN_PQR_DEFINE() 	{}
380#define	SYN_PQR_D		0, 1, 2, 3
381#define	SYN_PQR_X		4, 5, 6, 7
382
383#define	REC_PQR_STRIDE		2
384#define	REC_PQR_DEFINE() 	{}
385#define	REC_PQR_X		0, 1
386#define	REC_PQR_Y		2, 3
387#define	REC_PQR_Z		4, 5
388#define	REC_PQR_XS		6, 7
389#define	REC_PQR_YS		8, 9
390
391
392#include <sys/vdev_raidz_impl.h>
393#include "vdev_raidz_math_impl.h"
394
395DEFINE_GEN_METHODS(avx2);
396DEFINE_REC_METHODS(avx2);
397
398static boolean_t
399raidz_will_avx2_work(void)
400{
401	return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
402}
403
404const raidz_impl_ops_t vdev_raidz_avx2_impl = {
405	.init = NULL,
406	.fini = NULL,
407	.gen = RAIDZ_GEN_METHODS(avx2),
408	.rec = RAIDZ_REC_METHODS(avx2),
409	.is_supported = &raidz_will_avx2_work,
410	.name = "avx2"
411};
412
413#endif /* defined(__x86_64) && defined(HAVE_AVX2) */
414