1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (C) 2016 Gvozden Ne��kovi��. All rights reserved. 23 */ 24#include <sys/isa_defs.h> 25 26#if defined(__x86_64) && defined(HAVE_AVX2) 27 28#include <sys/types.h> 29#include <sys/simd.h> 30 31#ifdef __linux__ 32#define __asm __asm__ __volatile__ 33#endif 34 35#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N 36#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) 37 38#define VR0_(REG, ...) "ymm"#REG 39#define VR1_(_1, REG, ...) "ymm"#REG 40#define VR2_(_1, _2, REG, ...) "ymm"#REG 41#define VR3_(_1, _2, _3, REG, ...) "ymm"#REG 42#define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG 43#define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG 44#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG 45#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG 46 47#define VR0(r...) VR0_(r) 48#define VR1(r...) VR1_(r) 49#define VR2(r...) VR2_(r, 1) 50#define VR3(r...) VR3_(r, 1, 2) 51#define VR4(r...) VR4_(r, 1, 2) 52#define VR5(r...) VR5_(r, 1, 2, 3) 53#define VR6(r...) VR6_(r, 1, 2, 3, 4) 54#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) 55 56#define R_01(REG1, REG2, ...) REG1, REG2 57#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 58#define R_23(REG...) _R_23(REG, 1, 2, 3) 59 60#define ZFS_ASM_BUG() ASSERT(0) 61 62extern const uint8_t gf_clmul_mod_lt[4*256][16]; 63 64#define ELEM_SIZE 32 65 66typedef struct v { 67 uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); 68} v_t; 69 70 71#define XOR_ACC(src, r...) \ 72{ \ 73 switch (REG_CNT(r)) { \ 74 case 4: \ 75 __asm( \ 76 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 77 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 78 "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ 79 "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ 80 : : [SRC] "r" (src)); \ 81 break; \ 82 case 2: \ 83 __asm( \ 84 "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ 85 "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ 86 : : [SRC] "r" (src)); \ 87 break; \ 88 default: \ 89 ZFS_ASM_BUG(); \ 90 } \ 91} 92 93#define XOR(r...) \ 94{ \ 95 switch (REG_CNT(r)) { \ 96 case 8: \ 97 __asm( \ 98 "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ 99 "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ 100 "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ 101 "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ 102 break; \ 103 case 4: \ 104 __asm( \ 105 "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ 106 "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ 107 break; \ 108 default: \ 109 ZFS_ASM_BUG(); \ 110 } \ 111} 112 113#define ZERO(r...) XOR(r, r) 114 115#define COPY(r...) \ 116{ \ 117 switch (REG_CNT(r)) { \ 118 case 8: \ 119 __asm( \ 120 "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \ 121 "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \ 122 "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \ 123 "vmovdqa %" VR3(r) ", %" VR7(r)); \ 124 break; \ 125 case 4: \ 126 __asm( \ 127 "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \ 128 "vmovdqa %" VR1(r) ", %" VR3(r)); \ 129 break; \ 130 default: \ 131 ZFS_ASM_BUG(); \ 132 } \ 133} 134 135#define LOAD(src, r...) \ 136{ \ 137 switch (REG_CNT(r)) { \ 138 case 4: \ 139 __asm( \ 140 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 141 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 142 "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \ 143 "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \ 144 : : [SRC] "r" (src)); \ 145 break; \ 146 case 2: \ 147 __asm( \ 148 "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ 149 "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ 150 : : [SRC] "r" (src)); \ 151 break; \ 152 default: \ 153 ZFS_ASM_BUG(); \ 154 } \ 155} 156 157#define STORE(dst, r...) \ 158{ \ 159 switch (REG_CNT(r)) { \ 160 case 4: \ 161 __asm( \ 162 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 163 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 164 "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \ 165 "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \ 166 : : [DST] "r" (dst)); \ 167 break; \ 168 case 2: \ 169 __asm( \ 170 "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ 171 "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ 172 : : [DST] "r" (dst)); \ 173 break; \ 174 default: \ 175 ZFS_ASM_BUG(); \ 176 } \ 177} 178 179#define FLUSH() \ 180{ \ 181 __asm("vzeroupper"); \ 182} 183 184#define MUL2_SETUP() \ 185{ \ 186 __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ 187 __asm("vpbroadcastq %xmm14, %ymm14"); \ 188 __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \ 189} 190 191#define _MUL2(r...) \ 192{ \ 193 switch (REG_CNT(r)) { \ 194 case 2: \ 195 __asm( \ 196 "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \ 197 "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \ 198 "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ 199 "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ 200 "vpand %ymm14, %ymm12, %ymm12\n" \ 201 "vpand %ymm14, %ymm13, %ymm13\n" \ 202 "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \ 203 "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \ 204 break; \ 205 default: \ 206 ZFS_ASM_BUG(); \ 207 } \ 208} 209 210#define MUL2(r...) \ 211{ \ 212 switch (REG_CNT(r)) { \ 213 case 4: \ 214 _MUL2(R_01(r)); \ 215 _MUL2(R_23(r)); \ 216 break; \ 217 case 2: \ 218 _MUL2(r); \ 219 break; \ 220 default: \ 221 ZFS_ASM_BUG(); \ 222 } \ 223} 224 225#define MUL4(r...) \ 226{ \ 227 MUL2(r); \ 228 MUL2(r); \ 229} 230 231#define _0f "ymm15" 232#define _as "ymm14" 233#define _bs "ymm13" 234#define _ltmod "ymm12" 235#define _ltmul "ymm11" 236#define _ta "ymm10" 237#define _tb "ymm15" 238 239static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; 240 241#define _MULx2(c, r...) \ 242{ \ 243 switch (REG_CNT(r)) { \ 244 case 2: \ 245 __asm( \ 246 "vpbroadcastb (%[mask]), %%" _0f "\n" \ 247 /* upper bits */ \ 248 "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \ 249 "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \ 250 \ 251 "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ 252 "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ 253 "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ 254 "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ 255 "vpand %%" _0f ", %%" _as ", %%" _as "\n" \ 256 "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \ 257 \ 258 "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ 259 "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ 260 "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ 261 "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ 262 /* lower bits */ \ 263 "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \ 264 "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \ 265 \ 266 "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \ 267 "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \ 268 \ 269 "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ 270 "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ 271 "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ 272 "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ 273 \ 274 "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ 275 "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ 276 "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ 277 "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ 278 : : [mask] "r" (&_mul_mask), \ 279 [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ 280 break; \ 281 default: \ 282 ZFS_ASM_BUG(); \ 283 } \ 284} 285 286#define MUL(c, r...) \ 287{ \ 288 switch (REG_CNT(r)) { \ 289 case 4: \ 290 _MULx2(c, R_01(r)); \ 291 _MULx2(c, R_23(r)); \ 292 break; \ 293 case 2: \ 294 _MULx2(c, R_01(r)); \ 295 break; \ 296 default: \ 297 ZFS_ASM_BUG(); \ 298 } \ 299} 300 301#define raidz_math_begin() kfpu_begin() 302#define raidz_math_end() \ 303{ \ 304 FLUSH(); \ 305 kfpu_end(); \ 306} 307 308 309#define SYN_STRIDE 4 310 311#define ZERO_STRIDE 4 312#define ZERO_DEFINE() {} 313#define ZERO_D 0, 1, 2, 3 314 315#define COPY_STRIDE 4 316#define COPY_DEFINE() {} 317#define COPY_D 0, 1, 2, 3 318 319#define ADD_STRIDE 4 320#define ADD_DEFINE() {} 321#define ADD_D 0, 1, 2, 3 322 323#define MUL_STRIDE 4 324#define MUL_DEFINE() {} 325#define MUL_D 0, 1, 2, 3 326 327#define GEN_P_STRIDE 4 328#define GEN_P_DEFINE() {} 329#define GEN_P_P 0, 1, 2, 3 330 331#define GEN_PQ_STRIDE 4 332#define GEN_PQ_DEFINE() {} 333#define GEN_PQ_D 0, 1, 2, 3 334#define GEN_PQ_C 4, 5, 6, 7 335 336#define GEN_PQR_STRIDE 4 337#define GEN_PQR_DEFINE() {} 338#define GEN_PQR_D 0, 1, 2, 3 339#define GEN_PQR_C 4, 5, 6, 7 340 341#define SYN_Q_DEFINE() {} 342#define SYN_Q_D 0, 1, 2, 3 343#define SYN_Q_X 4, 5, 6, 7 344 345#define SYN_R_DEFINE() {} 346#define SYN_R_D 0, 1, 2, 3 347#define SYN_R_X 4, 5, 6, 7 348 349#define SYN_PQ_DEFINE() {} 350#define SYN_PQ_D 0, 1, 2, 3 351#define SYN_PQ_X 4, 5, 6, 7 352 353#define REC_PQ_STRIDE 2 354#define REC_PQ_DEFINE() {} 355#define REC_PQ_X 0, 1 356#define REC_PQ_Y 2, 3 357#define REC_PQ_T 4, 5 358 359#define SYN_PR_DEFINE() {} 360#define SYN_PR_D 0, 1, 2, 3 361#define SYN_PR_X 4, 5, 6, 7 362 363#define REC_PR_STRIDE 2 364#define REC_PR_DEFINE() {} 365#define REC_PR_X 0, 1 366#define REC_PR_Y 2, 3 367#define REC_PR_T 4, 5 368 369#define SYN_QR_DEFINE() {} 370#define SYN_QR_D 0, 1, 2, 3 371#define SYN_QR_X 4, 5, 6, 7 372 373#define REC_QR_STRIDE 2 374#define REC_QR_DEFINE() {} 375#define REC_QR_X 0, 1 376#define REC_QR_Y 2, 3 377#define REC_QR_T 4, 5 378 379#define SYN_PQR_DEFINE() {} 380#define SYN_PQR_D 0, 1, 2, 3 381#define SYN_PQR_X 4, 5, 6, 7 382 383#define REC_PQR_STRIDE 2 384#define REC_PQR_DEFINE() {} 385#define REC_PQR_X 0, 1 386#define REC_PQR_Y 2, 3 387#define REC_PQR_Z 4, 5 388#define REC_PQR_XS 6, 7 389#define REC_PQR_YS 8, 9 390 391 392#include <sys/vdev_raidz_impl.h> 393#include "vdev_raidz_math_impl.h" 394 395DEFINE_GEN_METHODS(avx2); 396DEFINE_REC_METHODS(avx2); 397 398static boolean_t 399raidz_will_avx2_work(void) 400{ 401 return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available()); 402} 403 404const raidz_impl_ops_t vdev_raidz_avx2_impl = { 405 .init = NULL, 406 .fini = NULL, 407 .gen = RAIDZ_GEN_METHODS(avx2), 408 .rec = RAIDZ_REC_METHODS(avx2), 409 .is_supported = &raidz_will_avx2_work, 410 .name = "avx2" 411}; 412 413#endif /* defined(__x86_64) && defined(HAVE_AVX2) */ 414