1;; Machine description for AArch64 SVE2. 2;; Copyright (C) 2019-2020 Free Software Foundation, Inc. 3;; Contributed by ARM Ltd. 4;; 5;; This file is part of GCC. 6;; 7;; GCC is free software; you can redistribute it and/or modify it 8;; under the terms of the GNU General Public License as published by 9;; the Free Software Foundation; either version 3, or (at your option) 10;; any later version. 11;; 12;; GCC is distributed in the hope that it will be useful, but 13;; WITHOUT ANY WARRANTY; without even the implied warranty of 14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15;; General Public License for more details. 16;; 17;; You should have received a copy of the GNU General Public License 18;; along with GCC; see the file COPYING3. If not see 19;; <http://www.gnu.org/licenses/>. 20 21;; The file is organised into the following sections (search for the full 22;; line): 23;; 24;; == Moves 25;; ---- Non-temporal gather loads 26;; ---- Non-temporal scatter stores 27;; 28;; == Uniform binary arithmnetic 29;; ---- [INT] Multiplication 30;; ---- [INT] Scaled high-part multiplication 31;; ---- [INT] General binary arithmetic that maps to unspecs 32;; ---- [INT] Saturating binary arithmetic 33;; ---- [INT] Saturating left shifts 34;; 35;; == Uniform ternary arithmnetic 36;; ---- [INT] General ternary arithmetic that maps to unspecs 37;; ---- [INT] Multiply-and-accumulate operations 38;; ---- [INT] Binary logic operations with rotation 39;; ---- [INT] Ternary logic operations 40;; ---- [INT] Shift-and-accumulate operations 41;; ---- [INT] Shift-and-insert operations 42;; ---- [INT] Sum of absolute differences 43;; 44;; == Extending arithmetic 45;; ---- [INT] Wide binary arithmetic 46;; ---- [INT] Long binary arithmetic 47;; ---- [INT] Long left shifts 48;; ---- [INT] Long binary arithmetic with accumulation 49;; ---- [FP] Long multiplication with accumulation 50;; 51;; == Narrowing arithnetic 52;; ---- [INT] Narrowing unary arithmetic 53;; ---- [INT] Narrowing binary arithmetic 54;; ---- [INT] Narrowing right shifts 55;; 56;; == Pairwise arithmetic 57;; ---- [INT] Pairwise arithmetic 58;; ---- [FP] Pairwise arithmetic 59;; ---- [INT] Pairwise arithmetic with accumulation 60;; 61;; == Complex arithmetic 62;; ---- [INT] Complex binary operations 63;; ---- [INT] Complex ternary operations 64;; ---- [INT] Complex dot product 65;; 66;; == Conversions 67;; ---- [FP<-FP] Widening conversions 68;; ---- [FP<-FP] Narrowing conversions 69;; 70;; == Other arithmetic 71;; ---- [INT] Reciprocal approximation 72;; ---- [INT<-FP] Base-2 logarithm 73;; ---- [INT] Polynomial multiplication 74;; 75;; == Permutation 76;; ---- [INT,FP] General permutes 77;; ---- [INT] Optional bit-permute extensions 78;; 79;; == General 80;; ---- Check for aliases between pointers 81;; ---- Histogram processing 82;; ---- String matching 83;; 84;; == Crypotographic extensions 85;; ---- Optional AES extensions 86;; ---- Optional SHA-3 extensions 87;; ---- Optional SM4 extensions 88 89;; ========================================================================= 90;; == Moves 91;; ========================================================================= 92 93;; ------------------------------------------------------------------------- 94;; ---- Non-temporal gather loads 95;; ------------------------------------------------------------------------- 96;; Includes gather forms of: 97;; - LDNT1B 98;; - LDNT1D 99;; - LDNT1H 100;; - LDNT1W 101;; ------------------------------------------------------------------------- 102 103;; Non-extending loads. 104(define_insn "@aarch64_gather_ldnt<mode>" 105 [(set (match_operand:SVE_FULL_SD 0 "register_operand" "=w, w") 106 (unspec:SVE_FULL_SD 107 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 108 (match_operand:DI 2 "aarch64_reg_or_zero" "Z, r") 109 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w, w") 110 (mem:BLK (scratch))] 111 UNSPEC_LDNT1_GATHER))] 112 "TARGET_SVE2" 113 "@ 114 ldnt1<Vesize>\t%0.<Vetype>, %1/z, [%3.<Vetype>] 115 ldnt1<Vesize>\t%0.<Vetype>, %1/z, [%3.<Vetype>, %2]" 116) 117 118;; Extending loads. 119(define_insn_and_rewrite "@aarch64_gather_ldnt_<ANY_EXTEND:optab><SVE_FULL_SDI:mode><SVE_PARTIAL_I:mode>" 120 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, w") 121 (unspec:SVE_FULL_SDI 122 [(match_operand:<SVE_FULL_SDI:VPRED> 4 "general_operand" "UplDnm, UplDnm") 123 (ANY_EXTEND:SVE_FULL_SDI 124 (unspec:SVE_PARTIAL_I 125 [(match_operand:<SVE_FULL_SDI:VPRED> 1 "register_operand" "Upl, Upl") 126 (match_operand:DI 2 "aarch64_reg_or_zero" "Z, r") 127 (match_operand:<SVE_FULL_SDI:V_INT_EQUIV> 3 "register_operand" "w, w") 128 (mem:BLK (scratch))] 129 UNSPEC_LDNT1_GATHER))] 130 UNSPEC_PRED_X))] 131 "TARGET_SVE2 132 && (~<SVE_FULL_SDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0" 133 "@ 134 ldnt1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_SDI:Vetype>, %1/z, [%3.<SVE_FULL_SDI:Vetype>] 135 ldnt1<ANY_EXTEND:s><SVE_PARTIAL_I:Vesize>\t%0.<SVE_FULL_SDI:Vetype>, %1/z, [%3.<SVE_FULL_SDI:Vetype>, %2]" 136 "&& !CONSTANT_P (operands[4])" 137 { 138 operands[4] = CONSTM1_RTX (<SVE_FULL_SDI:VPRED>mode); 139 } 140) 141 142;; ------------------------------------------------------------------------- 143;; ---- Non-temporal scatter stores 144;; ------------------------------------------------------------------------- 145;; Includes scatter forms of: 146;; - STNT1B 147;; - STNT1D 148;; - STNT1H 149;; - STNT1W 150;; ------------------------------------------------------------------------- 151 152;; Non-truncating stores. 153(define_insn "@aarch64_scatter_stnt<mode>" 154 [(set (mem:BLK (scratch)) 155 (unspec:BLK 156 [(match_operand:<VPRED> 0 "register_operand" "Upl, Upl") 157 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, r") 158 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w") 159 (match_operand:SVE_FULL_SD 3 "register_operand" "w, w")] 160 161 UNSPEC_STNT1_SCATTER))] 162 "TARGET_SVE" 163 "@ 164 stnt1<Vesize>\t%3.<Vetype>, %0, [%2.<Vetype>] 165 stnt1<Vesize>\t%3.<Vetype>, %0, [%2.<Vetype>, %1]" 166) 167 168;; Truncating stores. 169(define_insn "@aarch64_scatter_stnt_<SVE_FULL_SDI:mode><SVE_PARTIAL_I:mode>" 170 [(set (mem:BLK (scratch)) 171 (unspec:BLK 172 [(match_operand:<SVE_FULL_SDI:VPRED> 0 "register_operand" "Upl, Upl") 173 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, r") 174 (match_operand:<SVE_FULL_SDI:V_INT_EQUIV> 2 "register_operand" "w, w") 175 (truncate:SVE_PARTIAL_I 176 (match_operand:SVE_FULL_SDI 3 "register_operand" "w, w"))] 177 UNSPEC_STNT1_SCATTER))] 178 "TARGET_SVE2 179 && (~<SVE_FULL_SDI:narrower_mask> & <SVE_PARTIAL_I:self_mask>) == 0" 180 "@ 181 stnt1<SVE_PARTIAL_I:Vesize>\t%3.<SVE_FULL_SDI:Vetype>, %0, [%2.<SVE_FULL_SDI:Vetype>] 182 stnt1<SVE_PARTIAL_I:Vesize>\t%3.<SVE_FULL_SDI:Vetype>, %0, [%2.<SVE_FULL_SDI:Vetype>, %1]" 183) 184 185;; ========================================================================= 186;; == Uniform binary arithmnetic 187;; ========================================================================= 188 189;; ------------------------------------------------------------------------- 190;; ---- [INT] Multiplication 191;; ------------------------------------------------------------------------- 192;; Includes the lane forms of: 193;; - MUL 194;; ------------------------------------------------------------------------- 195 196(define_insn "@aarch64_mul_lane_<mode>" 197 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") 198 (mult:SVE_FULL_HSDI 199 (unspec:SVE_FULL_HSDI 200 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "<sve_lane_con>") 201 (match_operand:SI 3 "const_int_operand")] 202 UNSPEC_SVE_LANE_SELECT) 203 (match_operand:SVE_FULL_HSDI 1 "register_operand" "w")))] 204 "TARGET_SVE2" 205 "mul\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]" 206) 207 208;; ------------------------------------------------------------------------- 209;; ---- [INT] Scaled high-part multiplication 210;; ------------------------------------------------------------------------- 211;; The patterns in this section are synthetic. 212;; ------------------------------------------------------------------------- 213 214;; Unpredicated integer multiply-high-with-(round-and-)scale. 215(define_expand "<su>mulh<r>s<mode>3" 216 [(set (match_operand:SVE_FULL_BHSI 0 "register_operand") 217 (unspec:SVE_FULL_BHSI 218 [(match_dup 3) 219 (unspec:SVE_FULL_BHSI 220 [(match_operand:SVE_FULL_BHSI 1 "register_operand") 221 (match_operand:SVE_FULL_BHSI 2 "register_operand")] 222 MULHRS)] 223 UNSPEC_PRED_X))] 224 "TARGET_SVE2" 225 { 226 operands[3] = aarch64_ptrue_reg (<VPRED>mode); 227 228 rtx prod_b = gen_reg_rtx (<VWIDE>mode); 229 rtx prod_t = gen_reg_rtx (<VWIDE>mode); 230 emit_insn (gen_aarch64_sve_<su>mullb<Vwide> (prod_b, operands[1], 231 operands[2])); 232 emit_insn (gen_aarch64_sve_<su>mullt<Vwide> (prod_t, operands[1], 233 operands[2])); 234 235 rtx shift = GEN_INT (GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1); 236 emit_insn (gen_aarch64_sve_<r>shrnb<Vwide> (operands[0], prod_b, shift)); 237 emit_insn (gen_aarch64_sve_<r>shrnt<Vwide> (operands[0], operands[0], 238 prod_t, shift)); 239 240 DONE; 241 } 242) 243 244;; ------------------------------------------------------------------------- 245;; ---- [INT] General binary arithmetic that maps to unspecs 246;; ------------------------------------------------------------------------- 247;; Includes: 248;; - SHADD 249;; - SHSUB 250;; - SHSUBR 251;; - SQRSHL 252;; - SQRSHLR 253;; - SRHADD 254;; - SRSHL 255;; - SRSHLR 256;; - SUQADD 257;; - UHADD 258;; - UHSUB 259;; - UHSUBR 260;; - UQRSHL 261;; - UQRSHLR 262;; - URHADD 263;; - URSHL 264;; - URSHLR 265;; - USQADD 266;; ------------------------------------------------------------------------- 267 268;; Integer average (floor). 269(define_expand "<u>avg<mode>3_floor" 270 [(set (match_operand:SVE_FULL_I 0 "register_operand") 271 (unspec:SVE_FULL_I 272 [(match_dup 3) 273 (unspec:SVE_FULL_I 274 [(match_operand:SVE_FULL_I 1 "register_operand") 275 (match_operand:SVE_FULL_I 2 "register_operand")] 276 HADD)] 277 UNSPEC_PRED_X))] 278 "TARGET_SVE2" 279 { 280 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 281 } 282) 283 284;; Integer average (rounding). 285(define_expand "<u>avg<mode>3_ceil" 286 [(set (match_operand:SVE_FULL_I 0 "register_operand") 287 (unspec:SVE_FULL_I 288 [(match_dup 3) 289 (unspec:SVE_FULL_I 290 [(match_operand:SVE_FULL_I 1 "register_operand") 291 (match_operand:SVE_FULL_I 2 "register_operand")] 292 RHADD)] 293 UNSPEC_PRED_X))] 294 "TARGET_SVE2" 295 { 296 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 297 } 298) 299 300;; The immediate form of SQADD acts as an immediate form of SUQADD 301;; over its full range. In contrast to the ss_plus pattern, we do 302;; not need to treat byte immediates specially. E.g.: 303;; 304;; SQADD Z0.B, Z0.B, #128 305;; 306;; is equivalent to: 307;; 308;; MOV Z1.B, #128 309;; SUQADD Z0.B, P0/M, Z0.B, Z1.B 310;; 311;; even though it's not equivalent to: 312;; 313;; MOV Z1.B, #128 314;; SQADD Z0.B, P0/M, Z0.B, Z1.B // Saturating subtraction of 128 315(define_insn "@aarch64_sve_suqadd<mode>_const" 316 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 317 (unspec:SVE_FULL_I 318 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w") 319 (match_operand:SVE_FULL_I 2 "aarch64_sve_arith_immediate")] 320 UNSPEC_SUQADD))] 321 "TARGET_SVE2" 322 "@ 323 sqadd\t%0.<Vetype>, %0.<Vetype>, #%D2 324 movprfx\t%0, %1\;sqadd\t%0.<Vetype>, %0.<Vetype>, #%D2" 325 [(set_attr "movprfx" "*,yes")] 326) 327 328;; General predicated binary arithmetic. All operations handled here 329;; are commutative or have a reversed form. 330(define_insn "@aarch64_pred_<sve_int_op><mode>" 331 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w") 332 (unspec:SVE_FULL_I 333 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 334 (unspec:SVE_FULL_I 335 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w, w") 336 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w")] 337 SVE2_COND_INT_BINARY_REV)] 338 UNSPEC_PRED_X))] 339 "TARGET_SVE2" 340 "@ 341 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 342 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 343 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 344 [(set_attr "movprfx" "*,*,yes")] 345) 346 347;; Predicated binary arithmetic with merging. 348(define_expand "@cond_<sve_int_op><mode>" 349 [(set (match_operand:SVE_FULL_I 0 "register_operand") 350 (unspec:SVE_FULL_I 351 [(match_operand:<VPRED> 1 "register_operand") 352 (unspec:SVE_FULL_I 353 [(match_dup 5) 354 (unspec:SVE_FULL_I 355 [(match_operand:SVE_FULL_I 2 "register_operand") 356 (match_operand:SVE_FULL_I 3 "register_operand")] 357 SVE2_COND_INT_BINARY)] 358 UNSPEC_PRED_X) 359 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero")] 360 UNSPEC_SEL))] 361 "TARGET_SVE2" 362 { 363 operands[5] = CONSTM1_RTX (<MODE>mode); 364 } 365) 366 367;; Predicated binary arithmetic, merging with the first input. 368(define_insn_and_rewrite "*cond_<sve_int_op><mode>_2" 369 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 370 (unspec:SVE_FULL_I 371 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 372 (unspec:SVE_FULL_I 373 [(match_operand 4) 374 (unspec:SVE_FULL_I 375 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w") 376 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")] 377 SVE2_COND_INT_BINARY)] 378 UNSPEC_PRED_X) 379 (match_dup 2)] 380 UNSPEC_SEL))] 381 "TARGET_SVE2" 382 "@ 383 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 384 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 385 "&& !CONSTANT_P (operands[4])" 386 { 387 operands[4] = CONSTM1_RTX (<VPRED>mode); 388 } 389 [(set_attr "movprfx" "*,yes")] 390) 391 392;; Predicated binary arithmetic, merging with the second input. 393(define_insn_and_rewrite "*cond_<sve_int_op><mode>_3" 394 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 395 (unspec:SVE_FULL_I 396 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 397 (unspec:SVE_FULL_I 398 [(match_operand 4) 399 (unspec:SVE_FULL_I 400 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w") 401 (match_operand:SVE_FULL_I 3 "register_operand" "0, w")] 402 SVE2_COND_INT_BINARY_REV)] 403 UNSPEC_PRED_X) 404 (match_dup 3)] 405 UNSPEC_SEL))] 406 "TARGET_SVE2" 407 "@ 408 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 409 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>" 410 "&& !CONSTANT_P (operands[4])" 411 { 412 operands[4] = CONSTM1_RTX (<VPRED>mode); 413 } 414 [(set_attr "movprfx" "*,yes")] 415) 416 417;; Predicated binary operations, merging with an independent value. 418(define_insn_and_rewrite "*cond_<sve_int_op><mode>_any" 419 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, ?&w") 420 (unspec:SVE_FULL_I 421 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl") 422 (unspec:SVE_FULL_I 423 [(match_operand 5) 424 (unspec:SVE_FULL_I 425 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w, w, w, w") 426 (match_operand:SVE_FULL_I 3 "register_operand" "w, 0, w, w, w")] 427 SVE2_COND_INT_BINARY_REV)] 428 UNSPEC_PRED_X) 429 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, 0, w")] 430 UNSPEC_SEL))] 431 "TARGET_SVE2 432 && !rtx_equal_p (operands[2], operands[4]) 433 && !rtx_equal_p (operands[3], operands[4])" 434 "@ 435 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 436 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 437 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 438 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 439 #" 440 "&& 1" 441 { 442 if (reload_completed 443 && register_operand (operands[4], <MODE>mode) 444 && !rtx_equal_p (operands[0], operands[4])) 445 { 446 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2], 447 operands[4], operands[1])); 448 operands[4] = operands[2] = operands[0]; 449 } 450 else if (!CONSTANT_P (operands[5])) 451 operands[5] = CONSTM1_RTX (<VPRED>mode); 452 else 453 FAIL; 454 } 455 [(set_attr "movprfx" "yes")] 456) 457 458;; Predicated binary operations with no reverse form, merging with zero. 459;; At present we don't generate these patterns via a cond_* optab, 460;; so there's no correctness requirement to handle merging with an 461;; independent value. 462(define_insn_and_rewrite "*cond_<sve_int_op><mode>_z" 463 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w") 464 (unspec:SVE_FULL_I 465 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 466 (unspec:SVE_FULL_I 467 [(match_operand 5) 468 (unspec:SVE_FULL_I 469 [(match_operand:SVE_FULL_I 2 "register_operand" "0, w") 470 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")] 471 SVE2_COND_INT_BINARY_NOREV)] 472 UNSPEC_PRED_X) 473 (match_operand:SVE_FULL_I 4 "aarch64_simd_imm_zero")] 474 UNSPEC_SEL))] 475 "TARGET_SVE2" 476 "@ 477 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 478 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 479 "&& !CONSTANT_P (operands[5])" 480 { 481 operands[5] = CONSTM1_RTX (<VPRED>mode); 482 } 483 [(set_attr "movprfx" "yes")] 484) 485 486;; ------------------------------------------------------------------------- 487;; ---- [INT] Saturating binary arithmetic 488;; ------------------------------------------------------------------------- 489;; Includes: 490;; - SQDMULH 491;; - SQRDMULH 492;; ------------------------------------------------------------------------- 493 494(define_insn "@aarch64_sve_<sve_int_op><mode>" 495 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w") 496 (unspec:SVE_FULL_I 497 [(match_operand:SVE_FULL_I 1 "register_operand" "w") 498 (match_operand:SVE_FULL_I 2 "register_operand" "w")] 499 SVE2_INT_BINARY))] 500 "TARGET_SVE2" 501 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 502) 503 504(define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>" 505 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") 506 (unspec:SVE_FULL_HSDI 507 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w") 508 (unspec:SVE_FULL_HSDI 509 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "<sve_lane_con>") 510 (match_operand:SI 3 "const_int_operand")] 511 UNSPEC_SVE_LANE_SELECT)] 512 SVE2_INT_BINARY_LANE))] 513 "TARGET_SVE2" 514 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>[%3]" 515) 516 517;; ------------------------------------------------------------------------- 518;; ---- [INT] Saturating left shifts 519;; ------------------------------------------------------------------------- 520;; Includes: 521;; - SQSHL 522;; - SQSHLR 523;; - UQSHL 524;; - UQSHLR 525;; ------------------------------------------------------------------------- 526 527;; Predicated left shifts. 528(define_insn "@aarch64_pred_<sve_int_op><mode>" 529 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, w, ?&w, ?&w") 530 (unspec:SVE_FULL_I 531 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl") 532 (unspec:SVE_FULL_I 533 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w, w") 534 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, 0, D<lr>, w")] 535 SVE2_COND_INT_SHIFT)] 536 UNSPEC_PRED_X))] 537 "TARGET_SVE2" 538 "@ 539 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 540 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 541 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 542 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 543 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 544 [(set_attr "movprfx" "*,*,*,yes,yes")] 545) 546 547;; Predicated left shifts with merging. 548(define_expand "@cond_<sve_int_op><mode>" 549 [(set (match_operand:SVE_FULL_I 0 "register_operand") 550 (unspec:SVE_FULL_I 551 [(match_operand:<VPRED> 1 "register_operand") 552 (unspec:SVE_FULL_I 553 [(match_dup 5) 554 (unspec:SVE_FULL_I 555 [(match_operand:SVE_FULL_I 2 "register_operand") 556 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand")] 557 SVE2_COND_INT_SHIFT)] 558 UNSPEC_PRED_X) 559 (match_operand:SVE_FULL_I 4 "register_operand")] 560 UNSPEC_SEL))] 561 "TARGET_SVE2" 562 { 563 operands[5] = CONSTM1_RTX (<VPRED>mode); 564 } 565) 566 567;; Predicated left shifts, merging with the first input. 568(define_insn_and_rewrite "*cond_<sve_int_op><mode>_2" 569 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, ?&w, ?&w") 570 (unspec:SVE_FULL_I 571 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl") 572 (unspec:SVE_FULL_I 573 [(match_operand 4) 574 (unspec:SVE_FULL_I 575 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w") 576 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, D<lr>, w")] 577 SVE2_COND_INT_SHIFT)] 578 UNSPEC_PRED_X) 579 (match_dup 2)] 580 UNSPEC_SEL))] 581 "TARGET_SVE2" 582 "@ 583 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 584 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 585 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 586 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 587 "&& !CONSTANT_P (operands[4])" 588 { 589 operands[4] = CONSTM1_RTX (<VPRED>mode); 590 } 591 [(set_attr "movprfx" "*,*,yes,yes")] 592) 593 594;; Predicated left shifts, merging with the second input. 595(define_insn_and_rewrite "*cond_<sve_int_op><mode>_3" 596 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 597 (unspec:SVE_FULL_I 598 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 599 (unspec:SVE_FULL_I 600 [(match_operand 4) 601 (unspec:SVE_FULL_I 602 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w") 603 (match_operand:SVE_FULL_I 3 "register_operand" "0, w")] 604 SVE2_COND_INT_SHIFT)] 605 UNSPEC_PRED_X) 606 (match_dup 3)] 607 UNSPEC_SEL))] 608 "TARGET_SVE2" 609 "@ 610 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 611 movprfx\t%0, %3\;<sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>" 612 "&& !CONSTANT_P (operands[4])" 613 { 614 operands[4] = CONSTM1_RTX (<VPRED>mode); 615 } 616 [(set_attr "movprfx" "*,yes")] 617) 618 619;; Predicated left shifts, merging with an independent value. 620(define_insn_and_rewrite "*cond_<sve_int_op><mode>_any" 621 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=&w, &w, &w, &w, &w, &w, &w, ?&w, ?&w") 622 (unspec:SVE_FULL_I 623 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl, Upl, Upl, Upl, Upl, Upl") 624 (unspec:SVE_FULL_I 625 [(match_operand 5) 626 (unspec:SVE_FULL_I 627 [(match_operand:SVE_FULL_I 2 "register_operand" "0, 0, w, w, w, w, w, w, w") 628 (match_operand:SVE_FULL_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, 0, D<lr>, w, D<lr>, w, D<lr>, w")] 629 SVE2_COND_INT_SHIFT)] 630 UNSPEC_PRED_X) 631 (match_operand:SVE_FULL_I 4 "aarch64_simd_reg_or_zero" "Dz, Dz, Dz, Dz, Dz, 0, 0, w, w")] 632 UNSPEC_SEL))] 633 "TARGET_SVE2 634 && !rtx_equal_p (operands[2], operands[4]) 635 && (CONSTANT_P (operands[4]) || !rtx_equal_p (operands[3], operands[4]))" 636 "@ 637 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 638 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 639 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 640 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 641 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 642 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 643 movprfx\t%0.<Vetype>, %1/m, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 644 # 645 #" 646 "&& 1" 647 { 648 if (reload_completed 649 && register_operand (operands[4], <MODE>mode) 650 && !rtx_equal_p (operands[0], operands[4])) 651 { 652 emit_insn (gen_vcond_mask_<mode><vpred> (operands[0], operands[2], 653 operands[4], operands[1])); 654 operands[4] = operands[2] = operands[0]; 655 } 656 else if (!CONSTANT_P (operands[5])) 657 operands[5] = CONSTM1_RTX (<VPRED>mode); 658 else 659 FAIL; 660 } 661 [(set_attr "movprfx" "yes")] 662) 663 664;; ========================================================================= 665;; == Uniform ternary arithmnetic 666;; ========================================================================= 667 668;; ------------------------------------------------------------------------- 669;; ---- [INT] General ternary arithmetic that maps to unspecs 670;; ------------------------------------------------------------------------- 671;; Includes: 672;; - ADCLB 673;; - ADCLT 674;; - EORBT 675;; - EORTB 676;; - SBCLB 677;; - SBCLT 678;; - SQRDMLAH 679;; - SQRDMLSH 680;; ------------------------------------------------------------------------- 681 682(define_insn "@aarch64_sve_<sve_int_op><mode>" 683 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 684 (unspec:SVE_FULL_I 685 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w") 686 (match_operand:SVE_FULL_I 3 "register_operand" "w, w") 687 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")] 688 SVE2_INT_TERNARY))] 689 "TARGET_SVE2" 690 "@ 691 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype> 692 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>" 693 [(set_attr "movprfx" "*,yes")] 694) 695 696(define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>" 697 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 698 (unspec:SVE_FULL_HSDI 699 [(match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w") 700 (unspec:SVE_FULL_HSDI 701 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 702 (match_operand:SI 4 "const_int_operand")] 703 UNSPEC_SVE_LANE_SELECT) 704 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")] 705 SVE2_INT_TERNARY_LANE))] 706 "TARGET_SVE2" 707 "@ 708 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4] 709 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]" 710 [(set_attr "movprfx" "*,yes")] 711) 712 713;; ------------------------------------------------------------------------- 714;; ---- [INT] Multiply-and-accumulate operations 715;; ------------------------------------------------------------------------- 716;; Includes the lane forms of: 717;; - MLA 718;; - MLS 719;; ------------------------------------------------------------------------- 720 721(define_insn "@aarch64_sve_add_mul_lane_<mode>" 722 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 723 (plus:SVE_FULL_HSDI 724 (mult:SVE_FULL_HSDI 725 (unspec:SVE_FULL_HSDI 726 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 727 (match_operand:SI 4 "const_int_operand")] 728 UNSPEC_SVE_LANE_SELECT) 729 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w")) 730 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))] 731 "TARGET_SVE2" 732 "@ 733 mla\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4] 734 movprfx\t%0, %1\;mla\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]" 735 [(set_attr "movprfx" "*,yes")] 736) 737 738(define_insn "@aarch64_sve_sub_mul_lane_<mode>" 739 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 740 (minus:SVE_FULL_HSDI 741 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w") 742 (mult:SVE_FULL_HSDI 743 (unspec:SVE_FULL_HSDI 744 [(match_operand:SVE_FULL_HSDI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 745 (match_operand:SI 4 "const_int_operand")] 746 UNSPEC_SVE_LANE_SELECT) 747 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w, w"))))] 748 "TARGET_SVE2" 749 "@ 750 mls\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4] 751 movprfx\t%0, %1\;mls\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4]" 752 [(set_attr "movprfx" "*,yes")] 753) 754 755;; ------------------------------------------------------------------------- 756;; ---- [INT] Binary logic operations with rotation 757;; ------------------------------------------------------------------------- 758;; Includes: 759;; - XAR 760;; ------------------------------------------------------------------------- 761 762(define_insn "@aarch64_sve2_xar<mode>" 763 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 764 (rotatert:SVE_FULL_I 765 (xor:SVE_FULL_I 766 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w") 767 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")) 768 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")))] 769 "TARGET_SVE2" 770 "@ 771 xar\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3 772 movprfx\t%0, %1\;xar\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #%3" 773 [(set_attr "movprfx" "*,yes")] 774) 775 776;; ------------------------------------------------------------------------- 777;; ---- [INT] Ternary logic operations 778;; ------------------------------------------------------------------------- 779;; Includes: 780;; - BCAX 781;; - BSL 782;; - BSL1N 783;; - BSL2N 784;; - EOR3 785;; - NBSL 786;; ------------------------------------------------------------------------- 787 788;; Unpredicated exclusive OR of AND. 789(define_expand "@aarch64_sve2_bcax<mode>" 790 [(set (match_operand:SVE_FULL_I 0 "register_operand") 791 (xor:SVE_FULL_I 792 (and:SVE_FULL_I 793 (unspec:SVE_FULL_I 794 [(match_dup 4) 795 (not:SVE_FULL_I 796 (match_operand:SVE_FULL_I 3 "register_operand"))] 797 UNSPEC_PRED_X) 798 (match_operand:SVE_FULL_I 2 "register_operand")) 799 (match_operand:SVE_FULL_I 1 "register_operand")))] 800 "TARGET_SVE2" 801 { 802 operands[4] = CONSTM1_RTX (<VPRED>mode); 803 } 804) 805 806(define_insn_and_rewrite "*aarch64_sve2_bcax<mode>" 807 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 808 (xor:SVE_FULL_I 809 (and:SVE_FULL_I 810 (unspec:SVE_FULL_I 811 [(match_operand 4) 812 (not:SVE_FULL_I 813 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))] 814 UNSPEC_PRED_X) 815 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")) 816 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))] 817 "TARGET_SVE2" 818 "@ 819 bcax\t%0.d, %0.d, %2.d, %3.d 820 movprfx\t%0, %1\;bcax\t%0.d, %0.d, %2.d, %3.d" 821 "&& !CONSTANT_P (operands[4])" 822 { 823 operands[4] = CONSTM1_RTX (<VPRED>mode); 824 } 825 [(set_attr "movprfx" "*,yes")] 826) 827 828;; Unpredicated 3-way exclusive OR. 829(define_insn "@aarch64_sve2_eor3<mode>" 830 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, w, w, ?&w") 831 (xor:SVE_FULL_I 832 (xor:SVE_FULL_I 833 (match_operand:SVE_FULL_I 1 "register_operand" "0, w, w, w") 834 (match_operand:SVE_FULL_I 2 "register_operand" "w, 0, w, w")) 835 (match_operand:SVE_FULL_I 3 "register_operand" "w, w, 0, w")))] 836 "TARGET_SVE2" 837 "@ 838 eor3\t%0.d, %0.d, %2.d, %3.d 839 eor3\t%0.d, %0.d, %1.d, %3.d 840 eor3\t%0.d, %0.d, %1.d, %2.d 841 movprfx\t%0, %1\;eor3\t%0.d, %0.d, %2.d, %3.d" 842 [(set_attr "movprfx" "*,*,*,yes")] 843) 844 845;; Use NBSL for vector NOR. 846(define_insn_and_rewrite "*aarch64_sve2_nor<mode>" 847 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 848 (unspec:SVE_FULL_I 849 [(match_operand 3) 850 (and:SVE_FULL_I 851 (not:SVE_FULL_I 852 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w")) 853 (not:SVE_FULL_I 854 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")))] 855 UNSPEC_PRED_X))] 856 "TARGET_SVE2" 857 "@ 858 nbsl\t%0.d, %0.d, %2.d, %0.d 859 movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %0.d" 860 "&& !CONSTANT_P (operands[3])" 861 { 862 operands[3] = CONSTM1_RTX (<VPRED>mode); 863 } 864 [(set_attr "movprfx" "*,yes")] 865) 866 867;; Use NBSL for vector NAND. 868(define_insn_and_rewrite "*aarch64_sve2_nand<mode>" 869 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 870 (unspec:SVE_FULL_I 871 [(match_operand 3) 872 (ior:SVE_FULL_I 873 (not:SVE_FULL_I 874 (match_operand:SVE_FULL_I 1 "register_operand" "%0, w")) 875 (not:SVE_FULL_I 876 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")))] 877 UNSPEC_PRED_X))] 878 "TARGET_SVE2" 879 "@ 880 nbsl\t%0.d, %0.d, %2.d, %2.d 881 movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %2.d" 882 "&& !CONSTANT_P (operands[3])" 883 { 884 operands[3] = CONSTM1_RTX (<VPRED>mode); 885 } 886 [(set_attr "movprfx" "*,yes")] 887) 888 889;; Unpredicated bitwise select. 890;; (op3 ? bsl_mov : bsl_dup) == (((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup) 891(define_expand "@aarch64_sve2_bsl<mode>" 892 [(set (match_operand:SVE_FULL_I 0 "register_operand") 893 (xor:SVE_FULL_I 894 (and:SVE_FULL_I 895 (xor:SVE_FULL_I 896 (match_operand:SVE_FULL_I 1 "register_operand") 897 (match_operand:SVE_FULL_I 2 "register_operand")) 898 (match_operand:SVE_FULL_I 3 "register_operand")) 899 (match_dup 2)))] 900 "TARGET_SVE2" 901) 902 903(define_insn "*aarch64_sve2_bsl<mode>" 904 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 905 (xor:SVE_FULL_I 906 (and:SVE_FULL_I 907 (xor:SVE_FULL_I 908 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w") 909 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")) 910 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")) 911 (match_dup BSL_DUP)))] 912 "TARGET_SVE2" 913 "@ 914 bsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d 915 movprfx\t%0, %<bsl_mov>\;bsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d" 916 [(set_attr "movprfx" "*,yes")] 917) 918 919;; Unpredicated bitwise inverted select. 920;; (~(op3 ? bsl_mov : bsl_dup)) == (~(((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup)) 921(define_expand "@aarch64_sve2_nbsl<mode>" 922 [(set (match_operand:SVE_FULL_I 0 "register_operand") 923 (unspec:SVE_FULL_I 924 [(match_dup 4) 925 (not:SVE_FULL_I 926 (xor:SVE_FULL_I 927 (and:SVE_FULL_I 928 (xor:SVE_FULL_I 929 (match_operand:SVE_FULL_I 1 "register_operand") 930 (match_operand:SVE_FULL_I 2 "register_operand")) 931 (match_operand:SVE_FULL_I 3 "register_operand")) 932 (match_dup 2)))] 933 UNSPEC_PRED_X))] 934 "TARGET_SVE2" 935 { 936 operands[4] = CONSTM1_RTX (<VPRED>mode); 937 } 938) 939 940(define_insn_and_rewrite "*aarch64_sve2_nbsl<mode>" 941 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 942 (unspec:SVE_FULL_I 943 [(match_operand 4) 944 (not:SVE_FULL_I 945 (xor:SVE_FULL_I 946 (and:SVE_FULL_I 947 (xor:SVE_FULL_I 948 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w") 949 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")) 950 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")) 951 (match_dup BSL_DUP)))] 952 UNSPEC_PRED_X))] 953 "TARGET_SVE2" 954 "@ 955 nbsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d 956 movprfx\t%0, %<bsl_mov>\;nbsl\t%0.d, %0.d, %<bsl_dup>.d, %3.d" 957 "&& !CONSTANT_P (operands[4])" 958 { 959 operands[4] = CONSTM1_RTX (<VPRED>mode); 960 } 961 [(set_attr "movprfx" "*,yes")] 962) 963 964;; Unpredicated bitwise select with inverted first operand. 965;; (op3 ? ~bsl_mov : bsl_dup) == ((~(bsl_mov ^ bsl_dup) & op3) ^ bsl_dup) 966(define_expand "@aarch64_sve2_bsl1n<mode>" 967 [(set (match_operand:SVE_FULL_I 0 "register_operand") 968 (xor:SVE_FULL_I 969 (and:SVE_FULL_I 970 (unspec:SVE_FULL_I 971 [(match_dup 4) 972 (not:SVE_FULL_I 973 (xor:SVE_FULL_I 974 (match_operand:SVE_FULL_I 1 "register_operand") 975 (match_operand:SVE_FULL_I 2 "register_operand")))] 976 UNSPEC_PRED_X) 977 (match_operand:SVE_FULL_I 3 "register_operand")) 978 (match_dup 2)))] 979 "TARGET_SVE2" 980 { 981 operands[4] = CONSTM1_RTX (<VPRED>mode); 982 } 983) 984 985(define_insn_and_rewrite "*aarch64_sve2_bsl1n<mode>" 986 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 987 (xor:SVE_FULL_I 988 (and:SVE_FULL_I 989 (unspec:SVE_FULL_I 990 [(match_operand 4) 991 (not:SVE_FULL_I 992 (xor:SVE_FULL_I 993 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w") 994 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")))] 995 UNSPEC_PRED_X) 996 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")) 997 (match_dup BSL_DUP)))] 998 "TARGET_SVE2" 999 "@ 1000 bsl1n\t%0.d, %0.d, %<bsl_dup>.d, %3.d 1001 movprfx\t%0, %<bsl_mov>\;bsl1n\t%0.d, %0.d, %<bsl_dup>.d, %3.d" 1002 "&& !CONSTANT_P (operands[4])" 1003 { 1004 operands[4] = CONSTM1_RTX (<VPRED>mode); 1005 } 1006 [(set_attr "movprfx" "*,yes")] 1007) 1008 1009;; Unpredicated bitwise select with inverted second operand. 1010;; (bsl_dup ? bsl_mov : ~op3) == ((bsl_dup & bsl_mov) | (~op3 & ~bsl_dup)) 1011(define_expand "@aarch64_sve2_bsl2n<mode>" 1012 [(set (match_operand:SVE_FULL_I 0 "register_operand") 1013 (ior:SVE_FULL_I 1014 (and:SVE_FULL_I 1015 (match_operand:SVE_FULL_I 1 "register_operand") 1016 (match_operand:SVE_FULL_I 3 "register_operand")) 1017 (unspec:SVE_FULL_I 1018 [(match_dup 4) 1019 (and:SVE_FULL_I 1020 (not:SVE_FULL_I 1021 (match_operand:SVE_FULL_I 2 "register_operand")) 1022 (not:SVE_FULL_I 1023 (match_dup 3)))] 1024 UNSPEC_PRED_X)))] 1025 "TARGET_SVE2" 1026 { 1027 operands[4] = CONSTM1_RTX (<VPRED>mode); 1028 } 1029) 1030 1031(define_insn_and_rewrite "*aarch64_sve2_bsl2n<mode>" 1032 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1033 (ior:SVE_FULL_I 1034 (and:SVE_FULL_I 1035 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w") 1036 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")) 1037 (unspec:SVE_FULL_I 1038 [(match_operand 4) 1039 (and:SVE_FULL_I 1040 (not:SVE_FULL_I 1041 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")) 1042 (not:SVE_FULL_I 1043 (match_dup BSL_DUP)))] 1044 UNSPEC_PRED_X)))] 1045 "TARGET_SVE2" 1046 "@ 1047 bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d 1048 movprfx\t%0, %<bsl_mov>\;bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d" 1049 "&& !CONSTANT_P (operands[4])" 1050 { 1051 operands[4] = CONSTM1_RTX (<VPRED>mode); 1052 } 1053 [(set_attr "movprfx" "*,yes")] 1054) 1055 1056;; Unpredicated bitwise select with inverted second operand, alternative form. 1057;; (bsl_dup ? bsl_mov : ~op3) == ((bsl_dup & bsl_mov) | (~bsl_dup & ~op3)) 1058(define_insn_and_rewrite "*aarch64_sve2_bsl2n<mode>" 1059 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1060 (ior:SVE_FULL_I 1061 (and:SVE_FULL_I 1062 (match_operand:SVE_FULL_I 1 "register_operand" "<bsl_1st>, w") 1063 (match_operand:SVE_FULL_I 2 "register_operand" "<bsl_2nd>, w")) 1064 (unspec:SVE_FULL_I 1065 [(match_operand 4) 1066 (and:SVE_FULL_I 1067 (not:SVE_FULL_I 1068 (match_dup BSL_DUP)) 1069 (not:SVE_FULL_I 1070 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")))] 1071 UNSPEC_PRED_X)))] 1072 "TARGET_SVE2" 1073 "@ 1074 bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d 1075 movprfx\t%0, %<bsl_mov>\;bsl2n\t%0.d, %0.d, %3.d, %<bsl_dup>.d" 1076 "&& !CONSTANT_P (operands[4])" 1077 { 1078 operands[4] = CONSTM1_RTX (<VPRED>mode); 1079 } 1080 [(set_attr "movprfx" "*,yes")] 1081) 1082 1083;; ------------------------------------------------------------------------- 1084;; ---- [INT] Shift-and-accumulate operations 1085;; ------------------------------------------------------------------------- 1086;; Includes: 1087;; - SRSRA 1088;; - SSRA 1089;; - URSRA 1090;; - USRA 1091;; ------------------------------------------------------------------------- 1092 1093;; Provide the natural unpredicated interface for SSRA and USRA. 1094(define_expand "@aarch64_sve_add_<sve_int_op><mode>" 1095 [(set (match_operand:SVE_FULL_I 0 "register_operand") 1096 (plus:SVE_FULL_I 1097 (unspec:SVE_FULL_I 1098 [(match_dup 4) 1099 (SHIFTRT:SVE_FULL_I 1100 (match_operand:SVE_FULL_I 2 "register_operand") 1101 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm"))] 1102 UNSPEC_PRED_X) 1103 (match_operand:SVE_FULL_I 1 "register_operand")))] 1104 "TARGET_SVE2" 1105 { 1106 operands[4] = CONSTM1_RTX (<VPRED>mode); 1107 } 1108) 1109 1110;; Pattern-match SSRA and USRA as a predicated operation whose predicate 1111;; isn't needed. 1112(define_insn_and_rewrite "*aarch64_sve2_sra<mode>" 1113 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1114 (plus:SVE_FULL_I 1115 (unspec:SVE_FULL_I 1116 [(match_operand 4) 1117 (SHIFTRT:SVE_FULL_I 1118 (match_operand:SVE_FULL_I 2 "register_operand" "w, w") 1119 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm"))] 1120 UNSPEC_PRED_X) 1121 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))] 1122 "TARGET_SVE2" 1123 "@ 1124 <sra_op>sra\t%0.<Vetype>, %2.<Vetype>, #%3 1125 movprfx\t%0, %1\;<sra_op>sra\t%0.<Vetype>, %2.<Vetype>, #%3" 1126 "&& !CONSTANT_P (operands[4])" 1127 { 1128 operands[4] = CONSTM1_RTX (<VPRED>mode); 1129 } 1130 [(set_attr "movprfx" "*,yes")] 1131) 1132 1133;; SRSRA and URSRA. 1134(define_insn "@aarch64_sve_add_<sve_int_op><mode>" 1135 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1136 (plus:SVE_FULL_I 1137 (unspec:SVE_FULL_I 1138 [(match_operand:SVE_FULL_I 2 "register_operand" "w, w") 1139 (match_operand:SVE_FULL_I 3 "aarch64_simd_rshift_imm")] 1140 VRSHR_N) 1141 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))] 1142 "TARGET_SVE2" 1143 "@ 1144 <sur>sra\t%0.<Vetype>, %2.<Vetype>, #%3 1145 movprfx\t%0, %1\;<sur>sra\t%0.<Vetype>, %2.<Vetype>, #%3" 1146 [(set_attr "movprfx" "*,yes")] 1147) 1148 1149;; ------------------------------------------------------------------------- 1150;; ---- [INT] Shift-and-insert operations 1151;; ------------------------------------------------------------------------- 1152;; Includes: 1153;; - SLI 1154;; - SRI 1155;; ------------------------------------------------------------------------- 1156 1157;; These instructions do not take MOVPRFX. 1158(define_insn "@aarch64_sve_<sve_int_op><mode>" 1159 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w") 1160 (unspec:SVE_FULL_I 1161 [(match_operand:SVE_FULL_I 1 "register_operand" "0") 1162 (match_operand:SVE_FULL_I 2 "register_operand" "w") 1163 (match_operand:SVE_FULL_I 3 "aarch64_simd_<lr>shift_imm")] 1164 SVE2_INT_SHIFT_INSERT))] 1165 "TARGET_SVE2" 1166 "<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, #%3" 1167) 1168 1169;; ------------------------------------------------------------------------- 1170;; ---- [INT] Sum of absolute differences 1171;; ------------------------------------------------------------------------- 1172;; Includes: 1173;; - SABA 1174;; - UABA 1175;; ------------------------------------------------------------------------- 1176 1177;; Provide the natural unpredicated interface for SABA and UABA. 1178(define_expand "@aarch64_sve2_<su>aba<mode>" 1179 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1180 (plus:SVE_FULL_I 1181 (minus:SVE_FULL_I 1182 (unspec:SVE_FULL_I 1183 [(match_dup 4) 1184 (USMAX:SVE_FULL_I 1185 (match_operand:SVE_FULL_I 2 "register_operand" "w, w") 1186 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))] 1187 UNSPEC_PRED_X) 1188 (unspec:SVE_FULL_I 1189 [(match_dup 4) 1190 (<max_opp>:SVE_FULL_I 1191 (match_dup 2) 1192 (match_dup 3))] 1193 UNSPEC_PRED_X)) 1194 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))] 1195 "TARGET_SVE2" 1196 { 1197 operands[4] = CONSTM1_RTX (<VPRED>mode); 1198 } 1199) 1200 1201;; Pattern-match SABA and UABA as an absolute-difference-and-accumulate 1202;; operation whose predicates aren't needed. 1203(define_insn "*aarch64_sve2_<su>aba<mode>" 1204 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1205 (plus:SVE_FULL_I 1206 (minus:SVE_FULL_I 1207 (unspec:SVE_FULL_I 1208 [(match_operand 4) 1209 (USMAX:SVE_FULL_I 1210 (match_operand:SVE_FULL_I 2 "register_operand" "w, w") 1211 (match_operand:SVE_FULL_I 3 "register_operand" "w, w"))] 1212 UNSPEC_PRED_X) 1213 (unspec:SVE_FULL_I 1214 [(match_operand 5) 1215 (<max_opp>:SVE_FULL_I 1216 (match_dup 2) 1217 (match_dup 3))] 1218 UNSPEC_PRED_X)) 1219 (match_operand:SVE_FULL_I 1 "register_operand" "0, w")))] 1220 "TARGET_SVE2" 1221 "@ 1222 <su>aba\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype> 1223 movprfx\t%0, %1\;<su>aba\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>" 1224 [(set_attr "movprfx" "*,yes")] 1225) 1226 1227;; ========================================================================= 1228;; == Extending arithmetic 1229;; ========================================================================= 1230 1231;; ------------------------------------------------------------------------- 1232;; ---- [INT] Wide binary arithmetic 1233;; ------------------------------------------------------------------------- 1234;; Includes: 1235;; - SADDWB 1236;; - SADDWT 1237;; - SSUBWB 1238;; - SSUBWT 1239;; - UADDWB 1240;; - UADDWT 1241;; - USUBWB 1242;; - USUBWT 1243;; ------------------------------------------------------------------------- 1244 1245(define_insn "@aarch64_sve_<sve_int_op><mode>" 1246 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") 1247 (unspec:SVE_FULL_HSDI 1248 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w") 1249 (match_operand:<VNARROW> 2 "register_operand" "w")] 1250 SVE2_INT_BINARY_WIDE))] 1251 "TARGET_SVE2" 1252 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Ventype>" 1253) 1254 1255;; ------------------------------------------------------------------------- 1256;; ---- [INT] Long binary arithmetic 1257;; ------------------------------------------------------------------------- 1258;; Includes: 1259;; - SABDLB 1260;; - SABDLT 1261;; - SADDLB 1262;; - SADDLBT 1263;; - SADDLT 1264;; - SMULLB 1265;; - SMULLT 1266;; - SQDMULLB 1267;; - SQDMULLT 1268;; - SSUBLB 1269;; - SSUBLBT 1270;; - SSUBLT 1271;; - SSUBLTB 1272;; - UABDLB 1273;; - UABDLT 1274;; - UADDLB 1275;; - UADDLT 1276;; - UMULLB 1277;; - UMULLT 1278;; - USUBLB 1279;; - USUBLT 1280;; ------------------------------------------------------------------------- 1281 1282(define_insn "@aarch64_sve_<sve_int_op><mode>" 1283 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") 1284 (unspec:SVE_FULL_HSDI 1285 [(match_operand:<VNARROW> 1 "register_operand" "w") 1286 (match_operand:<VNARROW> 2 "register_operand" "w")] 1287 SVE2_INT_BINARY_LONG))] 1288 "TARGET_SVE2" 1289 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>" 1290) 1291 1292(define_insn "@aarch64_sve_<sve_int_op>_lane_<mode>" 1293 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w") 1294 (unspec:SVE_FULL_SDI 1295 [(match_operand:<VNARROW> 1 "register_operand" "w") 1296 (unspec:<VNARROW> 1297 [(match_operand:<VNARROW> 2 "register_operand" "<sve_lane_con>") 1298 (match_operand:SI 3 "const_int_operand")] 1299 UNSPEC_SVE_LANE_SELECT)] 1300 SVE2_INT_BINARY_LONG_LANE))] 1301 "TARGET_SVE2" 1302 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3]" 1303) 1304 1305;; ------------------------------------------------------------------------- 1306;; ---- [INT] Long left shifts 1307;; ------------------------------------------------------------------------- 1308;; Includes: 1309;; - SSHLLB 1310;; - SSHLLT 1311;; - USHLLB 1312;; - USHLLT 1313;; ------------------------------------------------------------------------- 1314 1315;; The immediate range is enforced before generating the instruction. 1316(define_insn "@aarch64_sve_<sve_int_op><mode>" 1317 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w") 1318 (unspec:SVE_FULL_HSDI 1319 [(match_operand:<VNARROW> 1 "register_operand" "w") 1320 (match_operand:DI 2 "const_int_operand")] 1321 SVE2_INT_SHIFT_IMM_LONG))] 1322 "TARGET_SVE2" 1323 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, #%2" 1324) 1325 1326;; ------------------------------------------------------------------------- 1327;; ---- [INT] Long binary arithmetic with accumulation 1328;; ------------------------------------------------------------------------- 1329;; Includes: 1330;; - SABALB 1331;; - SABALT 1332;; - SMLALB 1333;; - SMLALT 1334;; - SMLSLB 1335;; - SMLSLT 1336;; - SQDMLALB 1337;; - SQDMLALBT 1338;; - SQDMLALT 1339;; - SQDMLSLB 1340;; - SQDMLSLBT 1341;; - SQDMLSLT 1342;; - UABALB 1343;; - UABALT 1344;; - UMLALB 1345;; - UMLALT 1346;; - UMLSLB 1347;; - UMLSLT 1348;; ------------------------------------------------------------------------- 1349 1350;; Non-saturating MLA operations. 1351(define_insn "@aarch64_sve_add_<sve_int_op><mode>" 1352 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 1353 (plus:SVE_FULL_HSDI 1354 (unspec:SVE_FULL_HSDI 1355 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1356 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1357 SVE2_INT_ADD_BINARY_LONG) 1358 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))] 1359 "TARGET_SVE2" 1360 "@ 1361 <sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype> 1362 movprfx\t%0, %1\;<sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>" 1363 [(set_attr "movprfx" "*,yes")] 1364) 1365 1366;; Non-saturating MLA operations with lane select. 1367(define_insn "@aarch64_sve_add_<sve_int_op>_lane_<mode>" 1368 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1369 (plus:SVE_FULL_SDI 1370 (unspec:SVE_FULL_SDI 1371 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1372 (unspec:<VNARROW> 1373 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1374 (match_operand:SI 4 "const_int_operand")] 1375 UNSPEC_SVE_LANE_SELECT)] 1376 SVE2_INT_ADD_BINARY_LONG_LANE) 1377 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")))] 1378 "TARGET_SVE2" 1379 "@ 1380 <sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4] 1381 movprfx\t%0, %1\;<sve_int_add_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]" 1382 [(set_attr "movprfx" "*,yes")] 1383) 1384 1385;; Saturating MLA operations. 1386(define_insn "@aarch64_sve_qadd_<sve_int_op><mode>" 1387 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 1388 (ss_plus:SVE_FULL_HSDI 1389 (unspec:SVE_FULL_HSDI 1390 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1391 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1392 SVE2_INT_QADD_BINARY_LONG) 1393 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w")))] 1394 "TARGET_SVE2" 1395 "@ 1396 <sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype> 1397 movprfx\t%0, %1\;<sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>" 1398 [(set_attr "movprfx" "*,yes")] 1399) 1400 1401;; Saturating MLA operations with lane select. 1402(define_insn "@aarch64_sve_qadd_<sve_int_op>_lane_<mode>" 1403 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1404 (ss_plus:SVE_FULL_SDI 1405 (unspec:SVE_FULL_SDI 1406 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1407 (unspec:<VNARROW> 1408 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1409 (match_operand:SI 4 "const_int_operand")] 1410 UNSPEC_SVE_LANE_SELECT)] 1411 SVE2_INT_QADD_BINARY_LONG_LANE) 1412 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w")))] 1413 "TARGET_SVE2" 1414 "@ 1415 <sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4] 1416 movprfx\t%0, %1\;<sve_int_qadd_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]" 1417 [(set_attr "movprfx" "*,yes")] 1418) 1419 1420;; Non-saturating MLS operations. 1421(define_insn "@aarch64_sve_sub_<sve_int_op><mode>" 1422 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 1423 (minus:SVE_FULL_HSDI 1424 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w") 1425 (unspec:SVE_FULL_HSDI 1426 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1427 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1428 SVE2_INT_SUB_BINARY_LONG)))] 1429 "TARGET_SVE2" 1430 "@ 1431 <sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype> 1432 movprfx\t%0, %1\;<sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>" 1433 [(set_attr "movprfx" "*,yes")] 1434) 1435 1436;; Non-saturating MLS operations with lane select. 1437(define_insn "@aarch64_sve_sub_<sve_int_op>_lane_<mode>" 1438 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1439 (minus:SVE_FULL_SDI 1440 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w") 1441 (unspec:SVE_FULL_SDI 1442 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1443 (unspec:<VNARROW> 1444 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1445 (match_operand:SI 4 "const_int_operand")] 1446 UNSPEC_SVE_LANE_SELECT)] 1447 SVE2_INT_SUB_BINARY_LONG_LANE)))] 1448 "TARGET_SVE2" 1449 "@ 1450 <sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4] 1451 movprfx\t%0, %1\;<sve_int_sub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]" 1452 [(set_attr "movprfx" "*,yes")] 1453) 1454 1455;; Saturating MLS operations. 1456(define_insn "@aarch64_sve_qsub_<sve_int_op><mode>" 1457 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 1458 (ss_minus:SVE_FULL_HSDI 1459 (match_operand:SVE_FULL_HSDI 1 "register_operand" "0, w") 1460 (unspec:SVE_FULL_HSDI 1461 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1462 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1463 SVE2_INT_QSUB_BINARY_LONG)))] 1464 "TARGET_SVE2" 1465 "@ 1466 <sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype> 1467 movprfx\t%0, %1\;<sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>" 1468 [(set_attr "movprfx" "*,yes")] 1469) 1470 1471;; Saturating MLS operations with lane select. 1472(define_insn "@aarch64_sve_qsub_<sve_int_op>_lane_<mode>" 1473 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1474 (ss_minus:SVE_FULL_SDI 1475 (match_operand:SVE_FULL_SDI 1 "register_operand" "0, w") 1476 (unspec:SVE_FULL_SDI 1477 [(match_operand:<VNARROW> 2 "register_operand" "w, w") 1478 (unspec:<VNARROW> 1479 [(match_operand:<VNARROW> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1480 (match_operand:SI 4 "const_int_operand")] 1481 UNSPEC_SVE_LANE_SELECT)] 1482 SVE2_INT_QSUB_BINARY_LONG_LANE)))] 1483 "TARGET_SVE2" 1484 "@ 1485 <sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4] 1486 movprfx\t%0, %1\;<sve_int_qsub_op>\t%0.<Vetype>, %2.<Ventype>, %3.<Ventype>[%4]" 1487 [(set_attr "movprfx" "*,yes")] 1488) 1489;; ------------------------------------------------------------------------- 1490;; ---- [FP] Long multiplication with accumulation 1491;; ------------------------------------------------------------------------- 1492;; Includes: 1493;; - FMLALB 1494;; - FMLALT 1495;; - FMLSLB 1496;; - FMLSLT 1497;; ------------------------------------------------------------------------- 1498 1499(define_insn "@aarch64_sve_<sve_fp_op><mode>" 1500 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w, ?&w") 1501 (unspec:VNx4SF_ONLY 1502 [(match_operand:<VNARROW> 1 "register_operand" "w, w") 1503 (match_operand:<VNARROW> 2 "register_operand" "w, w") 1504 (match_operand:VNx4SF_ONLY 3 "register_operand" "0, w")] 1505 SVE2_FP_TERNARY_LONG))] 1506 "TARGET_SVE2" 1507 "@ 1508 <sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype> 1509 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>" 1510 [(set_attr "movprfx" "*,yes")] 1511) 1512 1513(define_insn "@aarch64_<sve_fp_op>_lane_<mode>" 1514 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w, ?&w") 1515 (unspec:VNx4SF_ONLY 1516 [(match_operand:<VNARROW> 1 "register_operand" "w, w") 1517 (unspec:<VNARROW> 1518 [(match_operand:<VNARROW> 2 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1519 (match_operand:SI 3 "const_int_operand")] 1520 UNSPEC_SVE_LANE_SELECT) 1521 (match_operand:VNx4SF_ONLY 4 "register_operand" "0, w")] 1522 SVE2_FP_TERNARY_LONG_LANE))] 1523 "TARGET_SVE2" 1524 "@ 1525 <sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3] 1526 movprfx\t%0, %4\;<sve_fp_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>[%3]" 1527 [(set_attr "movprfx" "*,yes")] 1528) 1529 1530;; ========================================================================= 1531;; == Narrowing arithnetic 1532;; ========================================================================= 1533 1534;; ------------------------------------------------------------------------- 1535;; ---- [INT] Narrowing unary arithmetic 1536;; ------------------------------------------------------------------------- 1537;; Includes: 1538;; - SQXTNB 1539;; - SQXTNT 1540;; - SQXTUNB 1541;; - SQXTUNT 1542;; - UQXTNB 1543;; - UQXTNT 1544;; ------------------------------------------------------------------------- 1545 1546(define_insn "@aarch64_sve_<sve_int_op><mode>" 1547 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1548 (unspec:<VNARROW> 1549 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w")] 1550 SVE2_INT_UNARY_NARROWB))] 1551 "TARGET_SVE2" 1552 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>" 1553) 1554 1555;; These instructions do not take MOVPRFX. 1556(define_insn "@aarch64_sve_<sve_int_op><mode>" 1557 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1558 (unspec:<VNARROW> 1559 [(match_operand:<VNARROW> 1 "register_operand" "0") 1560 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")] 1561 SVE2_INT_UNARY_NARROWT))] 1562 "TARGET_SVE2" 1563 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>" 1564) 1565 1566;; ------------------------------------------------------------------------- 1567;; ---- [INT] Narrowing binary arithmetic 1568;; ------------------------------------------------------------------------- 1569;; Includes: 1570;; - ADDHNB 1571;; - ADDHNT 1572;; - RADDHNB 1573;; - RADDHNT 1574;; - RSUBHNB 1575;; - RSUBHNT 1576;; - SUBHNB 1577;; - SUBHNT 1578;; ------------------------------------------------------------------------- 1579 1580(define_insn "@aarch64_sve_<sve_int_op><mode>" 1581 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1582 (unspec:<VNARROW> 1583 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w") 1584 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w")] 1585 SVE2_INT_BINARY_NARROWB))] 1586 "TARGET_SVE2" 1587 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>, %2.<Vetype>" 1588) 1589 1590;; These instructions do not take MOVPRFX. 1591(define_insn "@aarch64_sve_<sve_int_op><mode>" 1592 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1593 (unspec:<VNARROW> 1594 [(match_operand:<VNARROW> 1 "register_operand" "0") 1595 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w") 1596 (match_operand:SVE_FULL_HSDI 3 "register_operand" "w")] 1597 SVE2_INT_BINARY_NARROWT))] 1598 "TARGET_SVE2" 1599 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>, %3.<Vetype>" 1600) 1601 1602;; ------------------------------------------------------------------------- 1603;; ---- [INT] Narrowing right shifts 1604;; ------------------------------------------------------------------------- 1605;; Includes: 1606;; - RSHRNB 1607;; - RSHRNT 1608;; - SHRNB 1609;; - SHRNT 1610;; - SQRSHRNB 1611;; - SQRSHRNT 1612;; - SQRSHRUNB 1613;; - SQRSHRUNT 1614;; - SQSHRNB 1615;; - SQSHRNT 1616;; - SQSHRUNB 1617;; - SQSHRUNT 1618;; - UQRSHRNB 1619;; - UQRSHRNT 1620;; - UQSHRNB 1621;; - UQSHRNT 1622;; ------------------------------------------------------------------------- 1623 1624;; The immediate range is enforced before generating the instruction. 1625(define_insn "@aarch64_sve_<sve_int_op><mode>" 1626 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1627 (unspec:<VNARROW> 1628 [(match_operand:SVE_FULL_HSDI 1 "register_operand" "w") 1629 (match_operand:DI 2 "const_int_operand")] 1630 SVE2_INT_SHIFT_IMM_NARROWB))] 1631 "TARGET_SVE2" 1632 "<sve_int_op>\t%0.<Ventype>, %1.<Vetype>, #%2" 1633) 1634 1635;; The immediate range is enforced before generating the instruction. 1636;; These instructions do not take MOVPRFX. 1637(define_insn "@aarch64_sve_<sve_int_op><mode>" 1638 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 1639 (unspec:<VNARROW> 1640 [(match_operand:<VNARROW> 1 "register_operand" "0") 1641 (match_operand:SVE_FULL_HSDI 2 "register_operand" "w") 1642 (match_operand:DI 3 "const_int_operand")] 1643 SVE2_INT_SHIFT_IMM_NARROWT))] 1644 "TARGET_SVE2" 1645 "<sve_int_op>\t%0.<Ventype>, %2.<Vetype>, #%3" 1646) 1647 1648;; ========================================================================= 1649;; == Pairwise arithmetic 1650;; ========================================================================= 1651 1652;; ------------------------------------------------------------------------- 1653;; ---- [INT] Pairwise arithmetic 1654;; ------------------------------------------------------------------------- 1655;; Includes: 1656;; - ADDP 1657;; - SMAXP 1658;; - SMINP 1659;; - UMAXP 1660;; - UMINP 1661;; ------------------------------------------------------------------------- 1662 1663(define_insn "@aarch64_pred_<sve_int_op><mode>" 1664 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1665 (unspec:SVE_FULL_I 1666 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1667 (match_operand:SVE_FULL_I 2 "register_operand" "0, w") 1668 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")] 1669 SVE2_INT_BINARY_PAIR))] 1670 "TARGET_SVE2" 1671 "@ 1672 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1673 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1674 [(set_attr "movprfx" "*,yes")] 1675) 1676 1677;; ------------------------------------------------------------------------- 1678;; ---- [FP] Pairwise arithmetic 1679;; ------------------------------------------------------------------------- 1680;; Includes: 1681;; - FADDP 1682;; - FMAXP 1683;; - FMAXNMP 1684;; - FMINP 1685;; - FMINNMP 1686;; ------------------------------------------------------------------------- 1687 1688(define_insn "@aarch64_pred_<sve_fp_op><mode>" 1689 [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w, ?&w") 1690 (unspec:SVE_FULL_F 1691 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1692 (match_operand:SVE_FULL_F 2 "register_operand" "0, w") 1693 (match_operand:SVE_FULL_F 3 "register_operand" "w, w")] 1694 SVE2_FP_BINARY_PAIR))] 1695 "TARGET_SVE2" 1696 "@ 1697 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1698 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1699 [(set_attr "movprfx" "*,yes")] 1700) 1701 1702;; ------------------------------------------------------------------------- 1703;; ---- [INT] Pairwise arithmetic with accumulation 1704;; ------------------------------------------------------------------------- 1705;; Includes: 1706;; - SADALP 1707;; - UADALP 1708;; ------------------------------------------------------------------------- 1709 1710;; Predicated pairwise absolute difference and accumulate with merging. 1711(define_expand "@cond_<sve_int_op><mode>" 1712 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand") 1713 (unspec:SVE_FULL_HSDI 1714 [(match_operand:<VPRED> 1 "register_operand") 1715 (unspec:SVE_FULL_HSDI 1716 [(match_dup 1) 1717 (match_operand:SVE_FULL_HSDI 2 "register_operand") 1718 (match_operand:<VNARROW> 3 "register_operand")] 1719 SVE2_INT_BINARY_PAIR_LONG) 1720 (match_operand:SVE_FULL_HSDI 4 "aarch64_simd_reg_or_zero")] 1721 UNSPEC_SEL))] 1722 "TARGET_SVE2" 1723{ 1724 /* Only target code is aware of these operations, so we don't need 1725 to handle the fully-general case. */ 1726 gcc_assert (rtx_equal_p (operands[2], operands[4]) 1727 || CONSTANT_P (operands[4])); 1728}) 1729 1730;; Predicated pairwise absolute difference and accumulate, merging with 1731;; the first input. 1732(define_insn_and_rewrite "*cond_<sve_int_op><mode>_2" 1733 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=w, ?&w") 1734 (unspec:SVE_FULL_HSDI 1735 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1736 (unspec:SVE_FULL_HSDI 1737 [(match_operand 4) 1738 (match_operand:SVE_FULL_HSDI 2 "register_operand" "0, w") 1739 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1740 SVE2_INT_BINARY_PAIR_LONG) 1741 (match_dup 2)] 1742 UNSPEC_SEL))] 1743 "TARGET_SVE2" 1744 "@ 1745 <sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype> 1746 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>" 1747 "&& !CONSTANT_P (operands[4])" 1748 { 1749 operands[4] = CONSTM1_RTX (<VPRED>mode); 1750 } 1751 [(set_attr "movprfx" "*,yes")] 1752) 1753 1754;; Predicated pairwise absolute difference and accumulate, merging with zero. 1755(define_insn_and_rewrite "*cond_<sve_int_op><mode>_z" 1756 [(set (match_operand:SVE_FULL_HSDI 0 "register_operand" "=&w, &w") 1757 (unspec:SVE_FULL_HSDI 1758 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1759 (unspec:SVE_FULL_HSDI 1760 [(match_operand 5) 1761 (match_operand:SVE_FULL_HSDI 2 "register_operand" "0, w") 1762 (match_operand:<VNARROW> 3 "register_operand" "w, w")] 1763 SVE2_INT_BINARY_PAIR_LONG) 1764 (match_operand:SVE_FULL_HSDI 4 "aarch64_simd_imm_zero")] 1765 UNSPEC_SEL))] 1766 "TARGET_SVE2" 1767 "@ 1768 movprfx\t%0.<Vetype>, %1/z, %0.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype> 1769 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %3.<Ventype>" 1770 "&& !CONSTANT_P (operands[5])" 1771 { 1772 operands[5] = CONSTM1_RTX (<VPRED>mode); 1773 } 1774 [(set_attr "movprfx" "yes")] 1775) 1776 1777;; ========================================================================= 1778;; == Complex arithmetic 1779;; ========================================================================= 1780 1781;; ------------------------------------------------------------------------- 1782;; ---- [INT] Complex binary operations 1783;; ------------------------------------------------------------------------- 1784;; Includes: 1785;; - CADD 1786;; - SQCADD 1787;; ------------------------------------------------------------------------- 1788 1789(define_insn "@aarch64_sve_<optab><mode>" 1790 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1791 (unspec:SVE_FULL_I 1792 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w") 1793 (match_operand:SVE_FULL_I 2 "register_operand" "w, w")] 1794 SVE2_INT_CADD))] 1795 "TARGET_SVE2" 1796 "@ 1797 <sve_int_op>\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #<rot> 1798 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %0.<Vetype>, %2.<Vetype>, #<rot>" 1799 [(set_attr "movprfx" "*,yes")] 1800) 1801 1802;; ------------------------------------------------------------------------- 1803;; ---- [INT] Complex ternary operations 1804;; ------------------------------------------------------------------------- 1805;; Includes: 1806;; - CMLA 1807;; - SQRDCMLA 1808;; ------------------------------------------------------------------------- 1809 1810(define_insn "@aarch64_sve_<optab><mode>" 1811 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w, ?&w") 1812 (unspec:SVE_FULL_I 1813 [(match_operand:SVE_FULL_I 1 "register_operand" "0, w") 1814 (match_operand:SVE_FULL_I 2 "register_operand" "w, w") 1815 (match_operand:SVE_FULL_I 3 "register_operand" "w, w")] 1816 SVE2_INT_CMLA))] 1817 "TARGET_SVE2" 1818 "@ 1819 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>, #<rot> 1820 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>, #<rot>" 1821 [(set_attr "movprfx" "*,yes")] 1822) 1823 1824(define_insn "@aarch64_<optab>_lane_<mode>" 1825 [(set (match_operand:SVE_FULL_HSI 0 "register_operand" "=w, ?&w") 1826 (unspec:SVE_FULL_HSI 1827 [(match_operand:SVE_FULL_HSI 1 "register_operand" "0, w") 1828 (match_operand:SVE_FULL_HSI 2 "register_operand" "w, w") 1829 (unspec:SVE_FULL_HSI 1830 [(match_operand:SVE_FULL_HSI 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1831 (match_operand:SI 4 "const_int_operand")] 1832 UNSPEC_SVE_LANE_SELECT)] 1833 SVE2_INT_CMLA))] 1834 "TARGET_SVE2" 1835 "@ 1836 <sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4], #<rot> 1837 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>[%4], #<rot>" 1838 [(set_attr "movprfx" "*,yes")] 1839) 1840 1841;; ------------------------------------------------------------------------- 1842;; ---- [INT] Complex dot product 1843;; ------------------------------------------------------------------------- 1844;; Includes: 1845;; - CDOT 1846;; ------------------------------------------------------------------------- 1847 1848(define_insn "@aarch64_sve_<optab><mode>" 1849 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1850 (unspec:SVE_FULL_SDI 1851 [(match_operand:SVE_FULL_SDI 1 "register_operand" "0, w") 1852 (match_operand:<VSI2QI> 2 "register_operand" "w, w") 1853 (match_operand:<VSI2QI> 3 "register_operand" "w, w")] 1854 SVE2_INT_CDOT))] 1855 "TARGET_SVE2" 1856 "@ 1857 <sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>, #<rot> 1858 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>, #<rot>" 1859 [(set_attr "movprfx" "*,yes")] 1860) 1861 1862(define_insn "@aarch64_<optab>_lane_<mode>" 1863 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w, ?&w") 1864 (unspec:SVE_FULL_SDI 1865 [(match_operand:SVE_FULL_SDI 1 "register_operand" "0, w") 1866 (match_operand:<VSI2QI> 2 "register_operand" "w, w") 1867 (unspec:<VSI2QI> 1868 [(match_operand:<VSI2QI> 3 "register_operand" "<sve_lane_con>, <sve_lane_con>") 1869 (match_operand:SI 4 "const_int_operand")] 1870 UNSPEC_SVE_LANE_SELECT)] 1871 SVE2_INT_CDOT))] 1872 "TARGET_SVE2" 1873 "@ 1874 <sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>[%4], #<rot> 1875 movprfx\t%0, %1\;<sve_int_op>\t%0.<Vetype>, %2.<Vetype_fourth>, %3.<Vetype_fourth>[%4], #<rot>" 1876 [(set_attr "movprfx" "*,yes")] 1877) 1878 1879;; ========================================================================= 1880;; == Conversions 1881;; ========================================================================= 1882 1883;; ------------------------------------------------------------------------- 1884;; ---- [FP<-FP] Widening conversions 1885;; ------------------------------------------------------------------------- 1886;; Includes: 1887;; - FCVTLT 1888;; ------------------------------------------------------------------------- 1889 1890;; Predicated convert long top. 1891(define_insn "@aarch64_pred_<sve_fp_op><mode>" 1892 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w") 1893 (unspec:SVE_FULL_SDF 1894 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1895 (match_operand:SI 3 "aarch64_sve_gp_strictness") 1896 (match_operand:<VNARROW> 2 "register_operand" "0")] 1897 SVE2_COND_FP_UNARY_LONG))] 1898 "TARGET_SVE2" 1899 "<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Ventype>" 1900) 1901 1902;; Predicated convert long top with merging. 1903(define_expand "@cond_<sve_fp_op><mode>" 1904 [(set (match_operand:SVE_FULL_SDF 0 "register_operand") 1905 (unspec:SVE_FULL_SDF 1906 [(match_operand:<VPRED> 1 "register_operand") 1907 (unspec:SVE_FULL_SDF 1908 [(match_dup 1) 1909 (const_int SVE_STRICT_GP) 1910 (match_operand:<VNARROW> 2 "register_operand")] 1911 SVE2_COND_FP_UNARY_LONG) 1912 (match_operand:SVE_FULL_SDF 3 "register_operand")] 1913 UNSPEC_SEL))] 1914 "TARGET_SVE2" 1915) 1916 1917;; These instructions do not take MOVPRFX. 1918(define_insn_and_rewrite "*cond_<sve_fp_op><mode>_relaxed" 1919 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w") 1920 (unspec:SVE_FULL_SDF 1921 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1922 (unspec:SVE_FULL_SDF 1923 [(match_operand 4) 1924 (const_int SVE_RELAXED_GP) 1925 (match_operand:<VNARROW> 2 "register_operand" "w")] 1926 SVE2_COND_FP_UNARY_LONG) 1927 (match_operand:SVE_FULL_SDF 3 "register_operand" "0")] 1928 UNSPEC_SEL))] 1929 "TARGET_SVE2" 1930 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Ventype>" 1931 "&& !rtx_equal_p (operands[1], operands[4])" 1932 { 1933 operands[4] = copy_rtx (operands[1]); 1934 } 1935) 1936 1937(define_insn "*cond_<sve_fp_op><mode>_strict" 1938 [(set (match_operand:SVE_FULL_SDF 0 "register_operand" "=w") 1939 (unspec:SVE_FULL_SDF 1940 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1941 (unspec:SVE_FULL_SDF 1942 [(match_dup 1) 1943 (const_int SVE_STRICT_GP) 1944 (match_operand:<VNARROW> 2 "register_operand" "w")] 1945 SVE2_COND_FP_UNARY_LONG) 1946 (match_operand:SVE_FULL_SDF 3 "register_operand" "0")] 1947 UNSPEC_SEL))] 1948 "TARGET_SVE2" 1949 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Ventype>" 1950) 1951 1952;; ------------------------------------------------------------------------- 1953;; ---- [FP<-FP] Narrowing conversions 1954;; ------------------------------------------------------------------------- 1955;; Includes: 1956;; - FCVTNT 1957;; - FCVTX 1958;; - FCVTXNT 1959;; ------------------------------------------------------------------------- 1960 1961;; Predicated FCVTNT. This doesn't give a natural aarch64_pred_*/cond_* 1962;; pair because the even elements always have to be supplied for active 1963;; elements, even if the inactive elements don't matter. 1964;; 1965;; These instructions do not take MOVPRFX. 1966(define_insn "@aarch64_sve_cvtnt<mode>" 1967 [(set (match_operand:SVE_FULL_HSF 0 "register_operand" "=w") 1968 (unspec:SVE_FULL_HSF 1969 [(match_operand:<VWIDE_PRED> 2 "register_operand" "Upl") 1970 (const_int SVE_STRICT_GP) 1971 (match_operand:SVE_FULL_HSF 1 "register_operand" "0") 1972 (match_operand:<VWIDE> 3 "register_operand" "w")] 1973 UNSPEC_COND_FCVTNT))] 1974 "TARGET_SVE2" 1975 "fcvtnt\t%0.<Vetype>, %2/m, %3.<Vewtype>" 1976) 1977 1978;; Predicated FCVTX (equivalent to what would be FCVTXNB, except that 1979;; it supports MOVPRFX). 1980(define_insn "@aarch64_pred_<sve_fp_op><mode>" 1981 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=w, ?&w") 1982 (unspec:VNx4SF_ONLY 1983 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl, Upl") 1984 (match_operand:SI 3 "aarch64_sve_gp_strictness") 1985 (match_operand:<VWIDE> 2 "register_operand" "0, w")] 1986 SVE2_COND_FP_UNARY_NARROWB))] 1987 "TARGET_SVE2" 1988 "@ 1989 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype> 1990 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>" 1991 [(set_attr "movprfx" "*,yes")] 1992) 1993 1994;; Predicated FCVTX with merging. 1995(define_expand "@cond_<sve_fp_op><mode>" 1996 [(set (match_operand:VNx4SF_ONLY 0 "register_operand") 1997 (unspec:VNx4SF_ONLY 1998 [(match_operand:<VWIDE_PRED> 1 "register_operand") 1999 (unspec:VNx4SF_ONLY 2000 [(match_dup 1) 2001 (const_int SVE_STRICT_GP) 2002 (match_operand:<VWIDE> 2 "register_operand")] 2003 SVE2_COND_FP_UNARY_NARROWB) 2004 (match_operand:VNx4SF_ONLY 3 "aarch64_simd_reg_or_zero")] 2005 UNSPEC_SEL))] 2006 "TARGET_SVE2" 2007) 2008 2009(define_insn_and_rewrite "*cond_<sve_fp_op><mode>_any_relaxed" 2010 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=&w, &w, &w") 2011 (unspec:VNx4SF_ONLY 2012 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl, Upl, Upl") 2013 (unspec:VNx4SF_ONLY 2014 [(match_operand 4) 2015 (const_int SVE_RELAXED_GP) 2016 (match_operand:<VWIDE> 2 "register_operand" "w, w, w")] 2017 SVE2_COND_FP_UNARY_NARROWB) 2018 (match_operand:VNx4SF_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")] 2019 UNSPEC_SEL))] 2020 "TARGET_SVE2 && !rtx_equal_p (operands[2], operands[3])" 2021 "@ 2022 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype> 2023 movprfx\t%0.<Vewtype>, %1/z, %2.<Vewtype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype> 2024 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>" 2025 "&& !rtx_equal_p (operands[1], operands[4])" 2026 { 2027 operands[4] = copy_rtx (operands[1]); 2028 } 2029 [(set_attr "movprfx" "*,yes,yes")] 2030) 2031 2032(define_insn "*cond_<sve_fp_op><mode>_any_strict" 2033 [(set (match_operand:VNx4SF_ONLY 0 "register_operand" "=&w, &w, &w") 2034 (unspec:VNx4SF_ONLY 2035 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl, Upl, Upl") 2036 (unspec:VNx4SF_ONLY 2037 [(match_dup 1) 2038 (const_int SVE_STRICT_GP) 2039 (match_operand:<VWIDE> 2 "register_operand" "w, w, w")] 2040 SVE2_COND_FP_UNARY_NARROWB) 2041 (match_operand:VNx4SF_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")] 2042 UNSPEC_SEL))] 2043 "TARGET_SVE2 && !rtx_equal_p (operands[2], operands[3])" 2044 "@ 2045 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype> 2046 movprfx\t%0.<Vewtype>, %1/z, %2.<Vewtype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype> 2047 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vewtype>" 2048 [(set_attr "movprfx" "*,yes,yes")] 2049) 2050 2051;; Predicated FCVTXNT. This doesn't give a natural aarch64_pred_*/cond_* 2052;; pair because the even elements always have to be supplied for active 2053;; elements, even if the inactive elements don't matter. 2054;; 2055;; These instructions do not take MOVPRFX. 2056(define_insn "@aarch64_sve2_cvtxnt<mode>" 2057 [(set (match_operand:<VNARROW> 0 "register_operand" "=w") 2058 (unspec:<VNARROW> 2059 [(match_operand:<VPRED> 2 "register_operand" "Upl") 2060 (const_int SVE_STRICT_GP) 2061 (match_operand:<VNARROW> 1 "register_operand" "0") 2062 (match_operand:VNx2DF_ONLY 3 "register_operand" "w")] 2063 UNSPEC_COND_FCVTXNT))] 2064 "TARGET_SVE2" 2065 "fcvtxnt\t%0.<Ventype>, %2/m, %3.<Vetype>" 2066) 2067 2068;; ========================================================================= 2069;; == Other arithmetic 2070;; ========================================================================= 2071 2072;; ------------------------------------------------------------------------- 2073;; ---- [INT] Reciprocal approximation 2074;; ------------------------------------------------------------------------- 2075;; Includes: 2076;; - URECPE 2077;; - URSQRTE 2078;; ------------------------------------------------------------------------- 2079 2080;; Predicated integer unary operations. 2081(define_insn "@aarch64_pred_<sve_int_op><mode>" 2082 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=w, ?&w") 2083 (unspec:VNx4SI_ONLY 2084 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2085 (unspec:VNx4SI_ONLY 2086 [(match_operand:VNx4SI_ONLY 2 "register_operand" "0, w")] 2087 SVE2_U32_UNARY)] 2088 UNSPEC_PRED_X))] 2089 "TARGET_SVE2" 2090 "@ 2091 <sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2092 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2093 [(set_attr "movprfx" "*,yes")] 2094) 2095 2096;; Predicated integer unary operations with merging. 2097(define_expand "@cond_<sve_int_op><mode>" 2098 [(set (match_operand:VNx4SI_ONLY 0 "register_operand") 2099 (unspec:VNx4SI_ONLY 2100 [(match_operand:<VPRED> 1 "register_operand") 2101 (unspec:VNx4SI_ONLY 2102 [(match_dup 4) 2103 (unspec:VNx4SI_ONLY 2104 [(match_operand:VNx4SI_ONLY 2 "register_operand")] 2105 SVE2_U32_UNARY)] 2106 UNSPEC_PRED_X) 2107 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero")] 2108 UNSPEC_SEL))] 2109 "TARGET_SVE2" 2110 { 2111 operands[4] = CONSTM1_RTX (<MODE>mode); 2112 } 2113) 2114 2115(define_insn_and_rewrite "*cond_<sve_int_op><mode>" 2116 [(set (match_operand:VNx4SI_ONLY 0 "register_operand" "=w, ?&w, ?&w") 2117 (unspec:VNx4SI_ONLY 2118 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2119 (unspec:VNx4SI_ONLY 2120 [(match_operand 4) 2121 (unspec:VNx4SI_ONLY 2122 [(match_operand:VNx4SI_ONLY 2 "register_operand" "w, w, w")] 2123 SVE2_U32_UNARY)] 2124 UNSPEC_PRED_X) 2125 (match_operand:VNx4SI_ONLY 3 "aarch64_simd_reg_or_zero" "0, Dz, w")] 2126 UNSPEC_SEL))] 2127 "TARGET_SVE2" 2128 "@ 2129 <sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2130 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2131 movprfx\t%0, %3\;<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2132 "&& !CONSTANT_P (operands[4])" 2133 { 2134 operands[4] = CONSTM1_RTX (<VPRED>mode); 2135 } 2136 [(set_attr "movprfx" "*,yes,yes")] 2137) 2138 2139;; ------------------------------------------------------------------------- 2140;; ---- [INT<-FP] Base-2 logarithm 2141;; ------------------------------------------------------------------------- 2142;; Includes: 2143;; - FLOGB 2144;; ------------------------------------------------------------------------- 2145 2146;; Predicated FLOGB. 2147(define_insn "@aarch64_pred_<sve_fp_op><mode>" 2148 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w, ?&w") 2149 (unspec:<V_INT_EQUIV> 2150 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2151 (match_operand:SI 3 "aarch64_sve_gp_strictness") 2152 (match_operand:SVE_FULL_F 2 "register_operand" "0, w")] 2153 SVE2_COND_INT_UNARY_FP))] 2154 "TARGET_SVE2" 2155 "@ 2156 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2157 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2158 [(set_attr "movprfx" "*,yes")] 2159) 2160 2161;; Predicated FLOGB with merging. 2162(define_expand "@cond_<sve_fp_op><mode>" 2163 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand") 2164 (unspec:<V_INT_EQUIV> 2165 [(match_operand:<VPRED> 1 "register_operand") 2166 (unspec:<V_INT_EQUIV> 2167 [(match_dup 1) 2168 (const_int SVE_STRICT_GP) 2169 (match_operand:SVE_FULL_F 2 "register_operand")] 2170 SVE2_COND_INT_UNARY_FP) 2171 (match_operand:<V_INT_EQUIV> 3 "aarch64_simd_reg_or_zero")] 2172 UNSPEC_SEL))] 2173 "TARGET_SVE2" 2174) 2175 2176(define_insn_and_rewrite "*cond_<sve_fp_op><mode>" 2177 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=&w, ?&w, ?&w") 2178 (unspec:<V_INT_EQUIV> 2179 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2180 (unspec:<V_INT_EQUIV> 2181 [(match_operand 4) 2182 (const_int SVE_RELAXED_GP) 2183 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")] 2184 SVE2_COND_INT_UNARY_FP) 2185 (match_operand:<V_INT_EQUIV> 3 "aarch64_simd_reg_or_zero" "0, Dz, w")] 2186 UNSPEC_SEL))] 2187 "TARGET_SVE2 && !rtx_equal_p (operands[2], operands[3])" 2188 "@ 2189 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2190 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2191 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2192 "&& !rtx_equal_p (operands[1], operands[4])" 2193 { 2194 operands[4] = copy_rtx (operands[1]); 2195 } 2196 [(set_attr "movprfx" "*,yes,yes")] 2197) 2198 2199(define_insn "*cond_<sve_fp_op><mode>_strict" 2200 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=&w, ?&w, ?&w") 2201 (unspec:<V_INT_EQUIV> 2202 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2203 (unspec:<V_INT_EQUIV> 2204 [(match_dup 1) 2205 (const_int SVE_STRICT_GP) 2206 (match_operand:SVE_FULL_F 2 "register_operand" "w, w, w")] 2207 SVE2_COND_INT_UNARY_FP) 2208 (match_operand:<V_INT_EQUIV> 3 "aarch64_simd_reg_or_zero" "0, Dz, w")] 2209 UNSPEC_SEL))] 2210 "TARGET_SVE2 && !rtx_equal_p (operands[2], operands[3])" 2211 "@ 2212 <sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2213 movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype> 2214 movprfx\t%0, %3\;<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2215 [(set_attr "movprfx" "*,yes,yes")] 2216) 2217 2218;; ------------------------------------------------------------------------- 2219;; ---- [INT] Polynomial multiplication 2220;; ------------------------------------------------------------------------- 2221;; Includes: 2222;; - PMUL 2223;; - PMULLB 2224;; - PMULLT 2225;; ------------------------------------------------------------------------- 2226 2227;; Uniform PMUL. 2228(define_insn "@aarch64_sve2_pmul<mode>" 2229 [(set (match_operand:VNx16QI_ONLY 0 "register_operand" "=w") 2230 (unspec:VNx16QI_ONLY 2231 [(match_operand:VNx16QI_ONLY 1 "register_operand" "w") 2232 (match_operand:VNx16QI_ONLY 2 "register_operand" "w")] 2233 UNSPEC_PMUL))] 2234 "TARGET_SVE2" 2235 "pmul\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 2236) 2237 2238;; Extending PMUL, with the results modeled as wider vectors. 2239;; This representation is only possible for .H and .D, not .Q. 2240(define_insn "@aarch64_sve_<optab><mode>" 2241 [(set (match_operand:SVE_FULL_HDI 0 "register_operand" "=w") 2242 (unspec:SVE_FULL_HDI 2243 [(match_operand:<VNARROW> 1 "register_operand" "w") 2244 (match_operand:<VNARROW> 2 "register_operand" "w")] 2245 SVE2_PMULL))] 2246 "TARGET_SVE2" 2247 "<sve_int_op>\t%0.<Vetype>, %1.<Ventype>, %2.<Ventype>" 2248) 2249 2250;; Extending PMUL, with the results modeled as pairs of values. 2251;; This representation works for .H, .D and .Q, with .Q requiring 2252;; the AES extension. (This is enforced by the mode iterator.) 2253(define_insn "@aarch64_sve_<optab><mode>" 2254 [(set (match_operand:SVE2_PMULL_PAIR_I 0 "register_operand" "=w") 2255 (unspec:SVE2_PMULL_PAIR_I 2256 [(match_operand:SVE2_PMULL_PAIR_I 1 "register_operand" "w") 2257 (match_operand:SVE2_PMULL_PAIR_I 2 "register_operand" "w")] 2258 SVE2_PMULL_PAIR))] 2259 "TARGET_SVE2" 2260 "<sve_int_op>\t%0.<Vewtype>, %1.<Vetype>, %2.<Vetype>" 2261) 2262 2263;; ========================================================================= 2264;; == Permutation 2265;; ========================================================================= 2266 2267;; ------------------------------------------------------------------------- 2268;; ---- [INT,FP] General permutes 2269;; ------------------------------------------------------------------------- 2270;; Includes: 2271;; - TBL (vector pair form) 2272;; - TBX 2273;; ------------------------------------------------------------------------- 2274 2275;; TBL on a pair of data vectors. 2276(define_insn "@aarch64_sve2_tbl2<mode>" 2277 [(set (match_operand:SVE_FULL 0 "register_operand" "=w") 2278 (unspec:SVE_FULL 2279 [(match_operand:<VDOUBLE> 1 "register_operand" "w") 2280 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")] 2281 UNSPEC_TBL2))] 2282 "TARGET_SVE2" 2283 "tbl\t%0.<Vetype>, %1, %2.<Vetype>" 2284) 2285 2286;; TBX. These instructions do not take MOVPRFX. 2287(define_insn "@aarch64_sve2_tbx<mode>" 2288 [(set (match_operand:SVE_FULL 0 "register_operand" "=w") 2289 (unspec:SVE_FULL 2290 [(match_operand:SVE_FULL 1 "register_operand" "0") 2291 (match_operand:SVE_FULL 2 "register_operand" "w") 2292 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w")] 2293 UNSPEC_TBX))] 2294 "TARGET_SVE2" 2295 "tbx\t%0.<Vetype>, %2.<Vetype>, %3.<Vetype>" 2296) 2297 2298;; ------------------------------------------------------------------------- 2299;; ---- [INT] Optional bit-permute extensions 2300;; ------------------------------------------------------------------------- 2301;; Includes: 2302;; - BDEP 2303;; - BEXT 2304;; - BGRP 2305;; ------------------------------------------------------------------------- 2306 2307(define_insn "@aarch64_sve_<sve_int_op><mode>" 2308 [(set (match_operand:SVE_FULL_I 0 "register_operand" "=w") 2309 (unspec:SVE_FULL_I 2310 [(match_operand:SVE_FULL_I 1 "register_operand" "w") 2311 (match_operand:SVE_FULL_I 2 "register_operand" "w")] 2312 SVE2_INT_BITPERM))] 2313 "TARGET_SVE2_BITPERM" 2314 "<sve_int_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 2315) 2316 2317;; ========================================================================= 2318;; == General 2319;; ========================================================================= 2320 2321;; ------------------------------------------------------------------------- 2322;; ---- Check for aliases between pointers 2323;; ------------------------------------------------------------------------- 2324;; The patterns in this section are synthetic: WHILERW and WHILEWR are 2325;; defined in aarch64-sve.md instead. 2326;; ------------------------------------------------------------------------- 2327 2328;; Use WHILERW and WHILEWR to accelerate alias checks. This is only 2329;; possible if the accesses we're checking are exactly the same size 2330;; as an SVE vector. 2331(define_expand "check_<raw_war>_ptrs<mode>" 2332 [(match_operand:GPI 0 "register_operand") 2333 (unspec:VNx16BI 2334 [(match_operand:GPI 1 "register_operand") 2335 (match_operand:GPI 2 "register_operand") 2336 (match_operand:GPI 3 "aarch64_bytes_per_sve_vector_operand") 2337 (match_operand:GPI 4 "const_int_operand")] 2338 SVE2_WHILE_PTR)] 2339 "TARGET_SVE2" 2340{ 2341 /* Use the widest predicate mode we can. */ 2342 unsigned int align = INTVAL (operands[4]); 2343 if (align > 8) 2344 align = 8; 2345 machine_mode pred_mode = aarch64_sve_pred_mode (align).require (); 2346 2347 /* Emit a WHILERW or WHILEWR, setting the condition codes based on 2348 the result. */ 2349 emit_insn (gen_while_ptest 2350 (<SVE2_WHILE_PTR:unspec>, <MODE>mode, pred_mode, 2351 gen_rtx_SCRATCH (pred_mode), operands[1], operands[2], 2352 CONSTM1_RTX (VNx16BImode), CONSTM1_RTX (pred_mode))); 2353 2354 /* Set operand 0 to true if the last bit of the predicate result is set, 2355 i.e. if all elements are free of dependencies. */ 2356 rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM); 2357 rtx cmp = gen_rtx_LTU (<MODE>mode, cc_reg, const0_rtx); 2358 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp, cc_reg)); 2359 DONE; 2360}) 2361 2362;; ------------------------------------------------------------------------- 2363;; ---- Histogram processing 2364;; ------------------------------------------------------------------------- 2365;; Includes: 2366;; - HISTCNT 2367;; - HISTSEG 2368;; ------------------------------------------------------------------------- 2369 2370(define_insn "@aarch64_sve2_histcnt<mode>" 2371 [(set (match_operand:SVE_FULL_SDI 0 "register_operand" "=w") 2372 (unspec:SVE_FULL_SDI 2373 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2374 (match_operand:SVE_FULL_SDI 2 "register_operand" "w") 2375 (match_operand:SVE_FULL_SDI 3 "register_operand" "w")] 2376 UNSPEC_HISTCNT))] 2377 "TARGET_SVE2" 2378 "histcnt\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 2379) 2380 2381(define_insn "@aarch64_sve2_histseg<mode>" 2382 [(set (match_operand:VNx16QI_ONLY 0 "register_operand" "=w") 2383 (unspec:VNx16QI_ONLY 2384 [(match_operand:VNx16QI_ONLY 1 "register_operand" "w") 2385 (match_operand:VNx16QI_ONLY 2 "register_operand" "w")] 2386 UNSPEC_HISTSEG))] 2387 "TARGET_SVE2" 2388 "histseg\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 2389) 2390 2391;; ------------------------------------------------------------------------- 2392;; ---- String matching 2393;; ------------------------------------------------------------------------- 2394;; Includes: 2395;; - MATCH 2396;; - NMATCH 2397;; ------------------------------------------------------------------------- 2398 2399;; Predicated string matching. 2400(define_insn "@aarch64_pred_<sve_int_op><mode>" 2401 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa") 2402 (unspec:<VPRED> 2403 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2404 (match_operand:SI 2 "aarch64_sve_ptrue_flag") 2405 (unspec:<VPRED> 2406 [(match_operand:SVE_FULL_BHI 3 "register_operand" "w") 2407 (match_operand:SVE_FULL_BHI 4 "register_operand" "w")] 2408 SVE2_MATCH)] 2409 UNSPEC_PRED_Z)) 2410 (clobber (reg:CC_NZC CC_REGNUM))] 2411 "TARGET_SVE2" 2412 "<sve_int_op>\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>" 2413) 2414 2415;; Predicated string matching in which both the flag and predicate results 2416;; are interesting. 2417(define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_cc" 2418 [(set (reg:CC_NZC CC_REGNUM) 2419 (unspec:CC_NZC 2420 [(match_operand:VNx16BI 1 "register_operand" "Upl") 2421 (match_operand 4) 2422 (match_operand:SI 5 "aarch64_sve_ptrue_flag") 2423 (unspec:<VPRED> 2424 [(match_operand 6) 2425 (match_operand:SI 7 "aarch64_sve_ptrue_flag") 2426 (unspec:<VPRED> 2427 [(match_operand:SVE_FULL_BHI 2 "register_operand" "w") 2428 (match_operand:SVE_FULL_BHI 3 "register_operand" "w")] 2429 SVE2_MATCH)] 2430 UNSPEC_PRED_Z)] 2431 UNSPEC_PTEST)) 2432 (set (match_operand:<VPRED> 0 "register_operand" "=Upa") 2433 (unspec:<VPRED> 2434 [(match_dup 6) 2435 (match_dup 7) 2436 (unspec:<VPRED> 2437 [(match_dup 2) 2438 (match_dup 3)] 2439 SVE2_MATCH)] 2440 UNSPEC_PRED_Z))] 2441 "TARGET_SVE2 2442 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])" 2443 "<sve_int_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 2444 "&& !rtx_equal_p (operands[4], operands[6])" 2445 { 2446 operands[6] = copy_rtx (operands[4]); 2447 operands[7] = operands[5]; 2448 } 2449) 2450 2451;; Predicated string matching in which only the flags result is interesting. 2452(define_insn_and_rewrite "*aarch64_pred_<sve_int_op><mode>_ptest" 2453 [(set (reg:CC_NZC CC_REGNUM) 2454 (unspec:CC_NZC 2455 [(match_operand:VNx16BI 1 "register_operand" "Upl") 2456 (match_operand 4) 2457 (match_operand:SI 5 "aarch64_sve_ptrue_flag") 2458 (unspec:<VPRED> 2459 [(match_operand 6) 2460 (match_operand:SI 7 "aarch64_sve_ptrue_flag") 2461 (unspec:<VPRED> 2462 [(match_operand:SVE_FULL_BHI 2 "register_operand" "w") 2463 (match_operand:SVE_FULL_BHI 3 "register_operand" "w")] 2464 SVE2_MATCH)] 2465 UNSPEC_PRED_Z)] 2466 UNSPEC_PTEST)) 2467 (clobber (match_scratch:<VPRED> 0 "=Upa"))] 2468 "TARGET_SVE2 2469 && aarch64_sve_same_pred_for_ptest_p (&operands[4], &operands[6])" 2470 "<sve_int_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 2471 "&& !rtx_equal_p (operands[4], operands[6])" 2472 { 2473 operands[6] = copy_rtx (operands[4]); 2474 operands[7] = operands[5]; 2475 } 2476) 2477 2478;; ========================================================================= 2479;; == Crypotographic extensions 2480;; ========================================================================= 2481 2482;; ------------------------------------------------------------------------- 2483;; ---- Optional AES extensions 2484;; ------------------------------------------------------------------------- 2485;; Includes: 2486;; - AESD 2487;; - AESE 2488;; - AESIMC 2489;; - AESMC 2490;; ------------------------------------------------------------------------- 2491 2492;; AESD and AESE. 2493(define_insn "aarch64_sve2_aes<aes_op>" 2494 [(set (match_operand:VNx16QI 0 "register_operand" "=w") 2495 (unspec:VNx16QI 2496 [(xor:VNx16QI 2497 (match_operand:VNx16QI 1 "register_operand" "%0") 2498 (match_operand:VNx16QI 2 "register_operand" "w"))] 2499 CRYPTO_AES))] 2500 "TARGET_SVE2_AES" 2501 "aes<aes_op>\t%0.b, %0.b, %2.b" 2502 [(set_attr "type" "crypto_aese")] 2503) 2504 2505;; AESMC and AESIMC. These instructions do not take MOVPRFX. 2506(define_insn "aarch64_sve2_aes<aesmc_op>" 2507 [(set (match_operand:VNx16QI 0 "register_operand" "=w") 2508 (unspec:VNx16QI 2509 [(match_operand:VNx16QI 1 "register_operand" "0")] 2510 CRYPTO_AESMC))] 2511 "TARGET_SVE2_AES" 2512 "aes<aesmc_op>\t%0.b, %0.b" 2513 [(set_attr "type" "crypto_aesmc")] 2514) 2515 2516;; When AESE/AESMC and AESD/AESIMC fusion is enabled, we really want 2517;; to keep the two together and enforce the register dependency without 2518;; scheduling or register allocation messing up the order or introducing 2519;; moves inbetween. Mash the two together during combine. 2520 2521(define_insn "*aarch64_sve2_aese_fused" 2522 [(set (match_operand:VNx16QI 0 "register_operand" "=w") 2523 (unspec:VNx16QI 2524 [(unspec:VNx16QI 2525 [(xor:VNx16QI 2526 (match_operand:VNx16QI 1 "register_operand" "%0") 2527 (match_operand:VNx16QI 2 "register_operand" "w"))] 2528 UNSPEC_AESE)] 2529 UNSPEC_AESMC))] 2530 "TARGET_SVE2_AES && aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)" 2531 "aese\t%0.b, %0.b, %2.b\;aesmc\t%0.b, %0.b" 2532 [(set_attr "type" "crypto_aese") 2533 (set_attr "length" "8")] 2534) 2535 2536(define_insn "*aarch64_sve2_aesd_fused" 2537 [(set (match_operand:VNx16QI 0 "register_operand" "=w") 2538 (unspec:VNx16QI 2539 [(unspec:VNx16QI 2540 [(xor:VNx16QI 2541 (match_operand:VNx16QI 1 "register_operand" "%0") 2542 (match_operand:VNx16QI 2 "register_operand" "w"))] 2543 UNSPEC_AESD)] 2544 UNSPEC_AESIMC))] 2545 "TARGET_SVE2_AES && aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)" 2546 "aesd\t%0.b, %0.b, %2.b\;aesimc\t%0.b, %0.b" 2547 [(set_attr "type" "crypto_aese") 2548 (set_attr "length" "8")] 2549) 2550 2551;; ------------------------------------------------------------------------- 2552;; ---- Optional SHA-3 extensions 2553;; ------------------------------------------------------------------------- 2554;; Includes: 2555;; - RAX1 2556;; ------------------------------------------------------------------------- 2557 2558(define_insn "aarch64_sve2_rax1" 2559 [(set (match_operand:VNx2DI 0 "register_operand" "=w") 2560 (xor:VNx2DI 2561 (rotate:VNx2DI 2562 (match_operand:VNx2DI 2 "register_operand" "w") 2563 (const_int 1)) 2564 (match_operand:VNx2DI 1 "register_operand" "w")))] 2565 "TARGET_SVE2_SHA3" 2566 "rax1\t%0.d, %1.d, %2.d" 2567 [(set_attr "type" "crypto_sha3")] 2568) 2569 2570;; ------------------------------------------------------------------------- 2571;; ---- Optional SM4 extensions 2572;; ------------------------------------------------------------------------- 2573;; Includes: 2574;; - SM4E 2575;; - SM4EKEY 2576;; ------------------------------------------------------------------------- 2577 2578;; These instructions do not take MOVPRFX. 2579(define_insn "aarch64_sve2_sm4e" 2580 [(set (match_operand:VNx4SI 0 "register_operand" "=w") 2581 (unspec:VNx4SI 2582 [(match_operand:VNx4SI 1 "register_operand" "0") 2583 (match_operand:VNx4SI 2 "register_operand" "w")] 2584 UNSPEC_SM4E))] 2585 "TARGET_SVE2_SM4" 2586 "sm4e\t%0.s, %0.s, %2.s" 2587 [(set_attr "type" "crypto_sm4")] 2588) 2589 2590(define_insn "aarch64_sve2_sm4ekey" 2591 [(set (match_operand:VNx4SI 0 "register_operand" "=w") 2592 (unspec:VNx4SI 2593 [(match_operand:VNx4SI 1 "register_operand" "w") 2594 (match_operand:VNx4SI 2 "register_operand" "w")] 2595 UNSPEC_SM4EKEY))] 2596 "TARGET_SVE2_SM4" 2597 "sm4ekey\t%0.s, %1.s, %2.s" 2598 [(set_attr "type" "crypto_sm4")] 2599) 2600