1;; Machine description for AArch64 architecture. 2;; Copyright (C) 2009-2020 Free Software Foundation, Inc. 3;; Contributed by ARM Ltd. 4;; 5;; This file is part of GCC. 6;; 7;; GCC is free software; you can redistribute it and/or modify it 8;; under the terms of the GNU General Public License as published by 9;; the Free Software Foundation; either version 3, or (at your option) 10;; any later version. 11;; 12;; GCC is distributed in the hope that it will be useful, but 13;; WITHOUT ANY WARRANTY; without even the implied warranty of 14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15;; General Public License for more details. 16;; 17;; You should have received a copy of the GNU General Public License 18;; along with GCC; see the file COPYING3. If not see 19;; <http://www.gnu.org/licenses/>. 20 21;; Register numbers 22(define_constants 23 [ 24 (R0_REGNUM 0) 25 (R1_REGNUM 1) 26 (R2_REGNUM 2) 27 (R3_REGNUM 3) 28 (R4_REGNUM 4) 29 (R5_REGNUM 5) 30 (R6_REGNUM 6) 31 (R7_REGNUM 7) 32 (R8_REGNUM 8) 33 (R9_REGNUM 9) 34 (R10_REGNUM 10) 35 (R11_REGNUM 11) 36 (R12_REGNUM 12) 37 (R13_REGNUM 13) 38 (R14_REGNUM 14) 39 (R15_REGNUM 15) 40 (R16_REGNUM 16) 41 (R17_REGNUM 17) 42 (R18_REGNUM 18) 43 (R19_REGNUM 19) 44 (R20_REGNUM 20) 45 (R21_REGNUM 21) 46 (R22_REGNUM 22) 47 (R23_REGNUM 23) 48 (R24_REGNUM 24) 49 (R25_REGNUM 25) 50 (R26_REGNUM 26) 51 (R27_REGNUM 27) 52 (R28_REGNUM 28) 53 (R29_REGNUM 29) 54 (R30_REGNUM 30) 55 (SP_REGNUM 31) 56 (V0_REGNUM 32) 57 (V1_REGNUM 33) 58 (V2_REGNUM 34) 59 (V3_REGNUM 35) 60 (V4_REGNUM 36) 61 (V5_REGNUM 37) 62 (V6_REGNUM 38) 63 (V7_REGNUM 39) 64 (V8_REGNUM 40) 65 (V9_REGNUM 41) 66 (V10_REGNUM 42) 67 (V11_REGNUM 43) 68 (V12_REGNUM 44) 69 (V13_REGNUM 45) 70 (V14_REGNUM 46) 71 (V15_REGNUM 47) 72 (V16_REGNUM 48) 73 (V17_REGNUM 49) 74 (V18_REGNUM 50) 75 (V19_REGNUM 51) 76 (V20_REGNUM 52) 77 (V21_REGNUM 53) 78 (V22_REGNUM 54) 79 (V23_REGNUM 55) 80 (V24_REGNUM 56) 81 (V25_REGNUM 57) 82 (V26_REGNUM 58) 83 (V27_REGNUM 59) 84 (V28_REGNUM 60) 85 (V29_REGNUM 61) 86 (V30_REGNUM 62) 87 (V31_REGNUM 63) 88 (SFP_REGNUM 64) 89 (AP_REGNUM 65) 90 (CC_REGNUM 66) 91 ;; Defined only to make the DWARF description simpler. 92 (VG_REGNUM 67) 93 (P0_REGNUM 68) 94 (P1_REGNUM 69) 95 (P2_REGNUM 70) 96 (P3_REGNUM 71) 97 (P4_REGNUM 72) 98 (P5_REGNUM 73) 99 (P6_REGNUM 74) 100 (P7_REGNUM 75) 101 (P8_REGNUM 76) 102 (P9_REGNUM 77) 103 (P10_REGNUM 78) 104 (P11_REGNUM 79) 105 (P12_REGNUM 80) 106 (P13_REGNUM 81) 107 (P14_REGNUM 82) 108 (P15_REGNUM 83) 109 (LAST_SAVED_REGNUM 83) 110 (FFR_REGNUM 84) 111 ;; "FFR token": a fake register used for representing the scheduling 112 ;; restrictions on FFR-related operations. 113 (FFRT_REGNUM 85) 114 ;; Scratch register used by stack clash protection to calculate 115 ;; SVE CFA offsets during probing. 116 (STACK_CLASH_SVE_CFA_REGNUM 11) 117 ;; Scratch registers for prologue/epilogue use. 118 (EP0_REGNUM 12) 119 (EP1_REGNUM 13) 120 ;; A couple of call-clobbered registers that we need to reserve when 121 ;; tracking speculation this is not ABI, so is subject to change. 122 (SPECULATION_SCRATCH_REGNUM 14) 123 (SPECULATION_TRACKER_REGNUM 15) 124 ;; Scratch registers used in frame layout. 125 (IP0_REGNUM 16) 126 (IP1_REGNUM 17) 127 (FP_REGNUM 29) 128 (LR_REGNUM 30) 129 ] 130) 131 132(define_c_enum "unspec" [ 133 UNSPEC_AUTIA1716 134 UNSPEC_AUTIB1716 135 UNSPEC_AUTIASP 136 UNSPEC_AUTIBSP 137 UNSPEC_CALLEE_ABI 138 UNSPEC_CASESI 139 UNSPEC_CRC32B 140 UNSPEC_CRC32CB 141 UNSPEC_CRC32CH 142 UNSPEC_CRC32CW 143 UNSPEC_CRC32CX 144 UNSPEC_CRC32H 145 UNSPEC_CRC32W 146 UNSPEC_CRC32X 147 UNSPEC_FCVTZS 148 UNSPEC_FCVTZU 149 UNSPEC_FJCVTZS 150 UNSPEC_FRINT32Z 151 UNSPEC_FRINT32X 152 UNSPEC_FRINT64Z 153 UNSPEC_FRINT64X 154 UNSPEC_URECPE 155 UNSPEC_FRECPE 156 UNSPEC_FRECPS 157 UNSPEC_FRECPX 158 UNSPEC_FRINTA 159 UNSPEC_FRINTI 160 UNSPEC_FRINTM 161 UNSPEC_FRINTN 162 UNSPEC_FRINTP 163 UNSPEC_FRINTX 164 UNSPEC_FRINTZ 165 UNSPEC_GOTSMALLPIC 166 UNSPEC_GOTSMALLPIC28K 167 UNSPEC_GOTSMALLTLS 168 UNSPEC_GOTTINYPIC 169 UNSPEC_GOTTINYTLS 170 UNSPEC_LD1 171 UNSPEC_LD2 172 UNSPEC_LD2_DREG 173 UNSPEC_LD2_DUP 174 UNSPEC_LD3 175 UNSPEC_LD3_DREG 176 UNSPEC_LD3_DUP 177 UNSPEC_LD4 178 UNSPEC_LD4_DREG 179 UNSPEC_LD4_DUP 180 UNSPEC_LD2_LANE 181 UNSPEC_LD3_LANE 182 UNSPEC_LD4_LANE 183 UNSPEC_MB 184 UNSPEC_NOP 185 UNSPEC_PACIA1716 186 UNSPEC_PACIB1716 187 UNSPEC_PACIASP 188 UNSPEC_PACIBSP 189 UNSPEC_PRLG_STK 190 UNSPEC_REV 191 UNSPEC_RBIT 192 UNSPEC_SABAL 193 UNSPEC_SABDL2 194 UNSPEC_SADALP 195 UNSPEC_SCVTF 196 UNSPEC_SISD_NEG 197 UNSPEC_SISD_SSHL 198 UNSPEC_SISD_USHL 199 UNSPEC_SSHL_2S 200 UNSPEC_ST1 201 UNSPEC_ST2 202 UNSPEC_ST3 203 UNSPEC_ST4 204 UNSPEC_ST2_LANE 205 UNSPEC_ST3_LANE 206 UNSPEC_ST4_LANE 207 UNSPEC_TLS 208 UNSPEC_TLSDESC 209 UNSPEC_TLSLE12 210 UNSPEC_TLSLE24 211 UNSPEC_TLSLE32 212 UNSPEC_TLSLE48 213 UNSPEC_UABAL 214 UNSPEC_UABDL2 215 UNSPEC_UADALP 216 UNSPEC_UCVTF 217 UNSPEC_USHL_2S 218 UNSPEC_VSTRUCTDUMMY 219 UNSPEC_SSP_SYSREG 220 UNSPEC_SP_SET 221 UNSPEC_SP_TEST 222 UNSPEC_RSQRT 223 UNSPEC_RSQRTE 224 UNSPEC_RSQRTS 225 UNSPEC_NZCV 226 UNSPEC_XPACLRI 227 UNSPEC_LD1_SVE 228 UNSPEC_ST1_SVE 229 UNSPEC_LDNT1_SVE 230 UNSPEC_STNT1_SVE 231 UNSPEC_LD1RQ 232 UNSPEC_LD1_GATHER 233 UNSPEC_LDFF1_GATHER 234 UNSPEC_LDNT1_GATHER 235 UNSPEC_ST1_SCATTER 236 UNSPEC_STNT1_SCATTER 237 UNSPEC_PRED_X 238 UNSPEC_PRED_Z 239 UNSPEC_PTEST 240 UNSPEC_PTRUE 241 UNSPEC_UNPACKSHI 242 UNSPEC_UNPACKUHI 243 UNSPEC_UNPACKSLO 244 UNSPEC_UNPACKULO 245 UNSPEC_PACK 246 UNSPEC_WHILEGE 247 UNSPEC_WHILEGT 248 UNSPEC_WHILEHI 249 UNSPEC_WHILEHS 250 UNSPEC_WHILELE 251 UNSPEC_WHILELO 252 UNSPEC_WHILELS 253 UNSPEC_WHILELT 254 UNSPEC_WHILERW 255 UNSPEC_WHILEWR 256 UNSPEC_LDN 257 UNSPEC_STN 258 UNSPEC_INSR 259 UNSPEC_CLASTA 260 UNSPEC_CLASTB 261 UNSPEC_FADDA 262 UNSPEC_REV_SUBREG 263 UNSPEC_REINTERPRET 264 UNSPEC_SPECULATION_TRACKER 265 UNSPEC_SPECULATION_TRACKER_REV 266 UNSPEC_COPYSIGN 267 UNSPEC_TTEST ; Represent transaction test. 268 UNSPEC_UPDATE_FFR 269 UNSPEC_UPDATE_FFRT 270 UNSPEC_RDFFR 271 UNSPEC_WRFFR 272 ;; Represents an SVE-style lane index, in which the indexing applies 273 ;; within the containing 128-bit block. 274 UNSPEC_SVE_LANE_SELECT 275 UNSPEC_SVE_CNT_PAT 276 UNSPEC_SVE_PREFETCH 277 UNSPEC_SVE_PREFETCH_GATHER 278 UNSPEC_SVE_COMPACT 279 UNSPEC_SVE_SPLICE 280 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag. 281 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag. 282 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space. 283 UNSPEC_LD1RO 284 UNSPEC_SALT_ADDR 285 UNSPECV_PATCHABLE_AREA 286]) 287 288(define_c_enum "unspecv" [ 289 UNSPECV_EH_RETURN ; Represent EH_RETURN 290 UNSPECV_GET_FPCR ; Represent fetch of FPCR content. 291 UNSPECV_SET_FPCR ; Represent assign of FPCR content. 292 UNSPECV_GET_FPSR ; Represent fetch of FPSR content. 293 UNSPECV_SET_FPSR ; Represent assign of FPSR content. 294 UNSPECV_BLOCKAGE ; Represent a blockage 295 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing. 296 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier. 297 UNSPECV_BTI_NOARG ; Represent BTI. 298 UNSPECV_BTI_C ; Represent BTI c. 299 UNSPECV_BTI_J ; Represent BTI j. 300 UNSPECV_BTI_JC ; Represent BTI jc. 301 UNSPECV_TSTART ; Represent transaction start. 302 UNSPECV_TCOMMIT ; Represent transaction commit. 303 UNSPECV_TCANCEL ; Represent transaction cancel. 304 UNSPEC_RNDR ; Represent RNDR 305 UNSPEC_RNDRRS ; Represent RNDRRS 306 ] 307) 308 309;; These constants are used as a const_int in various SVE unspecs 310;; to indicate whether the governing predicate is known to be a PTRUE. 311(define_constants 312 [; Indicates that the predicate might not be a PTRUE. 313 (SVE_MAYBE_NOT_PTRUE 0) 314 315 ; Indicates that the predicate is known to be a PTRUE. 316 (SVE_KNOWN_PTRUE 1)]) 317 318;; These constants are used as a const_int in predicated SVE FP arithmetic 319;; to indicate whether the operation is allowed to make additional lanes 320;; active without worrying about the effect on faulting behavior. 321(define_constants 322 [; Indicates either that all lanes are active or that the instruction may 323 ; operate on inactive inputs even if doing so could induce a fault. 324 (SVE_RELAXED_GP 0) 325 326 ; Indicates that some lanes might be inactive and that the instruction 327 ; must not operate on inactive inputs if doing so could induce a fault. 328 (SVE_STRICT_GP 1)]) 329 330;; If further include files are added the defintion of MD_INCLUDES 331;; must be updated. 332 333(include "constraints.md") 334(include "predicates.md") 335(include "iterators.md") 336 337;; ------------------------------------------------------------------- 338;; Instruction types and attributes 339;; ------------------------------------------------------------------- 340 341; The "type" attribute is included here from AArch32 backend to be able 342; to share pipeline descriptions. 343(include "../arm/types.md") 344 345;; It is important to set the fp or simd attributes to yes when a pattern 346;; alternative uses the FP or SIMD register files, usually signified by use of 347;; the 'w' constraint. This will ensure that the alternative will be 348;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd 349;; architecture extensions. If all the alternatives in a pattern use the 350;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT 351;; or TARGET_SIMD. 352 353;; Attributes of the architecture required to support the instruction (or 354;; alternative). This attribute is used to compute attribute "enabled", use type 355;; "any" to enable an alternative in all cases. 356 357(define_enum "arches" [ any rcpc8_4 fp simd sve fp16]) 358 359(define_enum_attr "arch" "arches" (const_string "any")) 360 361;; [For compatibility with Arm in pipeline models] 362;; Attribute that specifies whether or not the instruction touches fp 363;; registers. 364;; Note that this attribute is not used anywhere in either the arm or aarch64 365;; backends except in the scheduling description for xgene1. In that 366;; scheduling description this attribute is used to subclass the load_4 and 367;; load_8 types. 368(define_attr "fp" "no,yes" 369 (if_then_else 370 (eq_attr "arch" "fp") 371 (const_string "yes") 372 (const_string "no"))) 373 374(define_attr "arch_enabled" "no,yes" 375 (if_then_else 376 (ior 377 (eq_attr "arch" "any") 378 379 (and (eq_attr "arch" "rcpc8_4") 380 (match_test "AARCH64_ISA_RCPC8_4")) 381 382 (and (eq_attr "arch" "fp") 383 (match_test "TARGET_FLOAT")) 384 385 (and (eq_attr "arch" "simd") 386 (match_test "TARGET_SIMD")) 387 388 (and (eq_attr "arch" "fp16") 389 (match_test "TARGET_FP_F16INST")) 390 391 (and (eq_attr "arch" "sve") 392 (match_test "TARGET_SVE"))) 393 (const_string "yes") 394 (const_string "no"))) 395 396;; Attribute that controls whether an alternative is enabled or not. 397;; Currently it is only used to disable alternatives which touch fp or simd 398;; registers when -mgeneral-regs-only is specified or to require a special 399;; architecture support. 400(define_attr "enabled" "no,yes" (attr "arch_enabled")) 401 402;; Attribute that specifies whether we are dealing with a branch to a 403;; label that is far away, i.e. further away than the maximum/minimum 404;; representable in a signed 21-bits number. 405;; 0 :=: no 406;; 1 :=: yes 407(define_attr "far_branch" "" (const_int 0)) 408 409;; Attribute that specifies whether the alternative uses MOVPRFX. 410(define_attr "movprfx" "no,yes" (const_string "no")) 411 412;; Attribute to specify that an alternative has the length of a single 413;; instruction plus a speculation barrier. 414(define_attr "sls_length" "none,retbr,casesi" (const_string "none")) 415 416(define_attr "length" "" 417 (cond [(eq_attr "movprfx" "yes") 418 (const_int 8) 419 420 (eq_attr "sls_length" "retbr") 421 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4) 422 (match_test "TARGET_SB") (const_int 8)] 423 (const_int 12)) 424 425 (eq_attr "sls_length" "casesi") 426 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16) 427 (match_test "TARGET_SB") (const_int 20)] 428 (const_int 24)) 429 ] 430 (const_int 4))) 431 432;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has 433;; no predicated insns. 434(define_attr "predicated" "yes,no" (const_string "no")) 435 436;; Set to true on an insn that requires the speculation tracking state to be 437;; in the tracking register before the insn issues. Otherwise the compiler 438;; may chose to hold the tracking state encoded in SP. 439(define_attr "speculation_barrier" "true,false" (const_string "false")) 440 441;; ------------------------------------------------------------------- 442;; Pipeline descriptions and scheduling 443;; ------------------------------------------------------------------- 444 445;; Processor types. 446(include "aarch64-tune.md") 447 448;; Scheduling 449(include "../arm/cortex-a53.md") 450(include "../arm/cortex-a57.md") 451(include "../arm/exynos-m1.md") 452(include "falkor.md") 453(include "saphira.md") 454(include "thunderx.md") 455(include "../arm/xgene1.md") 456(include "thunderx2t99.md") 457(include "tsv110.md") 458(include "thunderx3t110.md") 459 460;; ------------------------------------------------------------------- 461;; Jumps and other miscellaneous insns 462;; ------------------------------------------------------------------- 463 464(define_insn "indirect_jump" 465 [(set (pc) (match_operand:DI 0 "register_operand" "r"))] 466 "" 467 { 468 output_asm_insn ("br\\t%0", operands); 469 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 470 } 471 [(set_attr "type" "branch") 472 (set_attr "sls_length" "retbr")] 473) 474 475(define_insn "jump" 476 [(set (pc) (label_ref (match_operand 0 "" "")))] 477 "" 478 "b\\t%l0" 479 [(set_attr "type" "branch")] 480) 481 482(define_expand "cbranch<mode>4" 483 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 484 [(match_operand:GPI 1 "register_operand") 485 (match_operand:GPI 2 "aarch64_plus_operand")]) 486 (label_ref (match_operand 3 "" "")) 487 (pc)))] 488 "" 489 " 490 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1], 491 operands[2]); 492 operands[2] = const0_rtx; 493 " 494) 495 496(define_expand "cbranch<mode>4" 497 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 498 [(match_operand:GPF 1 "register_operand") 499 (match_operand:GPF 2 "aarch64_fp_compare_operand")]) 500 (label_ref (match_operand 3 "" "")) 501 (pc)))] 502 "" 503 " 504 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1], 505 operands[2]); 506 operands[2] = const0_rtx; 507 " 508) 509 510(define_expand "cbranchcc4" 511 [(set (pc) (if_then_else 512 (match_operator 0 "aarch64_comparison_operator" 513 [(match_operand 1 "cc_register") 514 (match_operand 2 "const0_operand")]) 515 (label_ref (match_operand 3 "" "")) 516 (pc)))] 517 "" 518 "") 519 520(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>" 521 [(set (match_operand:CC_ONLY 1 "cc_register" "") 522 (if_then_else:CC_ONLY 523 (match_operator 4 "aarch64_comparison_operator" 524 [(match_operand 0 "cc_register" "") 525 (const_int 0)]) 526 (compare:CC_ONLY 527 (match_operand:GPI 2 "register_operand" "r,r,r") 528 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")) 529 (unspec:CC_ONLY 530 [(match_operand 5 "immediate_operand")] 531 UNSPEC_NZCV)))] 532 "" 533 "@ 534 ccmp\\t%<w>2, %<w>3, %k5, %m4 535 ccmp\\t%<w>2, %3, %k5, %m4 536 ccmn\\t%<w>2, #%n3, %k5, %m4" 537 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 538) 539 540(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>" 541 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "") 542 (if_then_else:CCFP_CCFPE 543 (match_operator 4 "aarch64_comparison_operator" 544 [(match_operand 0 "cc_register" "") 545 (const_int 0)]) 546 (compare:CCFP_CCFPE 547 (match_operand:GPF 2 "register_operand" "w") 548 (match_operand:GPF 3 "register_operand" "w")) 549 (unspec:CCFP_CCFPE 550 [(match_operand 5 "immediate_operand")] 551 UNSPEC_NZCV)))] 552 "TARGET_FLOAT" 553 "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4" 554 [(set_attr "type" "fccmp<s>")] 555) 556 557(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev" 558 [(set (match_operand:CC_ONLY 1 "cc_register" "") 559 (if_then_else:CC_ONLY 560 (match_operator 4 "aarch64_comparison_operator" 561 [(match_operand 0 "cc_register" "") 562 (const_int 0)]) 563 (unspec:CC_ONLY 564 [(match_operand 5 "immediate_operand")] 565 UNSPEC_NZCV) 566 (compare:CC_ONLY 567 (match_operand:GPI 2 "register_operand" "r,r,r") 568 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))))] 569 "" 570 "@ 571 ccmp\\t%<w>2, %<w>3, %k5, %M4 572 ccmp\\t%<w>2, %3, %k5, %M4 573 ccmn\\t%<w>2, #%n3, %k5, %M4" 574 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 575) 576 577(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev" 578 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "") 579 (if_then_else:CCFP_CCFPE 580 (match_operator 4 "aarch64_comparison_operator" 581 [(match_operand 0 "cc_register" "") 582 (const_int 0)]) 583 (unspec:CCFP_CCFPE 584 [(match_operand 5 "immediate_operand")] 585 UNSPEC_NZCV) 586 (compare:CCFP_CCFPE 587 (match_operand:GPF 2 "register_operand" "w") 588 (match_operand:GPF 3 "register_operand" "w"))))] 589 "TARGET_FLOAT" 590 "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4" 591 [(set_attr "type" "fccmp<s>")] 592) 593 594;; Expansion of signed mod by a power of 2 using CSNEG. 595;; For x0 % n where n is a power of 2 produce: 596;; negs x1, x0 597;; and x0, x0, #(n - 1) 598;; and x1, x1, #(n - 1) 599;; csneg x0, x0, x1, mi 600 601(define_expand "mod<mode>3" 602 [(match_operand:GPI 0 "register_operand") 603 (match_operand:GPI 1 "register_operand") 604 (match_operand:GPI 2 "const_int_operand")] 605 "" 606 { 607 HOST_WIDE_INT val = INTVAL (operands[2]); 608 609 if (val <= 0 610 || exact_log2 (val) <= 0 611 || !aarch64_bitmask_imm (val - 1, <MODE>mode)) 612 FAIL; 613 614 rtx mask = GEN_INT (val - 1); 615 616 /* In the special case of x0 % 2 we can do the even shorter: 617 cmp x0, xzr 618 and x0, x0, 1 619 cneg x0, x0, lt. */ 620 if (val == 2) 621 { 622 rtx masked = gen_reg_rtx (<MODE>mode); 623 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx); 624 emit_insn (gen_and<mode>3 (masked, operands[1], mask)); 625 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx); 626 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked)); 627 DONE; 628 } 629 630 rtx neg_op = gen_reg_rtx (<MODE>mode); 631 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1])); 632 633 /* Extract the condition register and mode. */ 634 rtx cmp = XVECEXP (PATTERN (insn), 0, 0); 635 rtx cc_reg = SET_DEST (cmp); 636 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx); 637 638 rtx masked_pos = gen_reg_rtx (<MODE>mode); 639 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask)); 640 641 rtx masked_neg = gen_reg_rtx (<MODE>mode); 642 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask)); 643 644 emit_insn (gen_csneg3<mode>_insn (operands[0], cond, 645 masked_neg, masked_pos)); 646 DONE; 647 } 648) 649 650(define_insn "condjump" 651 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 652 [(match_operand 1 "cc_register" "") (const_int 0)]) 653 (label_ref (match_operand 2 "" "")) 654 (pc)))] 655 "" 656 { 657 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc., 658 but the "." is required for SVE conditions. */ 659 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode; 660 if (get_attr_length (insn) == 8) 661 return aarch64_gen_far_branch (operands, 2, "Lbcond", 662 use_dot_p ? "b.%M0\\t" : "b%M0\\t"); 663 else 664 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2"; 665 } 666 [(set_attr "type" "branch") 667 (set (attr "length") 668 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 669 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 670 (const_int 4) 671 (const_int 8))) 672 (set (attr "far_branch") 673 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 674 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 675 (const_int 0) 676 (const_int 1)))] 677) 678 679;; For a 24-bit immediate CST we can optimize the compare for equality 680;; and branch sequence from: 681;; mov x0, #imm1 682;; movk x0, #imm2, lsl 16 /* x0 contains CST. */ 683;; cmp x1, x0 684;; b<ne,eq> .Label 685;; into the shorter: 686;; sub x0, x1, #(CST & 0xfff000) 687;; subs x0, x0, #(CST & 0x000fff) 688;; b<ne,eq> .Label 689(define_insn_and_split "*compare_condjump<GPI:mode>" 690 [(set (pc) (if_then_else (EQL 691 (match_operand:GPI 0 "register_operand" "r") 692 (match_operand:GPI 1 "aarch64_imm24" "n")) 693 (label_ref:P (match_operand 2 "" "")) 694 (pc)))] 695 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode) 696 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode) 697 && !reload_completed" 698 "#" 699 "&& true" 700 [(const_int 0)] 701 { 702 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff; 703 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000; 704 rtx tmp = gen_reg_rtx (<GPI:MODE>mode); 705 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm))); 706 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm))); 707 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM); 708 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode, 709 cc_reg, const0_rtx); 710 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2])); 711 DONE; 712 } 713) 714 715(define_expand "casesi" 716 [(match_operand:SI 0 "register_operand") ; Index 717 (match_operand:SI 1 "const_int_operand") ; Lower bound 718 (match_operand:SI 2 "const_int_operand") ; Total range 719 (match_operand:DI 3 "" "") ; Table label 720 (match_operand:DI 4 "" "")] ; Out of range label 721 "" 722 { 723 if (operands[1] != const0_rtx) 724 { 725 rtx reg = gen_reg_rtx (SImode); 726 727 /* Canonical RTL says that if you have: 728 729 (minus (X) (CONST)) 730 731 then this should be emitted as: 732 733 (plus (X) (-CONST)) 734 735 The use of trunc_int_for_mode ensures that the resulting 736 constant can be represented in SImode, this is important 737 for the corner case where operand[1] is INT_MIN. */ 738 739 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode)); 740 741 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate) 742 (operands[1], SImode)) 743 operands[1] = force_reg (SImode, operands[1]); 744 emit_insn (gen_addsi3 (reg, operands[0], operands[1])); 745 operands[0] = reg; 746 } 747 748 if (!aarch64_plus_operand (operands[2], SImode)) 749 operands[2] = force_reg (SImode, operands[2]); 750 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx, 751 const0_rtx), 752 operands[0], operands[2], operands[4])); 753 754 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3])); 755 operands[2] 756 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]), 757 UNSPEC_CASESI); 758 operands[2] = gen_rtx_MEM (DImode, operands[2]); 759 MEM_READONLY_P (operands[2]) = 1; 760 MEM_NOTRAP_P (operands[2]) = 1; 761 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3])); 762 DONE; 763 } 764) 765 766(define_expand "casesi_dispatch" 767 [(parallel 768 [(set (pc) (match_operand:DI 0 "")) 769 (clobber (reg:CC CC_REGNUM)) 770 (clobber (match_scratch:DI 2)) 771 (clobber (match_scratch:DI 3)) 772 (use (label_ref:DI (match_operand 1 "")))])] 773 "") 774 775(define_insn "*casesi_dispatch" 776 [(parallel 777 [(set (pc) 778 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r") 779 (match_operand:SI 1 "register_operand" "r")] 780 UNSPEC_CASESI))) 781 (clobber (reg:CC CC_REGNUM)) 782 (clobber (match_scratch:DI 3 "=r")) 783 (clobber (match_scratch:DI 4 "=r")) 784 (use (label_ref:DI (match_operand 2 "" "")))])] 785 "" 786 "* 787 return aarch64_output_casesi (operands); 788 " 789 [(set_attr "sls_length" "casesi") 790 (set_attr "type" "branch")] 791) 792 793(define_insn "nop" 794 [(unspec[(const_int 0)] UNSPEC_NOP)] 795 "" 796 "nop" 797 [(set_attr "type" "no_insn")] 798) 799 800(define_insn "prefetch" 801 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp") 802 (match_operand:QI 1 "const_int_operand" "") 803 (match_operand:QI 2 "const_int_operand" ""))] 804 "" 805 { 806 const char * pftype[2][4] = 807 { 808 {"prfm\\tPLDL1STRM, %0", 809 "prfm\\tPLDL3KEEP, %0", 810 "prfm\\tPLDL2KEEP, %0", 811 "prfm\\tPLDL1KEEP, %0"}, 812 {"prfm\\tPSTL1STRM, %0", 813 "prfm\\tPSTL3KEEP, %0", 814 "prfm\\tPSTL2KEEP, %0", 815 "prfm\\tPSTL1KEEP, %0"}, 816 }; 817 818 int locality = INTVAL (operands[2]); 819 820 gcc_assert (IN_RANGE (locality, 0, 3)); 821 822 /* PRFM accepts the same addresses as a 64-bit LDR so wrap 823 the address into a DImode MEM so that aarch64_print_operand knows 824 how to print it. */ 825 operands[0] = gen_rtx_MEM (DImode, operands[0]); 826 return pftype[INTVAL(operands[1])][locality]; 827 } 828 [(set_attr "type" "load_4")] 829) 830 831(define_insn "trap" 832 [(trap_if (const_int 1) (const_int 8))] 833 "" 834 "brk #1000" 835 [(set_attr "type" "trap")]) 836 837(define_expand "prologue" 838 [(clobber (const_int 0))] 839 "" 840 " 841 aarch64_expand_prologue (); 842 DONE; 843 " 844) 845 846(define_expand "epilogue" 847 [(clobber (const_int 0))] 848 "" 849 " 850 aarch64_expand_epilogue (false); 851 DONE; 852 " 853) 854 855(define_expand "sibcall_epilogue" 856 [(clobber (const_int 0))] 857 "" 858 " 859 aarch64_expand_epilogue (true); 860 DONE; 861 " 862) 863 864(define_insn "*do_return" 865 [(return)] 866 "" 867 { 868 const char *ret = NULL; 869 if (aarch64_return_address_signing_enabled () 870 && TARGET_ARMV8_3 871 && !crtl->calls_eh_return) 872 { 873 if (aarch64_ra_sign_key == AARCH64_KEY_B) 874 ret = "retab"; 875 else 876 ret = "retaa"; 877 } 878 else 879 ret = "ret"; 880 output_asm_insn (ret, operands); 881 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 882 } 883 [(set_attr "type" "branch") 884 (set_attr "sls_length" "retbr")] 885) 886 887(define_expand "return" 888 [(simple_return)] 889 "aarch64_use_return_insn_p ()" 890 "" 891) 892 893(define_insn "simple_return" 894 [(simple_return)] 895 "" 896 { 897 output_asm_insn ("ret", operands); 898 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 899 } 900 [(set_attr "type" "branch") 901 (set_attr "sls_length" "retbr")] 902) 903 904(define_insn "*cb<optab><mode>1" 905 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r") 906 (const_int 0)) 907 (label_ref (match_operand 1 "" "")) 908 (pc)))] 909 "!aarch64_track_speculation" 910 { 911 if (get_attr_length (insn) == 8) 912 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, "); 913 else 914 return "<cbz>\\t%<w>0, %l1"; 915 } 916 [(set_attr "type" "branch") 917 (set (attr "length") 918 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576)) 919 (lt (minus (match_dup 1) (pc)) (const_int 1048572))) 920 (const_int 4) 921 (const_int 8))) 922 (set (attr "far_branch") 923 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 924 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 925 (const_int 0) 926 (const_int 1)))] 927) 928 929(define_insn "*tb<optab><mode>1" 930 [(set (pc) (if_then_else 931 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r") 932 (const_int 1) 933 (match_operand 1 934 "aarch64_simd_shift_imm_<mode>" "n")) 935 (const_int 0)) 936 (label_ref (match_operand 2 "" "")) 937 (pc))) 938 (clobber (reg:CC CC_REGNUM))] 939 "!aarch64_track_speculation" 940 { 941 if (get_attr_length (insn) == 8) 942 { 943 if (get_attr_far_branch (insn) == 1) 944 return aarch64_gen_far_branch (operands, 2, "Ltb", 945 "<inv_tb>\\t%<w>0, %1, "); 946 else 947 { 948 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1])); 949 return "tst\t%<w>0, %1\;<bcond>\t%l2"; 950 } 951 } 952 else 953 return "<tbz>\t%<w>0, %1, %l2"; 954 } 955 [(set_attr "type" "branch") 956 (set (attr "length") 957 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768)) 958 (lt (minus (match_dup 2) (pc)) (const_int 32764))) 959 (const_int 4) 960 (const_int 8))) 961 (set (attr "far_branch") 962 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 963 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 964 (const_int 0) 965 (const_int 1)))] 966 967) 968 969(define_insn "*cb<optab><mode>1" 970 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r") 971 (const_int 0)) 972 (label_ref (match_operand 1 "" "")) 973 (pc))) 974 (clobber (reg:CC CC_REGNUM))] 975 "!aarch64_track_speculation" 976 { 977 if (get_attr_length (insn) == 8) 978 { 979 if (get_attr_far_branch (insn) == 1) 980 return aarch64_gen_far_branch (operands, 1, "Ltb", 981 "<inv_tb>\\t%<w>0, <sizem1>, "); 982 else 983 { 984 char buf[64]; 985 uint64_t val = ((uint64_t) 1) 986 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1); 987 sprintf (buf, "tst\t%%<w>0, %" PRId64, val); 988 output_asm_insn (buf, operands); 989 return "<bcond>\t%l1"; 990 } 991 } 992 else 993 return "<tbz>\t%<w>0, <sizem1>, %l1"; 994 } 995 [(set_attr "type" "branch") 996 (set (attr "length") 997 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768)) 998 (lt (minus (match_dup 1) (pc)) (const_int 32764))) 999 (const_int 4) 1000 (const_int 8))) 1001 (set (attr "far_branch") 1002 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576)) 1003 (lt (minus (match_dup 1) (pc)) (const_int 1048572))) 1004 (const_int 0) 1005 (const_int 1)))] 1006) 1007 1008;; ------------------------------------------------------------------- 1009;; Subroutine calls and sibcalls 1010;; ------------------------------------------------------------------- 1011 1012(define_expand "call" 1013 [(parallel 1014 [(call (match_operand 0 "memory_operand") 1015 (match_operand 1 "general_operand")) 1016 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1017 (clobber (reg:DI LR_REGNUM))])] 1018 "" 1019 " 1020 { 1021 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false); 1022 DONE; 1023 }" 1024) 1025 1026(define_insn "*call_insn" 1027 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucr, Usf")) 1028 (match_operand 1 "" "")) 1029 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1030 (clobber (reg:DI LR_REGNUM))] 1031 "" 1032 "@ 1033 * return aarch64_indirect_call_asm (operands[0]); 1034 bl\\t%c0" 1035 [(set_attr "type" "call, call")]) 1036 1037(define_expand "call_value" 1038 [(parallel 1039 [(set (match_operand 0 "") 1040 (call (match_operand 1 "memory_operand") 1041 (match_operand 2 "general_operand"))) 1042 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1043 (clobber (reg:DI LR_REGNUM))])] 1044 "" 1045 " 1046 { 1047 aarch64_expand_call (operands[0], operands[1], operands[3], false); 1048 DONE; 1049 }" 1050) 1051 1052(define_insn "*call_value_insn" 1053 [(set (match_operand 0 "" "") 1054 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "Ucr, Usf")) 1055 (match_operand 2 "" ""))) 1056 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1057 (clobber (reg:DI LR_REGNUM))] 1058 "" 1059 "@ 1060 * return aarch64_indirect_call_asm (operands[1]); 1061 bl\\t%c1" 1062 [(set_attr "type" "call, call")] 1063) 1064 1065(define_expand "sibcall" 1066 [(parallel 1067 [(call (match_operand 0 "memory_operand") 1068 (match_operand 1 "general_operand")) 1069 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1070 (return)])] 1071 "" 1072 { 1073 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true); 1074 DONE; 1075 } 1076) 1077 1078(define_expand "sibcall_value" 1079 [(parallel 1080 [(set (match_operand 0 "") 1081 (call (match_operand 1 "memory_operand") 1082 (match_operand 2 "general_operand"))) 1083 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1084 (return)])] 1085 "" 1086 { 1087 aarch64_expand_call (operands[0], operands[1], operands[3], true); 1088 DONE; 1089 } 1090) 1091 1092(define_insn "*sibcall_insn" 1093 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf")) 1094 (match_operand 1 "")) 1095 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1096 (return)] 1097 "SIBLING_CALL_P (insn)" 1098 { 1099 if (which_alternative == 0) 1100 { 1101 output_asm_insn ("br\\t%0", operands); 1102 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 1103 } 1104 return "b\\t%c0"; 1105 } 1106 [(set_attr "type" "branch, branch") 1107 (set_attr "sls_length" "retbr,none")] 1108) 1109 1110(define_insn "*sibcall_value_insn" 1111 [(set (match_operand 0 "") 1112 (call (mem:DI 1113 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf")) 1114 (match_operand 2 ""))) 1115 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1116 (return)] 1117 "SIBLING_CALL_P (insn)" 1118 { 1119 if (which_alternative == 0) 1120 { 1121 output_asm_insn ("br\\t%1", operands); 1122 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 1123 } 1124 return "b\\t%c1"; 1125 } 1126 [(set_attr "type" "branch, branch") 1127 (set_attr "sls_length" "retbr,none")] 1128) 1129 1130;; Call subroutine returning any type. 1131 1132(define_expand "untyped_call" 1133 [(parallel [(call (match_operand 0 "") 1134 (const_int 0)) 1135 (match_operand 1 "") 1136 (match_operand 2 "")])] 1137 "" 1138{ 1139 int i; 1140 1141 /* Untyped calls always use the default ABI. It's only possible to use 1142 ABI variants if we know the type of the target function. */ 1143 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx)); 1144 1145 for (i = 0; i < XVECLEN (operands[2], 0); i++) 1146 { 1147 rtx set = XVECEXP (operands[2], 0, i); 1148 emit_move_insn (SET_DEST (set), SET_SRC (set)); 1149 } 1150 1151 /* The optimizer does not know that the call sets the function value 1152 registers we stored in the result block. We avoid problems by 1153 claiming that all hard registers are used and clobbered at this 1154 point. */ 1155 emit_insn (gen_blockage ()); 1156 DONE; 1157}) 1158 1159;; ------------------------------------------------------------------- 1160;; Moves 1161;; ------------------------------------------------------------------- 1162 1163(define_expand "mov<mode>" 1164 [(set (match_operand:SHORT 0 "nonimmediate_operand") 1165 (match_operand:SHORT 1 "general_operand"))] 1166 "" 1167 " 1168 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1169 operands[1] = force_reg (<MODE>mode, operands[1]); 1170 1171 if (GET_CODE (operands[1]) == CONST_POLY_INT) 1172 { 1173 aarch64_expand_mov_immediate (operands[0], operands[1]); 1174 DONE; 1175 } 1176 " 1177) 1178 1179(define_insn "*mov<mode>_aarch64" 1180 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w") 1181 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))] 1182 "(register_operand (operands[0], <MODE>mode) 1183 || aarch64_reg_or_zero (operands[1], <MODE>mode))" 1184{ 1185 switch (which_alternative) 1186 { 1187 case 0: 1188 return "mov\t%w0, %w1"; 1189 case 1: 1190 return "mov\t%w0, %1"; 1191 case 2: 1192 return aarch64_output_scalar_simd_mov_immediate (operands[1], 1193 <MODE>mode); 1194 case 3: 1195 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1196 case 4: 1197 return "ldr<size>\t%w0, %1"; 1198 case 5: 1199 return "ldr\t%<size>0, %1"; 1200 case 6: 1201 return "str<size>\t%w1, %0"; 1202 case 7: 1203 return "str\t%<size>1, %0"; 1204 case 8: 1205 return "umov\t%w0, %1.<v>[0]"; 1206 case 9: 1207 return "dup\t%0.<Vallxd>, %w1"; 1208 case 10: 1209 return "dup\t%<Vetype>0, %1.<v>[0]"; 1210 default: 1211 gcc_unreachable (); 1212 } 1213} 1214 ;; The "mov_imm" type for CNT is just a placeholder. 1215 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4, 1216 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup") 1217 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")] 1218) 1219 1220(define_expand "mov<mode>" 1221 [(set (match_operand:GPI 0 "nonimmediate_operand") 1222 (match_operand:GPI 1 "general_operand"))] 1223 "" 1224 " 1225 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0]) 1226 && CONST_INT_P (operands[1]) && <MODE>mode == DImode 1227 && aarch64_split_dimode_const_store (operands[0], operands[1])) 1228 DONE; 1229 1230 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1231 operands[1] = force_reg (<MODE>mode, operands[1]); 1232 1233 /* Lower moves of symbolic constants into individual instructions. 1234 Doing this now is sometimes necessary for correctness, since some 1235 sequences require temporary pseudo registers. Lowering now is also 1236 often better for optimization, since more RTL passes get the 1237 chance to optimize the individual instructions. 1238 1239 When called after RA, also split multi-instruction moves into 1240 smaller pieces now, since we can't be sure that sure that there 1241 will be a following split pass. */ 1242 if (CONST_INT_P (operands[1]) 1243 ? (reload_completed 1244 && !aarch64_mov_imm_operand (operands[1], <MODE>mode)) 1245 : CONSTANT_P (operands[1])) 1246 { 1247 aarch64_expand_mov_immediate (operands[0], operands[1]); 1248 DONE; 1249 } 1250 " 1251) 1252 1253(define_insn_and_split "*movsi_aarch64" 1254 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w") 1255 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))] 1256 "(register_operand (operands[0], SImode) 1257 || aarch64_reg_or_zero (operands[1], SImode))" 1258 "@ 1259 mov\\t%w0, %w1 1260 mov\\t%w0, %w1 1261 mov\\t%w0, %w1 1262 mov\\t%w0, %1 1263 # 1264 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1265 ldr\\t%w0, %1 1266 ldr\\t%s0, %1 1267 str\\t%w1, %0 1268 str\\t%s1, %0 1269 adr\\t%x0, %c1 1270 adrp\\t%x0, %A1 1271 fmov\\t%s0, %w1 1272 fmov\\t%w0, %s1 1273 fmov\\t%s0, %s1 1274 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);" 1275 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode) 1276 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))" 1277 [(const_int 0)] 1278 "{ 1279 aarch64_expand_mov_immediate (operands[0], operands[1]); 1280 DONE; 1281 }" 1282 ;; The "mov_imm" type for CNT is just a placeholder. 1283 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4, 1284 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move") 1285 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")] 1286) 1287 1288(define_insn_and_split "*movdi_aarch64" 1289 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w") 1290 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))] 1291 "(register_operand (operands[0], DImode) 1292 || aarch64_reg_or_zero (operands[1], DImode))" 1293 "@ 1294 mov\\t%x0, %x1 1295 mov\\t%0, %x1 1296 mov\\t%x0, %1 1297 mov\\t%x0, %1 1298 mov\\t%w0, %1 1299 # 1300 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1301 ldr\\t%x0, %1 1302 ldr\\t%d0, %1 1303 str\\t%x1, %0 1304 str\\t%d1, %0 1305 adr\\t%x0, %c1 1306 adrp\\t%x0, %A1 1307 fmov\\t%d0, %x1 1308 fmov\\t%x0, %d1 1309 fmov\\t%d0, %d1 1310 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);" 1311 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode)) 1312 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))" 1313 [(const_int 0)] 1314 "{ 1315 aarch64_expand_mov_immediate (operands[0], operands[1]); 1316 DONE; 1317 }" 1318 ;; The "mov_imm" type for CNTD is just a placeholder. 1319 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm, 1320 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov, 1321 neon_move") 1322 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")] 1323) 1324 1325(define_insn "insv_imm<mode>" 1326 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 1327 (const_int 16) 1328 (match_operand:GPI 1 "const_int_operand" "n")) 1329 (match_operand:GPI 2 "const_int_operand" "n"))] 1330 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode) 1331 && UINTVAL (operands[1]) % 16 == 0" 1332 "movk\\t%<w>0, %X2, lsl %1" 1333 [(set_attr "type" "mov_imm")] 1334) 1335 1336;; Match MOVK as a normal AND and IOR operation. 1337(define_insn "aarch64_movk<mode>" 1338 [(set (match_operand:GPI 0 "register_operand" "=r") 1339 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 1340 (match_operand:GPI 2 "const_int_operand")) 1341 (match_operand:GPI 3 "const_int_operand")))] 1342 "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode), 1343 rtx_mode_t (operands[3], <MODE>mode)) >= 0" 1344 { 1345 int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode), 1346 rtx_mode_t (operands[3], <MODE>mode)); 1347 operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode); 1348 operands[3] = gen_int_mode (shift, SImode); 1349 return "movk\\t%<w>0, #%X2, lsl %3"; 1350 } 1351 [(set_attr "type" "mov_imm")] 1352) 1353 1354(define_expand "movti" 1355 [(set (match_operand:TI 0 "nonimmediate_operand") 1356 (match_operand:TI 1 "general_operand"))] 1357 "" 1358 " 1359 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1360 operands[1] = force_reg (TImode, operands[1]); 1361 1362 if (GET_CODE (operands[1]) == CONST_POLY_INT) 1363 { 1364 emit_move_insn (gen_lowpart (DImode, operands[0]), 1365 gen_lowpart (DImode, operands[1])); 1366 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx); 1367 DONE; 1368 } 1369 " 1370) 1371 1372(define_insn "*movti_aarch64" 1373 [(set (match_operand:TI 0 1374 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m") 1375 (match_operand:TI 1 1376 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))] 1377 "(register_operand (operands[0], TImode) 1378 || aarch64_reg_or_zero (operands[1], TImode))" 1379 "@ 1380 # 1381 # 1382 # 1383 mov\\t%0.16b, %1.16b 1384 ldp\\t%0, %H0, %1 1385 stp\\t%1, %H1, %0 1386 stp\\txzr, xzr, %0 1387 ldr\\t%q0, %1 1388 str\\t%q1, %0" 1389 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \ 1390 load_16,store_16,store_16,\ 1391 load_16,store_16") 1392 (set_attr "length" "8,8,8,4,4,4,4,4,4") 1393 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")] 1394) 1395 1396;; Split a TImode register-register or register-immediate move into 1397;; its component DImode pieces, taking care to handle overlapping 1398;; source and dest registers. 1399(define_split 1400 [(set (match_operand:TI 0 "register_operand" "") 1401 (match_operand:TI 1 "aarch64_reg_or_imm" ""))] 1402 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])" 1403 [(const_int 0)] 1404{ 1405 aarch64_split_128bit_move (operands[0], operands[1]); 1406 DONE; 1407}) 1408 1409(define_expand "mov<mode>" 1410 [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand") 1411 (match_operand:GPF_TF_F16_MOV 1 "general_operand"))] 1412 "" 1413 { 1414 if (!TARGET_FLOAT) 1415 { 1416 aarch64_err_no_fpadvsimd (<MODE>mode); 1417 FAIL; 1418 } 1419 1420 if (GET_CODE (operands[0]) == MEM 1421 && ! (GET_CODE (operands[1]) == CONST_DOUBLE 1422 && aarch64_float_const_zero_rtx_p (operands[1]))) 1423 operands[1] = force_reg (<MODE>mode, operands[1]); 1424 } 1425) 1426 1427(define_insn "*mov<mode>_aarch64" 1428 [(set (match_operand:HFBF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r") 1429 (match_operand:HFBF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))] 1430 "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode) 1431 || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))" 1432 "@ 1433 movi\\t%0.4h, #0 1434 fmov\\t%h0, %w1 1435 dup\\t%w0.4h, %w1 1436 umov\\t%w0, %1.h[0] 1437 mov\\t%0.h[0], %1.h[0] 1438 fmov\\t%h0, %1 1439 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode); 1440 ldr\\t%h0, %1 1441 str\\t%h1, %0 1442 ldrh\\t%w0, %1 1443 strh\\t%w1, %0 1444 mov\\t%w0, %w1" 1445 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \ 1446 neon_move,f_loads,f_stores,load_4,store_4,mov_reg") 1447 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")] 1448) 1449 1450(define_insn "*movsf_aarch64" 1451 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r") 1452 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))] 1453 "TARGET_FLOAT && (register_operand (operands[0], SFmode) 1454 || aarch64_reg_or_fp_zero (operands[1], SFmode))" 1455 "@ 1456 movi\\t%0.2s, #0 1457 fmov\\t%s0, %w1 1458 fmov\\t%w0, %s1 1459 fmov\\t%s0, %s1 1460 fmov\\t%s0, %1 1461 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode); 1462 ldr\\t%s0, %1 1463 str\\t%s1, %0 1464 ldr\\t%w0, %1 1465 str\\t%w1, %0 1466 mov\\t%w0, %w1 1467 mov\\t%w0, %1" 1468 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\ 1469 f_loads,f_stores,load_4,store_4,mov_reg,\ 1470 fconsts") 1471 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")] 1472) 1473 1474(define_insn "*movdf_aarch64" 1475 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r") 1476 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))] 1477 "TARGET_FLOAT && (register_operand (operands[0], DFmode) 1478 || aarch64_reg_or_fp_zero (operands[1], DFmode))" 1479 "@ 1480 movi\\t%d0, #0 1481 fmov\\t%d0, %x1 1482 fmov\\t%x0, %d1 1483 fmov\\t%d0, %d1 1484 fmov\\t%d0, %1 1485 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode); 1486 ldr\\t%d0, %1 1487 str\\t%d1, %0 1488 ldr\\t%x0, %1 1489 str\\t%x1, %0 1490 mov\\t%x0, %x1 1491 mov\\t%x0, %1" 1492 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\ 1493 f_loadd,f_stored,load_8,store_8,mov_reg,\ 1494 fconstd") 1495 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")] 1496) 1497 1498(define_split 1499 [(set (match_operand:GPF_HF 0 "nonimmediate_operand") 1500 (match_operand:GPF_HF 1 "general_operand"))] 1501 "can_create_pseudo_p () 1502 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode) 1503 && !aarch64_float_const_representable_p (operands[1]) 1504 && aarch64_float_const_rtx_p (operands[1])" 1505 [(const_int 0)] 1506 { 1507 unsigned HOST_WIDE_INT ival; 1508 if (!aarch64_reinterpret_float_as_int (operands[1], &ival)) 1509 FAIL; 1510 1511 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode); 1512 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode)); 1513 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp)); 1514 DONE; 1515 } 1516) 1517 1518(define_insn "*movtf_aarch64" 1519 [(set (match_operand:TF 0 1520 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m") 1521 (match_operand:TF 1 1522 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))] 1523 "TARGET_FLOAT && (register_operand (operands[0], TFmode) 1524 || aarch64_reg_or_fp_zero (operands[1], TFmode))" 1525 "@ 1526 mov\\t%0.16b, %1.16b 1527 # 1528 # 1529 # 1530 movi\\t%0.2d, #0 1531 fmov\\t%s0, wzr 1532 ldr\\t%q0, %1 1533 str\\t%q1, %0 1534 ldp\\t%0, %H0, %1 1535 stp\\t%1, %H1, %0 1536 stp\\txzr, xzr, %0" 1537 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\ 1538 f_loadd,f_stored,load_16,store_16,store_16") 1539 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4") 1540 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")] 1541) 1542 1543(define_split 1544 [(set (match_operand:TF 0 "register_operand" "") 1545 (match_operand:TF 1 "aarch64_reg_or_imm" ""))] 1546 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])" 1547 [(const_int 0)] 1548 { 1549 aarch64_split_128bit_move (operands[0], operands[1]); 1550 DONE; 1551 } 1552) 1553 1554;; 0 is dst 1555;; 1 is src 1556;; 2 is size of copy in bytes 1557;; 3 is alignment 1558 1559(define_expand "cpymemdi" 1560 [(match_operand:BLK 0 "memory_operand") 1561 (match_operand:BLK 1 "memory_operand") 1562 (match_operand:DI 2 "immediate_operand") 1563 (match_operand:DI 3 "immediate_operand")] 1564 "!STRICT_ALIGNMENT" 1565{ 1566 if (aarch64_expand_cpymem (operands)) 1567 DONE; 1568 FAIL; 1569} 1570) 1571 1572;; Operands 1 and 3 are tied together by the final condition; so we allow 1573;; fairly lax checking on the second memory operation. 1574(define_insn "load_pair_sw_<SX:mode><SX2:mode>" 1575 [(set (match_operand:SX 0 "register_operand" "=r,w") 1576 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump")) 1577 (set (match_operand:SX2 2 "register_operand" "=r,w") 1578 (match_operand:SX2 3 "memory_operand" "m,m"))] 1579 "rtx_equal_p (XEXP (operands[3], 0), 1580 plus_constant (Pmode, 1581 XEXP (operands[1], 0), 1582 GET_MODE_SIZE (<SX:MODE>mode)))" 1583 "@ 1584 ldp\\t%w0, %w2, %1 1585 ldp\\t%s0, %s2, %1" 1586 [(set_attr "type" "load_8,neon_load1_2reg") 1587 (set_attr "arch" "*,fp")] 1588) 1589 1590;; Storing different modes that can still be merged 1591(define_insn "load_pair_dw_<DX:mode><DX2:mode>" 1592 [(set (match_operand:DX 0 "register_operand" "=r,w") 1593 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump")) 1594 (set (match_operand:DX2 2 "register_operand" "=r,w") 1595 (match_operand:DX2 3 "memory_operand" "m,m"))] 1596 "rtx_equal_p (XEXP (operands[3], 0), 1597 plus_constant (Pmode, 1598 XEXP (operands[1], 0), 1599 GET_MODE_SIZE (<DX:MODE>mode)))" 1600 "@ 1601 ldp\\t%x0, %x2, %1 1602 ldp\\t%d0, %d2, %1" 1603 [(set_attr "type" "load_16,neon_load1_2reg") 1604 (set_attr "arch" "*,fp")] 1605) 1606 1607(define_insn "load_pair_dw_tftf" 1608 [(set (match_operand:TF 0 "register_operand" "=w") 1609 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump")) 1610 (set (match_operand:TF 2 "register_operand" "=w") 1611 (match_operand:TF 3 "memory_operand" "m"))] 1612 "TARGET_SIMD 1613 && rtx_equal_p (XEXP (operands[3], 0), 1614 plus_constant (Pmode, 1615 XEXP (operands[1], 0), 1616 GET_MODE_SIZE (TFmode)))" 1617 "ldp\\t%q0, %q2, %1" 1618 [(set_attr "type" "neon_ldp_q") 1619 (set_attr "fp" "yes")] 1620) 1621 1622;; Operands 0 and 2 are tied together by the final condition; so we allow 1623;; fairly lax checking on the second memory operation. 1624(define_insn "store_pair_sw_<SX:mode><SX2:mode>" 1625 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump") 1626 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w")) 1627 (set (match_operand:SX2 2 "memory_operand" "=m,m") 1628 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))] 1629 "rtx_equal_p (XEXP (operands[2], 0), 1630 plus_constant (Pmode, 1631 XEXP (operands[0], 0), 1632 GET_MODE_SIZE (<SX:MODE>mode)))" 1633 "@ 1634 stp\\t%w1, %w3, %0 1635 stp\\t%s1, %s3, %0" 1636 [(set_attr "type" "store_8,neon_store1_2reg") 1637 (set_attr "arch" "*,fp")] 1638) 1639 1640;; Storing different modes that can still be merged 1641(define_insn "store_pair_dw_<DX:mode><DX2:mode>" 1642 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump") 1643 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w")) 1644 (set (match_operand:DX2 2 "memory_operand" "=m,m") 1645 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))] 1646 "rtx_equal_p (XEXP (operands[2], 0), 1647 plus_constant (Pmode, 1648 XEXP (operands[0], 0), 1649 GET_MODE_SIZE (<DX:MODE>mode)))" 1650 "@ 1651 stp\\t%x1, %x3, %0 1652 stp\\t%d1, %d3, %0" 1653 [(set_attr "type" "store_16,neon_store1_2reg") 1654 (set_attr "arch" "*,fp")] 1655) 1656 1657(define_insn "store_pair_dw_tftf" 1658 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump") 1659 (match_operand:TF 1 "register_operand" "w")) 1660 (set (match_operand:TF 2 "memory_operand" "=m") 1661 (match_operand:TF 3 "register_operand" "w"))] 1662 "TARGET_SIMD && 1663 rtx_equal_p (XEXP (operands[2], 0), 1664 plus_constant (Pmode, 1665 XEXP (operands[0], 0), 1666 GET_MODE_SIZE (TFmode)))" 1667 "stp\\t%q1, %q3, %0" 1668 [(set_attr "type" "neon_stp_q") 1669 (set_attr "fp" "yes")] 1670) 1671 1672;; Load pair with post-index writeback. This is primarily used in function 1673;; epilogues. 1674(define_insn "loadwb_pair<GPI:mode>_<P:mode>" 1675 [(parallel 1676 [(set (match_operand:P 0 "register_operand" "=k") 1677 (plus:P (match_operand:P 1 "register_operand" "0") 1678 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1679 (set (match_operand:GPI 2 "register_operand" "=r") 1680 (mem:GPI (match_dup 1))) 1681 (set (match_operand:GPI 3 "register_operand" "=r") 1682 (mem:GPI (plus:P (match_dup 1) 1683 (match_operand:P 5 "const_int_operand" "n"))))])] 1684 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)" 1685 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4" 1686 [(set_attr "type" "load_<GPI:ldpstp_sz>")] 1687) 1688 1689(define_insn "loadwb_pair<GPF:mode>_<P:mode>" 1690 [(parallel 1691 [(set (match_operand:P 0 "register_operand" "=k") 1692 (plus:P (match_operand:P 1 "register_operand" "0") 1693 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1694 (set (match_operand:GPF 2 "register_operand" "=w") 1695 (mem:GPF (match_dup 1))) 1696 (set (match_operand:GPF 3 "register_operand" "=w") 1697 (mem:GPF (plus:P (match_dup 1) 1698 (match_operand:P 5 "const_int_operand" "n"))))])] 1699 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)" 1700 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4" 1701 [(set_attr "type" "neon_load1_2reg")] 1702) 1703 1704(define_insn "loadwb_pair<TX:mode>_<P:mode>" 1705 [(parallel 1706 [(set (match_operand:P 0 "register_operand" "=k") 1707 (plus:P (match_operand:P 1 "register_operand" "0") 1708 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1709 (set (match_operand:TX 2 "register_operand" "=w") 1710 (mem:TX (match_dup 1))) 1711 (set (match_operand:TX 3 "register_operand" "=w") 1712 (mem:TX (plus:P (match_dup 1) 1713 (match_operand:P 5 "const_int_operand" "n"))))])] 1714 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)" 1715 "ldp\\t%q2, %q3, [%1], %4" 1716 [(set_attr "type" "neon_ldp_q")] 1717) 1718 1719;; Store pair with pre-index writeback. This is primarily used in function 1720;; prologues. 1721(define_insn "storewb_pair<GPI:mode>_<P:mode>" 1722 [(parallel 1723 [(set (match_operand:P 0 "register_operand" "=&k") 1724 (plus:P (match_operand:P 1 "register_operand" "0") 1725 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1726 (set (mem:GPI (plus:P (match_dup 0) 1727 (match_dup 4))) 1728 (match_operand:GPI 2 "register_operand" "r")) 1729 (set (mem:GPI (plus:P (match_dup 0) 1730 (match_operand:P 5 "const_int_operand" "n"))) 1731 (match_operand:GPI 3 "register_operand" "r"))])] 1732 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)" 1733 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!" 1734 [(set_attr "type" "store_<GPI:ldpstp_sz>")] 1735) 1736 1737(define_insn "storewb_pair<GPF:mode>_<P:mode>" 1738 [(parallel 1739 [(set (match_operand:P 0 "register_operand" "=&k") 1740 (plus:P (match_operand:P 1 "register_operand" "0") 1741 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1742 (set (mem:GPF (plus:P (match_dup 0) 1743 (match_dup 4))) 1744 (match_operand:GPF 2 "register_operand" "w")) 1745 (set (mem:GPF (plus:P (match_dup 0) 1746 (match_operand:P 5 "const_int_operand" "n"))) 1747 (match_operand:GPF 3 "register_operand" "w"))])] 1748 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)" 1749 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!" 1750 [(set_attr "type" "neon_store1_2reg<q>")] 1751) 1752 1753(define_insn "storewb_pair<TX:mode>_<P:mode>" 1754 [(parallel 1755 [(set (match_operand:P 0 "register_operand" "=&k") 1756 (plus:P (match_operand:P 1 "register_operand" "0") 1757 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1758 (set (mem:TX (plus:P (match_dup 0) 1759 (match_dup 4))) 1760 (match_operand:TX 2 "register_operand" "w")) 1761 (set (mem:TX (plus:P (match_dup 0) 1762 (match_operand:P 5 "const_int_operand" "n"))) 1763 (match_operand:TX 3 "register_operand" "w"))])] 1764 "TARGET_SIMD 1765 && INTVAL (operands[5]) 1766 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)" 1767 "stp\\t%q2, %q3, [%0, %4]!" 1768 [(set_attr "type" "neon_stp_q")] 1769) 1770 1771;; ------------------------------------------------------------------- 1772;; Sign/Zero extension 1773;; ------------------------------------------------------------------- 1774 1775(define_expand "<optab>sidi2" 1776 [(set (match_operand:DI 0 "register_operand") 1777 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))] 1778 "" 1779) 1780 1781(define_insn "*extendsidi2_aarch64" 1782 [(set (match_operand:DI 0 "register_operand" "=r,r") 1783 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))] 1784 "" 1785 "@ 1786 sxtw\t%0, %w1 1787 ldrsw\t%0, %1" 1788 [(set_attr "type" "extend,load_4")] 1789) 1790 1791(define_insn "*load_pair_extendsidi2_aarch64" 1792 [(set (match_operand:DI 0 "register_operand" "=r") 1793 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump"))) 1794 (set (match_operand:DI 2 "register_operand" "=r") 1795 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))] 1796 "rtx_equal_p (XEXP (operands[3], 0), 1797 plus_constant (Pmode, 1798 XEXP (operands[1], 0), 1799 GET_MODE_SIZE (SImode)))" 1800 "ldpsw\\t%0, %2, %1" 1801 [(set_attr "type" "load_8")] 1802) 1803 1804(define_insn "*zero_extendsidi2_aarch64" 1805 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w") 1806 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))] 1807 "" 1808 "@ 1809 uxtw\t%0, %w1 1810 ldr\t%w0, %1 1811 fmov\t%s0, %w1 1812 ldr\t%s0, %1 1813 fmov\t%w0, %s1 1814 fmov\t%s0, %s1" 1815 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov") 1816 (set_attr "arch" "*,*,fp,fp,fp,fp")] 1817) 1818 1819(define_insn "*load_pair_zero_extendsidi2_aarch64" 1820 [(set (match_operand:DI 0 "register_operand" "=r,w") 1821 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))) 1822 (set (match_operand:DI 2 "register_operand" "=r,w") 1823 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))] 1824 "rtx_equal_p (XEXP (operands[3], 0), 1825 plus_constant (Pmode, 1826 XEXP (operands[1], 0), 1827 GET_MODE_SIZE (SImode)))" 1828 "@ 1829 ldp\t%w0, %w2, %1 1830 ldp\t%s0, %s2, %1" 1831 [(set_attr "type" "load_8,neon_load1_2reg") 1832 (set_attr "arch" "*,fp")] 1833) 1834 1835(define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2" 1836 [(set (match_operand:GPI 0 "register_operand") 1837 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))] 1838 "" 1839) 1840 1841(define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64" 1842 [(set (match_operand:GPI 0 "register_operand" "=r,r") 1843 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))] 1844 "" 1845 "@ 1846 sxt<SHORT:size>\t%<GPI:w>0, %w1 1847 ldrs<SHORT:size>\t%<GPI:w>0, %1" 1848 [(set_attr "type" "extend,load_4")] 1849) 1850 1851(define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64" 1852 [(set (match_operand:GPI 0 "register_operand" "=r,r,w") 1853 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))] 1854 "" 1855 "@ 1856 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask> 1857 ldr<SHORT:size>\t%w0, %1 1858 ldr\t%<SHORT:size>0, %1" 1859 [(set_attr "type" "logic_imm,load_4,f_loads") 1860 (set_attr "arch" "*,*,fp")] 1861) 1862 1863(define_expand "<optab>qihi2" 1864 [(set (match_operand:HI 0 "register_operand") 1865 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))] 1866 "" 1867) 1868 1869(define_insn "*extendqihi2_aarch64" 1870 [(set (match_operand:HI 0 "register_operand" "=r,r") 1871 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))] 1872 "" 1873 "@ 1874 sxtb\t%w0, %w1 1875 ldrsb\t%w0, %1" 1876 [(set_attr "type" "extend,load_4")] 1877) 1878 1879(define_insn "*zero_extendqihi2_aarch64" 1880 [(set (match_operand:HI 0 "register_operand" "=r,r") 1881 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))] 1882 "" 1883 "@ 1884 and\t%w0, %w1, 255 1885 ldrb\t%w0, %1" 1886 [(set_attr "type" "logic_imm,load_4")] 1887) 1888 1889;; ------------------------------------------------------------------- 1890;; Simple arithmetic 1891;; ------------------------------------------------------------------- 1892 1893(define_expand "add<mode>3" 1894 [(set 1895 (match_operand:GPI 0 "register_operand") 1896 (plus:GPI (match_operand:GPI 1 "register_operand") 1897 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))] 1898 "" 1899{ 1900 /* If operands[1] is a subreg extract the inner RTX. */ 1901 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]); 1902 1903 /* If the constant is too large for a single instruction and isn't frame 1904 based, split off the immediate so it is available for CSE. */ 1905 if (!aarch64_plus_immediate (operands[2], <MODE>mode) 1906 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode)) 1907 && can_create_pseudo_p () 1908 && (!REG_P (op1) 1909 || !REGNO_PTR_FRAME_P (REGNO (op1)))) 1910 operands[2] = force_reg (<MODE>mode, operands[2]); 1911 /* Some tunings prefer to avoid VL-based operations. 1912 Split off the poly immediate here. The rtx costs hook will reject attempts 1913 to combine them back. */ 1914 else if (GET_CODE (operands[2]) == CONST_POLY_INT 1915 && can_create_pseudo_p () 1916 && (aarch64_tune_params.extra_tuning_flags 1917 & AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS)) 1918 operands[2] = force_reg (<MODE>mode, operands[2]); 1919 /* Expand polynomial additions now if the destination is the stack 1920 pointer, since we don't want to use that as a temporary. */ 1921 else if (operands[0] == stack_pointer_rtx 1922 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)) 1923 { 1924 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1], 1925 operands[2], NULL_RTX, NULL_RTX); 1926 DONE; 1927 } 1928}) 1929 1930(define_insn "*add<mode>3_aarch64" 1931 [(set 1932 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk") 1933 (plus:GPI 1934 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk") 1935 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))] 1936 "" 1937 "@ 1938 add\\t%<w>0, %<w>1, %2 1939 add\\t%<w>0, %<w>1, %<w>2 1940 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas> 1941 sub\\t%<w>0, %<w>1, #%n2 1942 # 1943 * return aarch64_output_sve_scalar_inc_dec (operands[2]); 1944 * return aarch64_output_sve_addvl_addpl (operands[2]);" 1945 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders. 1946 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm") 1947 (set_attr "arch" "*,*,simd,*,*,sve,sve")] 1948) 1949 1950;; zero_extend version of above 1951(define_insn "*addsi3_aarch64_uxtw" 1952 [(set 1953 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r") 1954 (zero_extend:DI 1955 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk") 1956 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))] 1957 "" 1958 "@ 1959 add\\t%w0, %w1, %2 1960 add\\t%w0, %w1, %w2 1961 sub\\t%w0, %w1, #%n2 1962 #" 1963 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")] 1964) 1965 1966;; If there's a free register, and we can load the constant with a 1967;; single instruction, do so. This has a chance to improve scheduling. 1968(define_peephole2 1969 [(match_scratch:GPI 3 "r") 1970 (set (match_operand:GPI 0 "register_operand") 1971 (plus:GPI 1972 (match_operand:GPI 1 "register_operand") 1973 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))] 1974 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)" 1975 [(set (match_dup 3) (match_dup 2)) 1976 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))] 1977) 1978 1979(define_peephole2 1980 [(match_scratch:SI 3 "r") 1981 (set (match_operand:DI 0 "register_operand") 1982 (zero_extend:DI 1983 (plus:SI 1984 (match_operand:SI 1 "register_operand") 1985 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))] 1986 "aarch64_move_imm (INTVAL (operands[2]), SImode)" 1987 [(set (match_dup 3) (match_dup 2)) 1988 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))] 1989) 1990 1991;; After peephole2 has had a chance to run, split any remaining long 1992;; additions into two add immediates. 1993(define_split 1994 [(set (match_operand:GPI 0 "register_operand") 1995 (plus:GPI 1996 (match_operand:GPI 1 "register_operand") 1997 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))] 1998 "epilogue_completed" 1999 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3))) 2000 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))] 2001 { 2002 HOST_WIDE_INT i = INTVAL (operands[2]); 2003 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff)); 2004 operands[3] = GEN_INT (i - s); 2005 operands[4] = GEN_INT (s); 2006 } 2007) 2008 2009;; Match addition of polynomial offsets that require one temporary, for which 2010;; we can use the early-clobbered destination register. This is a separate 2011;; pattern so that the early clobber doesn't affect register allocation 2012;; for other forms of addition. However, we still need to provide an 2013;; all-register alternative, in case the offset goes out of range after 2014;; elimination. For completeness we might as well provide all GPR-based 2015;; alternatives from the main pattern. 2016;; 2017;; We don't have a pattern for additions requiring two temporaries since at 2018;; present LRA doesn't allow new scratches to be added during elimination. 2019;; Such offsets should be rare anyway. 2020;; 2021;; ??? But if we added LRA support for new scratches, much of the ugliness 2022;; here would go away. We could just handle all polynomial constants in 2023;; this pattern. 2024(define_insn_and_split "*add<mode>3_poly_1" 2025 [(set 2026 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r") 2027 (plus:GPI 2028 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,0,rk,rk") 2029 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uai,Uav,Uat")))] 2030 "TARGET_SVE && operands[0] != stack_pointer_rtx" 2031 "@ 2032 add\\t%<w>0, %<w>1, %2 2033 add\\t%<w>0, %<w>1, %<w>2 2034 sub\\t%<w>0, %<w>1, #%n2 2035 # 2036 * return aarch64_output_sve_scalar_inc_dec (operands[2]); 2037 * return aarch64_output_sve_addvl_addpl (operands[2]); 2038 #" 2039 "&& epilogue_completed 2040 && !reg_overlap_mentioned_p (operands[0], operands[1]) 2041 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)" 2042 [(const_int 0)] 2043 { 2044 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1], 2045 operands[2], operands[0], NULL_RTX); 2046 DONE; 2047 } 2048 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders. 2049 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")] 2050) 2051 2052(define_split 2053 [(set (match_operand:DI 0 "register_operand") 2054 (zero_extend:DI 2055 (plus:SI 2056 (match_operand:SI 1 "register_operand") 2057 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))] 2058 "epilogue_completed" 2059 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3))) 2060 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))] 2061 { 2062 HOST_WIDE_INT i = INTVAL (operands[2]); 2063 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff)); 2064 operands[3] = GEN_INT (i - s); 2065 operands[4] = GEN_INT (s); 2066 operands[5] = gen_lowpart (SImode, operands[0]); 2067 } 2068) 2069 2070(define_expand "addv<mode>4" 2071 [(match_operand:GPI 0 "register_operand") 2072 (match_operand:GPI 1 "register_operand") 2073 (match_operand:GPI 2 "aarch64_plus_operand") 2074 (label_ref (match_operand 3 "" ""))] 2075 "" 2076{ 2077 if (CONST_INT_P (operands[2])) 2078 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1], 2079 operands[2])); 2080 else 2081 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2])); 2082 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2083 2084 DONE; 2085}) 2086 2087(define_expand "uaddv<mode>4" 2088 [(match_operand:GPI 0 "register_operand") 2089 (match_operand:GPI 1 "register_operand") 2090 (match_operand:GPI 2 "register_operand") 2091 (label_ref (match_operand 3 "" ""))] 2092 "" 2093{ 2094 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2])); 2095 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]); 2096 2097 DONE; 2098}) 2099 2100(define_expand "addti3" 2101 [(set (match_operand:TI 0 "register_operand") 2102 (plus:TI (match_operand:TI 1 "register_operand") 2103 (match_operand:TI 2 "aarch64_reg_or_imm")))] 2104 "" 2105{ 2106 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2107 2108 aarch64_addti_scratch_regs (operands[1], operands[2], 2109 &low_dest, &op1_low, &op2_low, 2110 &high_dest, &op1_high, &op2_high); 2111 2112 if (op2_low == const0_rtx) 2113 { 2114 low_dest = op1_low; 2115 if (!aarch64_pluslong_operand (op2_high, DImode)) 2116 op2_high = force_reg (DImode, op2_high); 2117 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high)); 2118 } 2119 else 2120 { 2121 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2122 force_reg (DImode, op2_low))); 2123 emit_insn (gen_adddi3_carryin (high_dest, op1_high, 2124 force_reg (DImode, op2_high))); 2125 } 2126 2127 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2128 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2129 2130 DONE; 2131}) 2132 2133(define_expand "addvti4" 2134 [(match_operand:TI 0 "register_operand") 2135 (match_operand:TI 1 "register_operand") 2136 (match_operand:TI 2 "aarch64_reg_or_imm") 2137 (label_ref (match_operand 3 "" ""))] 2138 "" 2139{ 2140 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2141 2142 aarch64_addti_scratch_regs (operands[1], operands[2], 2143 &low_dest, &op1_low, &op2_low, 2144 &high_dest, &op1_high, &op2_high); 2145 2146 if (op2_low == const0_rtx) 2147 { 2148 low_dest = op1_low; 2149 emit_insn (gen_adddi3_compareV (high_dest, op1_high, 2150 force_reg (DImode, op2_high))); 2151 } 2152 else 2153 { 2154 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2155 force_reg (DImode, op2_low))); 2156 emit_insn (gen_adddi3_carryinV (high_dest, op1_high, 2157 force_reg (DImode, op2_high))); 2158 } 2159 2160 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2161 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2162 2163 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2164 DONE; 2165}) 2166 2167(define_expand "uaddvti4" 2168 [(match_operand:TI 0 "register_operand") 2169 (match_operand:TI 1 "register_operand") 2170 (match_operand:TI 2 "aarch64_reg_or_imm") 2171 (label_ref (match_operand 3 "" ""))] 2172 "" 2173{ 2174 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2175 2176 aarch64_addti_scratch_regs (operands[1], operands[2], 2177 &low_dest, &op1_low, &op2_low, 2178 &high_dest, &op1_high, &op2_high); 2179 2180 if (op2_low == const0_rtx) 2181 { 2182 low_dest = op1_low; 2183 emit_insn (gen_adddi3_compareC (high_dest, op1_high, 2184 force_reg (DImode, op2_high))); 2185 } 2186 else 2187 { 2188 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2189 force_reg (DImode, op2_low))); 2190 emit_insn (gen_adddi3_carryinC (high_dest, op1_high, 2191 force_reg (DImode, op2_high))); 2192 } 2193 2194 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2195 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2196 2197 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]); 2198 DONE; 2199 }) 2200 2201(define_insn "add<mode>3_compare0" 2202 [(set (reg:CC_NZ CC_REGNUM) 2203 (compare:CC_NZ 2204 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk") 2205 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J")) 2206 (const_int 0))) 2207 (set (match_operand:GPI 0 "register_operand" "=r,r,r") 2208 (plus:GPI (match_dup 1) (match_dup 2)))] 2209 "" 2210 "@ 2211 adds\\t%<w>0, %<w>1, %<w>2 2212 adds\\t%<w>0, %<w>1, %2 2213 subs\\t%<w>0, %<w>1, #%n2" 2214 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2215) 2216 2217;; zero_extend version of above 2218(define_insn "*addsi3_compare0_uxtw" 2219 [(set (reg:CC_NZ CC_REGNUM) 2220 (compare:CC_NZ 2221 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk") 2222 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J")) 2223 (const_int 0))) 2224 (set (match_operand:DI 0 "register_operand" "=r,r,r") 2225 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))] 2226 "" 2227 "@ 2228 adds\\t%w0, %w1, %w2 2229 adds\\t%w0, %w1, %2 2230 subs\\t%w0, %w1, #%n2" 2231 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2232) 2233 2234(define_insn "*add<mode>3_compareC_cconly" 2235 [(set (reg:CC_C CC_REGNUM) 2236 (compare:CC_C 2237 (plus:GPI 2238 (match_operand:GPI 0 "register_operand" "r,r,r") 2239 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")) 2240 (match_dup 0)))] 2241 "" 2242 "@ 2243 cmn\\t%<w>0, %<w>1 2244 cmn\\t%<w>0, %1 2245 cmp\\t%<w>0, #%n1" 2246 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2247) 2248 2249(define_insn "add<mode>3_compareC" 2250 [(set (reg:CC_C CC_REGNUM) 2251 (compare:CC_C 2252 (plus:GPI 2253 (match_operand:GPI 1 "register_operand" "rk,rk,rk") 2254 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J")) 2255 (match_dup 1))) 2256 (set (match_operand:GPI 0 "register_operand" "=r,r,r") 2257 (plus:GPI (match_dup 1) (match_dup 2)))] 2258 "" 2259 "@ 2260 adds\\t%<w>0, %<w>1, %<w>2 2261 adds\\t%<w>0, %<w>1, %2 2262 subs\\t%<w>0, %<w>1, #%n2" 2263 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2264) 2265 2266(define_insn "*add<mode>3_compareV_cconly_imm" 2267 [(set (reg:CC_V CC_REGNUM) 2268 (compare:CC_V 2269 (plus:<DWI> 2270 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r")) 2271 (match_operand:<DWI> 1 "const_scalar_int_operand" "")) 2272 (sign_extend:<DWI> 2273 (plus:GPI 2274 (match_dup 0) 2275 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))] 2276 "INTVAL (operands[1]) == INTVAL (operands[2])" 2277 "@ 2278 cmn\\t%<w>0, %<w>1 2279 cmp\\t%<w>0, #%n1" 2280 [(set_attr "type" "alus_imm")] 2281) 2282 2283(define_insn "*add<mode>3_compareV_cconly" 2284 [(set (reg:CC_V CC_REGNUM) 2285 (compare:CC_V 2286 (plus:<DWI> 2287 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r")) 2288 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2289 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))] 2290 "" 2291 "cmn\\t%<w>0, %<w>1" 2292 [(set_attr "type" "alus_sreg")] 2293) 2294 2295(define_insn "add<mode>3_compareV_imm" 2296 [(set (reg:CC_V CC_REGNUM) 2297 (compare:CC_V 2298 (plus:<DWI> 2299 (sign_extend:<DWI> 2300 (match_operand:GPI 1 "register_operand" "rk,rk")) 2301 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")) 2302 (sign_extend:<DWI> 2303 (plus:GPI (match_dup 1) (match_dup 2))))) 2304 (set (match_operand:GPI 0 "register_operand" "=r,r") 2305 (plus:GPI (match_dup 1) (match_dup 2)))] 2306 "" 2307 "@ 2308 adds\\t%<w>0, %<w>1, %<w>2 2309 subs\\t%<w>0, %<w>1, #%n2" 2310 [(set_attr "type" "alus_imm,alus_imm")] 2311) 2312 2313(define_insn "add<mode>3_compareV" 2314 [(set (reg:CC_V CC_REGNUM) 2315 (compare:CC_V 2316 (plus:<DWI> 2317 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk")) 2318 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2319 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2))))) 2320 (set (match_operand:GPI 0 "register_operand" "=r") 2321 (plus:GPI (match_dup 1) (match_dup 2)))] 2322 "" 2323 "adds\\t%<w>0, %<w>1, %<w>2" 2324 [(set_attr "type" "alus_sreg")] 2325) 2326 2327(define_insn "*adds_shift_imm_<mode>" 2328 [(set (reg:CC_NZ CC_REGNUM) 2329 (compare:CC_NZ 2330 (plus:GPI (ASHIFT:GPI 2331 (match_operand:GPI 1 "register_operand" "r") 2332 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 2333 (match_operand:GPI 3 "register_operand" "r")) 2334 (const_int 0))) 2335 (set (match_operand:GPI 0 "register_operand" "=r") 2336 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2)) 2337 (match_dup 3)))] 2338 "" 2339 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2" 2340 [(set_attr "type" "alus_shift_imm")] 2341) 2342 2343(define_insn "*subs_shift_imm_<mode>" 2344 [(set (reg:CC_NZ CC_REGNUM) 2345 (compare:CC_NZ 2346 (minus:GPI (match_operand:GPI 1 "register_operand" "r") 2347 (ASHIFT:GPI 2348 (match_operand:GPI 2 "register_operand" "r") 2349 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n"))) 2350 (const_int 0))) 2351 (set (match_operand:GPI 0 "register_operand" "=r") 2352 (minus:GPI (match_dup 1) 2353 (ASHIFT:GPI (match_dup 2) (match_dup 3))))] 2354 "" 2355 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3" 2356 [(set_attr "type" "alus_shift_imm")] 2357) 2358 2359(define_insn "*adds_mul_imm_<mode>" 2360 [(set (reg:CC_NZ CC_REGNUM) 2361 (compare:CC_NZ 2362 (plus:GPI (mult:GPI 2363 (match_operand:GPI 1 "register_operand" "r") 2364 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n")) 2365 (match_operand:GPI 3 "register_operand" "r")) 2366 (const_int 0))) 2367 (set (match_operand:GPI 0 "register_operand" "=r") 2368 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2)) 2369 (match_dup 3)))] 2370 "" 2371 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2" 2372 [(set_attr "type" "alus_shift_imm")] 2373) 2374 2375(define_insn "*subs_mul_imm_<mode>" 2376 [(set (reg:CC_NZ CC_REGNUM) 2377 (compare:CC_NZ 2378 (minus:GPI (match_operand:GPI 1 "register_operand" "r") 2379 (mult:GPI 2380 (match_operand:GPI 2 "register_operand" "r") 2381 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n"))) 2382 (const_int 0))) 2383 (set (match_operand:GPI 0 "register_operand" "=r") 2384 (minus:GPI (match_dup 1) 2385 (mult:GPI (match_dup 2) (match_dup 3))))] 2386 "" 2387 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3" 2388 [(set_attr "type" "alus_shift_imm")] 2389) 2390 2391(define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>" 2392 [(set (reg:CC_NZ CC_REGNUM) 2393 (compare:CC_NZ 2394 (plus:GPI 2395 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r")) 2396 (match_operand:GPI 2 "register_operand" "rk")) 2397 (const_int 0))) 2398 (set (match_operand:GPI 0 "register_operand" "=r") 2399 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))] 2400 "" 2401 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>" 2402 [(set_attr "type" "alus_ext")] 2403) 2404 2405(define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>" 2406 [(set (reg:CC_NZ CC_REGNUM) 2407 (compare:CC_NZ 2408 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 2409 (ANY_EXTEND:GPI 2410 (match_operand:ALLX 2 "register_operand" "r"))) 2411 (const_int 0))) 2412 (set (match_operand:GPI 0 "register_operand" "=r") 2413 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))] 2414 "" 2415 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>" 2416 [(set_attr "type" "alus_ext")] 2417) 2418 2419(define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>" 2420 [(set (reg:CC_NZ CC_REGNUM) 2421 (compare:CC_NZ 2422 (plus:GPI (ashift:GPI 2423 (ANY_EXTEND:GPI 2424 (match_operand:ALLX 1 "register_operand" "r")) 2425 (match_operand 2 "aarch64_imm3" "Ui3")) 2426 (match_operand:GPI 3 "register_operand" "rk")) 2427 (const_int 0))) 2428 (set (match_operand:GPI 0 "register_operand" "=rk") 2429 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1)) 2430 (match_dup 2)) 2431 (match_dup 3)))] 2432 "" 2433 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2" 2434 [(set_attr "type" "alus_ext")] 2435) 2436 2437(define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>" 2438 [(set (reg:CC_NZ CC_REGNUM) 2439 (compare:CC_NZ 2440 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 2441 (ashift:GPI 2442 (ANY_EXTEND:GPI 2443 (match_operand:ALLX 2 "register_operand" "r")) 2444 (match_operand 3 "aarch64_imm3" "Ui3"))) 2445 (const_int 0))) 2446 (set (match_operand:GPI 0 "register_operand" "=rk") 2447 (minus:GPI (match_dup 1) 2448 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2)) 2449 (match_dup 3))))] 2450 "" 2451 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3" 2452 [(set_attr "type" "alus_ext")] 2453) 2454 2455(define_insn "*adds_<optab><mode>_multp2" 2456 [(set (reg:CC_NZ CC_REGNUM) 2457 (compare:CC_NZ 2458 (plus:GPI (ANY_EXTRACT:GPI 2459 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 2460 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2461 (match_operand 3 "const_int_operand" "n") 2462 (const_int 0)) 2463 (match_operand:GPI 4 "register_operand" "rk")) 2464 (const_int 0))) 2465 (set (match_operand:GPI 0 "register_operand" "=r") 2466 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2)) 2467 (match_dup 3) 2468 (const_int 0)) 2469 (match_dup 4)))] 2470 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])" 2471 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2" 2472 [(set_attr "type" "alus_ext")] 2473) 2474 2475(define_insn "*subs_<optab><mode>_multp2" 2476 [(set (reg:CC_NZ CC_REGNUM) 2477 (compare:CC_NZ 2478 (minus:GPI (match_operand:GPI 4 "register_operand" "rk") 2479 (ANY_EXTRACT:GPI 2480 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 2481 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2482 (match_operand 3 "const_int_operand" "n") 2483 (const_int 0))) 2484 (const_int 0))) 2485 (set (match_operand:GPI 0 "register_operand" "=r") 2486 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI 2487 (mult:GPI (match_dup 1) (match_dup 2)) 2488 (match_dup 3) 2489 (const_int 0))))] 2490 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])" 2491 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2" 2492 [(set_attr "type" "alus_ext")] 2493) 2494 2495(define_insn "*add<mode>3nr_compare0" 2496 [(set (reg:CC_NZ CC_REGNUM) 2497 (compare:CC_NZ 2498 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r") 2499 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")) 2500 (const_int 0)))] 2501 "" 2502 "@ 2503 cmn\\t%<w>0, %<w>1 2504 cmn\\t%<w>0, %1 2505 cmp\\t%<w>0, #%n1" 2506 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2507) 2508 2509(define_insn "aarch64_sub<mode>_compare0" 2510 [(set (reg:CC_NZ CC_REGNUM) 2511 (compare:CC_NZ 2512 (minus:GPI (match_operand:GPI 0 "register_operand" "r") 2513 (match_operand:GPI 1 "aarch64_plus_operand" "r")) 2514 (const_int 0)))] 2515 "" 2516 "cmp\\t%<w>0, %<w>1" 2517 [(set_attr "type" "alus_sreg")] 2518) 2519 2520(define_insn "*compare_neg<mode>" 2521 [(set (reg:CC_Z CC_REGNUM) 2522 (compare:CC_Z 2523 (neg:GPI (match_operand:GPI 0 "register_operand" "r")) 2524 (match_operand:GPI 1 "register_operand" "r")))] 2525 "" 2526 "cmn\\t%<w>1, %<w>0" 2527 [(set_attr "type" "alus_sreg")] 2528) 2529 2530(define_insn "*add_<shift>_<mode>" 2531 [(set (match_operand:GPI 0 "register_operand" "=r") 2532 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r") 2533 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 2534 (match_operand:GPI 3 "register_operand" "r")))] 2535 "" 2536 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2" 2537 [(set_attr "type" "alu_shift_imm")] 2538) 2539 2540;; zero_extend version of above 2541(define_insn "*add_<shift>_si_uxtw" 2542 [(set (match_operand:DI 0 "register_operand" "=r") 2543 (zero_extend:DI 2544 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r") 2545 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 2546 (match_operand:SI 3 "register_operand" "r"))))] 2547 "" 2548 "add\\t%w0, %w3, %w1, <shift> %2" 2549 [(set_attr "type" "alu_shift_imm")] 2550) 2551 2552(define_insn "*add_mul_imm_<mode>" 2553 [(set (match_operand:GPI 0 "register_operand" "=r") 2554 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r") 2555 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n")) 2556 (match_operand:GPI 3 "register_operand" "r")))] 2557 "" 2558 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2" 2559 [(set_attr "type" "alu_shift_imm")] 2560) 2561 2562(define_insn "*add_<optab><ALLX:mode>_<GPI:mode>" 2563 [(set (match_operand:GPI 0 "register_operand" "=rk") 2564 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r")) 2565 (match_operand:GPI 2 "register_operand" "r")))] 2566 "" 2567 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>" 2568 [(set_attr "type" "alu_ext")] 2569) 2570 2571;; zero_extend version of above 2572(define_insn "*add_<optab><SHORT:mode>_si_uxtw" 2573 [(set (match_operand:DI 0 "register_operand" "=rk") 2574 (zero_extend:DI 2575 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r")) 2576 (match_operand:GPI 2 "register_operand" "r"))))] 2577 "" 2578 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>" 2579 [(set_attr "type" "alu_ext")] 2580) 2581 2582(define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>" 2583 [(set (match_operand:GPI 0 "register_operand" "=rk") 2584 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI 2585 (match_operand:ALLX 1 "register_operand" "r")) 2586 (match_operand 2 "aarch64_imm3" "Ui3")) 2587 (match_operand:GPI 3 "register_operand" "r")))] 2588 "" 2589 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2" 2590 [(set_attr "type" "alu_ext")] 2591) 2592 2593;; zero_extend version of above 2594(define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw" 2595 [(set (match_operand:DI 0 "register_operand" "=rk") 2596 (zero_extend:DI 2597 (plus:SI (ashift:SI (ANY_EXTEND:SI 2598 (match_operand:SHORT 1 "register_operand" "r")) 2599 (match_operand 2 "aarch64_imm3" "Ui3")) 2600 (match_operand:SI 3 "register_operand" "r"))))] 2601 "" 2602 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2" 2603 [(set_attr "type" "alu_ext")] 2604) 2605 2606(define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>" 2607 [(set (match_operand:GPI 0 "register_operand" "=rk") 2608 (plus:GPI (mult:GPI (ANY_EXTEND:GPI 2609 (match_operand:ALLX 1 "register_operand" "r")) 2610 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2611 (match_operand:GPI 3 "register_operand" "r")))] 2612 "" 2613 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2" 2614 [(set_attr "type" "alu_ext")] 2615) 2616 2617;; zero_extend version of above 2618(define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw" 2619 [(set (match_operand:DI 0 "register_operand" "=rk") 2620 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI 2621 (match_operand:SHORT 1 "register_operand" "r")) 2622 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2623 (match_operand:SI 3 "register_operand" "r"))))] 2624 "" 2625 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2" 2626 [(set_attr "type" "alu_ext")] 2627) 2628 2629(define_insn "*add_<optab><mode>_multp2" 2630 [(set (match_operand:GPI 0 "register_operand" "=rk") 2631 (plus:GPI (ANY_EXTRACT:GPI 2632 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 2633 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2634 (match_operand 3 "const_int_operand" "n") 2635 (const_int 0)) 2636 (match_operand:GPI 4 "register_operand" "r")))] 2637 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])" 2638 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2" 2639 [(set_attr "type" "alu_ext")] 2640) 2641 2642;; zero_extend version of above 2643(define_insn "*add_<optab>si_multp2_uxtw" 2644 [(set (match_operand:DI 0 "register_operand" "=rk") 2645 (zero_extend:DI 2646 (plus:SI (ANY_EXTRACT:SI 2647 (mult:SI (match_operand:SI 1 "register_operand" "r") 2648 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2649 (match_operand 3 "const_int_operand" "n") 2650 (const_int 0)) 2651 (match_operand:SI 4 "register_operand" "r"))))] 2652 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])" 2653 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2" 2654 [(set_attr "type" "alu_ext")] 2655) 2656 2657(define_expand "add<mode>3_carryin" 2658 [(set (match_operand:GPI 0 "register_operand") 2659 (plus:GPI 2660 (plus:GPI 2661 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0)) 2662 (match_operand:GPI 1 "aarch64_reg_or_zero")) 2663 (match_operand:GPI 2 "aarch64_reg_or_zero")))] 2664 "" 2665 "" 2666) 2667 2668;; Note that add with carry with two zero inputs is matched by cset, 2669;; and that add with carry with one zero input is matched by cinc. 2670 2671(define_insn "*add<mode>3_carryin" 2672 [(set (match_operand:GPI 0 "register_operand" "=r") 2673 (plus:GPI 2674 (plus:GPI 2675 (match_operand:GPI 3 "aarch64_carry_operation" "") 2676 (match_operand:GPI 1 "register_operand" "r")) 2677 (match_operand:GPI 2 "register_operand" "r")))] 2678 "" 2679 "adc\\t%<w>0, %<w>1, %<w>2" 2680 [(set_attr "type" "adc_reg")] 2681) 2682 2683;; zero_extend version of above 2684(define_insn "*addsi3_carryin_uxtw" 2685 [(set (match_operand:DI 0 "register_operand" "=r") 2686 (zero_extend:DI 2687 (plus:SI 2688 (plus:SI 2689 (match_operand:SI 3 "aarch64_carry_operation" "") 2690 (match_operand:SI 1 "register_operand" "r")) 2691 (match_operand:SI 2 "register_operand" "r"))))] 2692 "" 2693 "adc\\t%w0, %w1, %w2" 2694 [(set_attr "type" "adc_reg")] 2695) 2696 2697(define_expand "add<mode>3_carryinC" 2698 [(parallel 2699 [(set (match_dup 3) 2700 (compare:CC_ADC 2701 (plus:<DWI> 2702 (plus:<DWI> 2703 (match_dup 4) 2704 (zero_extend:<DWI> 2705 (match_operand:GPI 1 "register_operand"))) 2706 (zero_extend:<DWI> 2707 (match_operand:GPI 2 "register_operand"))) 2708 (match_dup 6))) 2709 (set (match_operand:GPI 0 "register_operand") 2710 (plus:GPI 2711 (plus:GPI (match_dup 5) (match_dup 1)) 2712 (match_dup 2)))])] 2713 "" 2714{ 2715 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM); 2716 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM); 2717 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx); 2718 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx); 2719 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode) 2720 << GET_MODE_BITSIZE (<MODE>mode), 2721 TImode); 2722}) 2723 2724(define_insn "*add<mode>3_carryinC_zero" 2725 [(set (reg:CC_ADC CC_REGNUM) 2726 (compare:CC_ADC 2727 (plus:<DWI> 2728 (match_operand:<DWI> 2 "aarch64_carry_operation" "") 2729 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2730 (match_operand 4 "const_scalar_int_operand" ""))) 2731 (set (match_operand:GPI 0 "register_operand" "=r") 2732 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "") 2733 (match_dup 1)))] 2734 "rtx_mode_t (operands[4], <DWI>mode) 2735 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))" 2736 "adcs\\t%<w>0, %<w>1, <w>zr" 2737 [(set_attr "type" "adc_reg")] 2738) 2739 2740(define_insn "*add<mode>3_carryinC" 2741 [(set (reg:CC_ADC CC_REGNUM) 2742 (compare:CC_ADC 2743 (plus:<DWI> 2744 (plus:<DWI> 2745 (match_operand:<DWI> 3 "aarch64_carry_operation" "") 2746 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2747 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2748 (match_operand 5 "const_scalar_int_operand" ""))) 2749 (set (match_operand:GPI 0 "register_operand" "=r") 2750 (plus:GPI 2751 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "") 2752 (match_dup 1)) 2753 (match_dup 2)))] 2754 "rtx_mode_t (operands[5], <DWI>mode) 2755 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))" 2756 "adcs\\t%<w>0, %<w>1, %<w>2" 2757 [(set_attr "type" "adc_reg")] 2758) 2759 2760(define_expand "add<mode>3_carryinV" 2761 [(parallel 2762 [(set (reg:CC_V CC_REGNUM) 2763 (compare:CC_V 2764 (plus:<DWI> 2765 (plus:<DWI> 2766 (match_dup 3) 2767 (sign_extend:<DWI> 2768 (match_operand:GPI 1 "register_operand"))) 2769 (sign_extend:<DWI> 2770 (match_operand:GPI 2 "register_operand"))) 2771 (sign_extend:<DWI> 2772 (plus:GPI 2773 (plus:GPI (match_dup 4) (match_dup 1)) 2774 (match_dup 2))))) 2775 (set (match_operand:GPI 0 "register_operand") 2776 (plus:GPI 2777 (plus:GPI (match_dup 4) (match_dup 1)) 2778 (match_dup 2)))])] 2779 "" 2780{ 2781 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM); 2782 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx); 2783 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx); 2784}) 2785 2786(define_insn "*add<mode>3_carryinV_zero" 2787 [(set (reg:CC_V CC_REGNUM) 2788 (compare:CC_V 2789 (plus:<DWI> 2790 (match_operand:<DWI> 2 "aarch64_carry_operation" "") 2791 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2792 (sign_extend:<DWI> 2793 (plus:GPI 2794 (match_operand:GPI 3 "aarch64_carry_operation" "") 2795 (match_dup 1))))) 2796 (set (match_operand:GPI 0 "register_operand" "=r") 2797 (plus:GPI (match_dup 3) (match_dup 1)))] 2798 "" 2799 "adcs\\t%<w>0, %<w>1, <w>zr" 2800 [(set_attr "type" "adc_reg")] 2801) 2802 2803(define_insn "*add<mode>3_carryinV" 2804 [(set (reg:CC_V CC_REGNUM) 2805 (compare:CC_V 2806 (plus:<DWI> 2807 (plus:<DWI> 2808 (match_operand:<DWI> 3 "aarch64_carry_operation" "") 2809 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2810 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2811 (sign_extend:<DWI> 2812 (plus:GPI 2813 (plus:GPI 2814 (match_operand:GPI 4 "aarch64_carry_operation" "") 2815 (match_dup 1)) 2816 (match_dup 2))))) 2817 (set (match_operand:GPI 0 "register_operand" "=r") 2818 (plus:GPI 2819 (plus:GPI (match_dup 4) (match_dup 1)) 2820 (match_dup 2)))] 2821 "" 2822 "adcs\\t%<w>0, %<w>1, %<w>2" 2823 [(set_attr "type" "adc_reg")] 2824) 2825 2826(define_insn "*add_uxt<mode>_shift2" 2827 [(set (match_operand:GPI 0 "register_operand" "=rk") 2828 (plus:GPI (and:GPI 2829 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 2830 (match_operand 2 "aarch64_imm3" "Ui3")) 2831 (match_operand 3 "const_int_operand" "n")) 2832 (match_operand:GPI 4 "register_operand" "r")))] 2833 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0" 2834 "* 2835 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]), 2836 INTVAL (operands[3]))); 2837 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";" 2838 [(set_attr "type" "alu_ext")] 2839) 2840 2841;; zero_extend version of above 2842(define_insn "*add_uxtsi_shift2_uxtw" 2843 [(set (match_operand:DI 0 "register_operand" "=rk") 2844 (zero_extend:DI 2845 (plus:SI (and:SI 2846 (ashift:SI (match_operand:SI 1 "register_operand" "r") 2847 (match_operand 2 "aarch64_imm3" "Ui3")) 2848 (match_operand 3 "const_int_operand" "n")) 2849 (match_operand:SI 4 "register_operand" "r"))))] 2850 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0" 2851 "* 2852 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 2853 INTVAL (operands[3]))); 2854 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";" 2855 [(set_attr "type" "alu_ext")] 2856) 2857 2858(define_insn "*add_uxt<mode>_multp2" 2859 [(set (match_operand:GPI 0 "register_operand" "=rk") 2860 (plus:GPI (and:GPI 2861 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 2862 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2863 (match_operand 3 "const_int_operand" "n")) 2864 (match_operand:GPI 4 "register_operand" "r")))] 2865 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0" 2866 "* 2867 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), 2868 INTVAL (operands[3]))); 2869 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";" 2870 [(set_attr "type" "alu_ext")] 2871) 2872 2873;; zero_extend version of above 2874(define_insn "*add_uxtsi_multp2_uxtw" 2875 [(set (match_operand:DI 0 "register_operand" "=rk") 2876 (zero_extend:DI 2877 (plus:SI (and:SI 2878 (mult:SI (match_operand:SI 1 "register_operand" "r") 2879 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 2880 (match_operand 3 "const_int_operand" "n")) 2881 (match_operand:SI 4 "register_operand" "r"))))] 2882 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0" 2883 "* 2884 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), 2885 INTVAL (operands[3]))); 2886 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";" 2887 [(set_attr "type" "alu_ext")] 2888) 2889 2890(define_insn "subsi3" 2891 [(set (match_operand:SI 0 "register_operand" "=rk") 2892 (minus:SI (match_operand:SI 1 "register_operand" "rk") 2893 (match_operand:SI 2 "register_operand" "r")))] 2894 "" 2895 "sub\\t%w0, %w1, %w2" 2896 [(set_attr "type" "alu_sreg")] 2897) 2898 2899;; zero_extend version of above 2900(define_insn "*subsi3_uxtw" 2901 [(set (match_operand:DI 0 "register_operand" "=rk") 2902 (zero_extend:DI 2903 (minus:SI (match_operand:SI 1 "register_operand" "rk") 2904 (match_operand:SI 2 "register_operand" "r"))))] 2905 "" 2906 "sub\\t%w0, %w1, %w2" 2907 [(set_attr "type" "alu_sreg")] 2908) 2909 2910(define_insn "subdi3" 2911 [(set (match_operand:DI 0 "register_operand" "=rk,w") 2912 (minus:DI (match_operand:DI 1 "register_operand" "rk,w") 2913 (match_operand:DI 2 "register_operand" "r,w")))] 2914 "" 2915 "@ 2916 sub\\t%x0, %x1, %x2 2917 sub\\t%d0, %d1, %d2" 2918 [(set_attr "type" "alu_sreg, neon_sub") 2919 (set_attr "arch" "*,simd")] 2920) 2921 2922(define_expand "subv<GPI:mode>4" 2923 [(match_operand:GPI 0 "register_operand") 2924 (match_operand:GPI 1 "register_operand") 2925 (match_operand:GPI 2 "aarch64_plus_operand") 2926 (label_ref (match_operand 3 "" ""))] 2927 "" 2928{ 2929 if (CONST_INT_P (operands[2])) 2930 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2])); 2931 else 2932 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2])); 2933 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2934 2935 DONE; 2936}) 2937 2938(define_insn "subv<GPI:mode>_insn" 2939 [(set (reg:CC_V CC_REGNUM) 2940 (compare:CC_V 2941 (sign_extend:<DWI> 2942 (minus:GPI 2943 (match_operand:GPI 1 "register_operand" "rk") 2944 (match_operand:GPI 2 "register_operand" "r"))) 2945 (minus:<DWI> (sign_extend:<DWI> (match_dup 1)) 2946 (sign_extend:<DWI> (match_dup 2))))) 2947 (set (match_operand:GPI 0 "register_operand" "=r") 2948 (minus:GPI (match_dup 1) (match_dup 2)))] 2949 "" 2950 "subs\\t%<w>0, %<w>1, %<w>2" 2951 [(set_attr "type" "alus_sreg")] 2952) 2953 2954(define_insn "subv<GPI:mode>_imm" 2955 [(set (reg:CC_V CC_REGNUM) 2956 (compare:CC_V 2957 (sign_extend:<DWI> 2958 (minus:GPI 2959 (match_operand:GPI 1 "register_operand" "rk,rk") 2960 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))) 2961 (minus:<DWI> (sign_extend:<DWI> (match_dup 1)) 2962 (match_dup 2)))) 2963 (set (match_operand:GPI 0 "register_operand" "=r,r") 2964 (minus:GPI (match_dup 1) (match_dup 2)))] 2965 "" 2966 "@ 2967 subs\\t%<w>0, %<w>1, %2 2968 adds\\t%<w>0, %<w>1, #%n2" 2969 [(set_attr "type" "alus_sreg")] 2970) 2971 2972(define_expand "negv<GPI:mode>3" 2973 [(match_operand:GPI 0 "register_operand") 2974 (match_operand:GPI 1 "register_operand") 2975 (label_ref (match_operand 2 "" ""))] 2976 "" 2977 { 2978 emit_insn (gen_negv<mode>_insn (operands[0], operands[1])); 2979 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]); 2980 2981 DONE; 2982 } 2983) 2984 2985(define_insn "negv<GPI:mode>_insn" 2986 [(set (reg:CC_V CC_REGNUM) 2987 (compare:CC_V 2988 (sign_extend:<DWI> 2989 (neg:GPI (match_operand:GPI 1 "register_operand" "r"))) 2990 (neg:<DWI> (sign_extend:<DWI> (match_dup 1))))) 2991 (set (match_operand:GPI 0 "register_operand" "=r") 2992 (neg:GPI (match_dup 1)))] 2993 "" 2994 "negs\\t%<w>0, %<w>1" 2995 [(set_attr "type" "alus_sreg")] 2996) 2997 2998(define_insn "negv<GPI:mode>_cmp_only" 2999 [(set (reg:CC_V CC_REGNUM) 3000 (compare:CC_V 3001 (sign_extend:<DWI> 3002 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))) 3003 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))] 3004 "" 3005 "negs\\t%<w>zr, %<w>0" 3006 [(set_attr "type" "alus_sreg")] 3007) 3008 3009(define_insn "*cmpv<GPI:mode>_insn" 3010 [(set (reg:CC_V CC_REGNUM) 3011 (compare:CC_V 3012 (sign_extend:<DWI> 3013 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r") 3014 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))) 3015 (minus:<DWI> (sign_extend:<DWI> (match_dup 0)) 3016 (sign_extend:<DWI> (match_dup 1)))))] 3017 "" 3018 "@ 3019 cmp\\t%<w>0, %<w>1 3020 cmp\\t%<w>0, %1 3021 cmp\\t%<w>0, #%n1" 3022 [(set_attr "type" "alus_sreg")] 3023) 3024 3025(define_expand "usubv<mode>4" 3026 [(match_operand:GPI 0 "register_operand") 3027 (match_operand:GPI 1 "aarch64_reg_or_zero") 3028 (match_operand:GPI 2 "aarch64_reg_or_zero") 3029 (label_ref (match_operand 3 "" ""))] 3030 "" 3031{ 3032 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2])); 3033 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]); 3034 3035 DONE; 3036}) 3037 3038(define_expand "subti3" 3039 [(set (match_operand:TI 0 "register_operand") 3040 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero") 3041 (match_operand:TI 2 "register_operand")))] 3042 "" 3043{ 3044 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 3045 3046 aarch64_subvti_scratch_regs (operands[1], operands[2], 3047 &low_dest, &op1_low, &op2_low, 3048 &high_dest, &op1_high, &op2_high); 3049 3050 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low)); 3051 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high)); 3052 3053 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 3054 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 3055 DONE; 3056}) 3057 3058(define_expand "subvti4" 3059 [(match_operand:TI 0 "register_operand") 3060 (match_operand:TI 1 "register_operand") 3061 (match_operand:TI 2 "aarch64_reg_or_imm") 3062 (label_ref (match_operand 3 "" ""))] 3063 "" 3064{ 3065 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 3066 3067 aarch64_subvti_scratch_regs (operands[1], operands[2], 3068 &low_dest, &op1_low, &op2_low, 3069 &high_dest, &op1_high, &op2_high); 3070 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low, 3071 high_dest, op1_high, op2_high, false); 3072 3073 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 3074 DONE; 3075}) 3076 3077(define_expand "usubvti4" 3078 [(match_operand:TI 0 "register_operand") 3079 (match_operand:TI 1 "register_operand") 3080 (match_operand:TI 2 "aarch64_reg_or_imm") 3081 (label_ref (match_operand 3 "" ""))] 3082 "" 3083{ 3084 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 3085 3086 aarch64_subvti_scratch_regs (operands[1], operands[2], 3087 &low_dest, &op1_low, &op2_low, 3088 &high_dest, &op1_high, &op2_high); 3089 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low, 3090 high_dest, op1_high, op2_high, true); 3091 3092 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]); 3093 DONE; 3094}) 3095 3096(define_expand "negvti3" 3097 [(match_operand:TI 0 "register_operand") 3098 (match_operand:TI 1 "register_operand") 3099 (label_ref (match_operand 2 "" ""))] 3100 "" 3101 { 3102 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]), 3103 gen_lowpart (DImode, operands[1]))); 3104 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]), 3105 gen_highpart (DImode, operands[1]))); 3106 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]); 3107 3108 DONE; 3109 } 3110) 3111 3112(define_insn "negdi_carryout" 3113 [(set (reg:CC CC_REGNUM) 3114 (compare:CC 3115 (const_int 0) (match_operand:DI 1 "register_operand" "r"))) 3116 (set (match_operand:DI 0 "register_operand" "=r") 3117 (neg:DI (match_dup 1)))] 3118 "" 3119 "negs\\t%0, %1" 3120 [(set_attr "type" "alus_sreg")] 3121) 3122 3123(define_insn "negvdi_carryinV" 3124 [(set (reg:CC_V CC_REGNUM) 3125 (compare:CC_V 3126 (neg:TI (plus:TI 3127 (ltu:TI (reg:CC CC_REGNUM) (const_int 0)) 3128 (sign_extend:TI (match_operand:DI 1 "register_operand" "r")))) 3129 (sign_extend:TI 3130 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0)) 3131 (match_dup 1)))))) 3132 (set (match_operand:DI 0 "register_operand" "=r") 3133 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0)) 3134 (match_dup 1))))] 3135 "" 3136 "ngcs\\t%0, %1" 3137 [(set_attr "type" "alus_sreg")] 3138) 3139 3140(define_insn "*sub<mode>3_compare0" 3141 [(set (reg:CC_NZ CC_REGNUM) 3142 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3143 (match_operand:GPI 2 "register_operand" "r")) 3144 (const_int 0))) 3145 (set (match_operand:GPI 0 "register_operand" "=r") 3146 (minus:GPI (match_dup 1) (match_dup 2)))] 3147 "" 3148 "subs\\t%<w>0, %<w>1, %<w>2" 3149 [(set_attr "type" "alus_sreg")] 3150) 3151 3152;; zero_extend version of above 3153(define_insn "*subsi3_compare0_uxtw" 3154 [(set (reg:CC_NZ CC_REGNUM) 3155 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk") 3156 (match_operand:SI 2 "register_operand" "r")) 3157 (const_int 0))) 3158 (set (match_operand:DI 0 "register_operand" "=r") 3159 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))] 3160 "" 3161 "subs\\t%w0, %w1, %w2" 3162 [(set_attr "type" "alus_sreg")] 3163) 3164 3165(define_insn "sub<mode>3_compare1_imm" 3166 [(set (reg:CC CC_REGNUM) 3167 (compare:CC 3168 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ,rkZ") 3169 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))) 3170 (set (match_operand:GPI 0 "register_operand" "=r,r") 3171 (plus:GPI 3172 (match_dup 1) 3173 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))] 3174 "UINTVAL (operands[2]) == -UINTVAL (operands[3])" 3175 "@ 3176 subs\\t%<w>0, %<w>1, %2 3177 adds\\t%<w>0, %<w>1, #%n2" 3178 [(set_attr "type" "alus_imm")] 3179) 3180 3181(define_insn "sub<mode>3_compare1" 3182 [(set (reg:CC CC_REGNUM) 3183 (compare:CC 3184 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ") 3185 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ"))) 3186 (set (match_operand:GPI 0 "register_operand" "=r") 3187 (minus:GPI (match_dup 1) (match_dup 2)))] 3188 "" 3189 "subs\\t%<w>0, %<w>1, %<w>2" 3190 [(set_attr "type" "alus_sreg")] 3191) 3192 3193(define_peephole2 3194 [(set (match_operand:GPI 0 "aarch64_general_reg") 3195 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero") 3196 (match_operand:GPI 2 "aarch64_reg_or_zero"))) 3197 (set (reg:CC CC_REGNUM) 3198 (compare:CC 3199 (match_dup 1) 3200 (match_dup 2)))] 3201 "!reg_overlap_mentioned_p (operands[0], operands[1]) 3202 && !reg_overlap_mentioned_p (operands[0], operands[2])" 3203 [(const_int 0)] 3204 { 3205 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], 3206 operands[2])); 3207 DONE; 3208 } 3209) 3210 3211;; Same as the above peephole but with the compare and minus in 3212;; swapped order. The restriction on overlap between operand 0 3213;; and operands 1 and 2 doesn't apply here. 3214(define_peephole2 3215 [(set (reg:CC CC_REGNUM) 3216 (compare:CC 3217 (match_operand:GPI 1 "aarch64_reg_or_zero") 3218 (match_operand:GPI 2 "aarch64_reg_or_zero"))) 3219 (set (match_operand:GPI 0 "aarch64_general_reg") 3220 (minus:GPI (match_dup 1) 3221 (match_dup 2)))] 3222 "" 3223 [(const_int 0)] 3224 { 3225 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], 3226 operands[2])); 3227 DONE; 3228 } 3229) 3230 3231(define_peephole2 3232 [(set (match_operand:GPI 0 "aarch64_general_reg") 3233 (plus:GPI (match_operand:GPI 1 "register_operand") 3234 (match_operand:GPI 2 "aarch64_plus_immediate"))) 3235 (set (reg:CC CC_REGNUM) 3236 (compare:CC 3237 (match_dup 1) 3238 (match_operand:GPI 3 "const_int_operand")))] 3239 "!reg_overlap_mentioned_p (operands[0], operands[1]) 3240 && INTVAL (operands[3]) == -INTVAL (operands[2])" 3241 [(const_int 0)] 3242 { 3243 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1], 3244 operands[3], operands[2])); 3245 DONE; 3246 } 3247) 3248 3249;; Same as the above peephole but with the compare and minus in 3250;; swapped order. The restriction on overlap between operand 0 3251;; and operands 1 doesn't apply here. 3252(define_peephole2 3253 [(set (reg:CC CC_REGNUM) 3254 (compare:CC 3255 (match_operand:GPI 1 "register_operand") 3256 (match_operand:GPI 3 "const_int_operand"))) 3257 (set (match_operand:GPI 0 "aarch64_general_reg") 3258 (plus:GPI (match_dup 1) 3259 (match_operand:GPI 2 "aarch64_plus_immediate")))] 3260 "INTVAL (operands[3]) == -INTVAL (operands[2])" 3261 [(const_int 0)] 3262 { 3263 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1], 3264 operands[3], operands[2])); 3265 DONE; 3266 } 3267) 3268 3269(define_insn "*sub_<shift>_<mode>" 3270 [(set (match_operand:GPI 0 "register_operand" "=r") 3271 (minus:GPI (match_operand:GPI 3 "register_operand" "r") 3272 (ASHIFT:GPI 3273 (match_operand:GPI 1 "register_operand" "r") 3274 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 3275 "" 3276 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2" 3277 [(set_attr "type" "alu_shift_imm")] 3278) 3279 3280;; zero_extend version of above 3281(define_insn "*sub_<shift>_si_uxtw" 3282 [(set (match_operand:DI 0 "register_operand" "=r") 3283 (zero_extend:DI 3284 (minus:SI (match_operand:SI 3 "register_operand" "r") 3285 (ASHIFT:SI 3286 (match_operand:SI 1 "register_operand" "r") 3287 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))] 3288 "" 3289 "sub\\t%w0, %w3, %w1, <shift> %2" 3290 [(set_attr "type" "alu_shift_imm")] 3291) 3292 3293(define_insn "*sub_mul_imm_<mode>" 3294 [(set (match_operand:GPI 0 "register_operand" "=r") 3295 (minus:GPI (match_operand:GPI 3 "register_operand" "r") 3296 (mult:GPI 3297 (match_operand:GPI 1 "register_operand" "r") 3298 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))] 3299 "" 3300 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2" 3301 [(set_attr "type" "alu_shift_imm")] 3302) 3303 3304;; zero_extend version of above 3305(define_insn "*sub_mul_imm_si_uxtw" 3306 [(set (match_operand:DI 0 "register_operand" "=r") 3307 (zero_extend:DI 3308 (minus:SI (match_operand:SI 3 "register_operand" "r") 3309 (mult:SI 3310 (match_operand:SI 1 "register_operand" "r") 3311 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))] 3312 "" 3313 "sub\\t%w0, %w3, %w1, lsl %p2" 3314 [(set_attr "type" "alu_shift_imm")] 3315) 3316 3317(define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>" 3318 [(set (match_operand:GPI 0 "register_operand" "=rk") 3319 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3320 (ANY_EXTEND:GPI 3321 (match_operand:ALLX 2 "register_operand" "r"))))] 3322 "" 3323 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>" 3324 [(set_attr "type" "alu_ext")] 3325) 3326 3327;; zero_extend version of above 3328(define_insn "*sub_<optab><SHORT:mode>_si_uxtw" 3329 [(set (match_operand:DI 0 "register_operand" "=rk") 3330 (zero_extend:DI 3331 (minus:SI (match_operand:SI 1 "register_operand" "rk") 3332 (ANY_EXTEND:SI 3333 (match_operand:SHORT 2 "register_operand" "r")))))] 3334 "" 3335 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>" 3336 [(set_attr "type" "alu_ext")] 3337) 3338 3339(define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>" 3340 [(set (match_operand:GPI 0 "register_operand" "=rk") 3341 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3342 (ashift:GPI (ANY_EXTEND:GPI 3343 (match_operand:ALLX 2 "register_operand" "r")) 3344 (match_operand 3 "aarch64_imm3" "Ui3"))))] 3345 "" 3346 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3" 3347 [(set_attr "type" "alu_ext")] 3348) 3349 3350;; zero_extend version of above 3351(define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw" 3352 [(set (match_operand:DI 0 "register_operand" "=rk") 3353 (zero_extend:DI 3354 (minus:SI (match_operand:SI 1 "register_operand" "rk") 3355 (ashift:SI (ANY_EXTEND:SI 3356 (match_operand:SHORT 2 "register_operand" "r")) 3357 (match_operand 3 "aarch64_imm3" "Ui3")))))] 3358 "" 3359 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3" 3360 [(set_attr "type" "alu_ext")] 3361) 3362 3363(define_insn "*sub_<optab><mode>_multp2" 3364 [(set (match_operand:GPI 0 "register_operand" "=rk") 3365 (minus:GPI (match_operand:GPI 4 "register_operand" "rk") 3366 (ANY_EXTRACT:GPI 3367 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3368 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 3369 (match_operand 3 "const_int_operand" "n") 3370 (const_int 0))))] 3371 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])" 3372 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2" 3373 [(set_attr "type" "alu_ext")] 3374) 3375 3376;; zero_extend version of above 3377(define_insn "*sub_<optab>si_multp2_uxtw" 3378 [(set (match_operand:DI 0 "register_operand" "=rk") 3379 (zero_extend:DI 3380 (minus:SI (match_operand:SI 4 "register_operand" "rk") 3381 (ANY_EXTRACT:SI 3382 (mult:SI (match_operand:SI 1 "register_operand" "r") 3383 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 3384 (match_operand 3 "const_int_operand" "n") 3385 (const_int 0)))))] 3386 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])" 3387 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2" 3388 [(set_attr "type" "alu_ext")] 3389) 3390 3391;; The hardware description is op1 + ~op2 + C. 3392;; = op1 + (-op2 + 1) + (1 - !C) 3393;; = op1 - op2 - 1 + 1 - !C 3394;; = op1 - op2 - !C. 3395;; We describe the latter. 3396 3397(define_insn "*sub<mode>3_carryin0" 3398 [(set (match_operand:GPI 0 "register_operand" "=r") 3399 (minus:GPI 3400 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3401 (match_operand:GPI 2 "aarch64_borrow_operation" "")))] 3402 "" 3403 "sbc\\t%<w>0, %<w>1, <w>zr" 3404 [(set_attr "type" "adc_reg")] 3405) 3406 3407;; zero_extend version of the above 3408(define_insn "*subsi3_carryin_uxtw" 3409 [(set (match_operand:DI 0 "register_operand" "=r") 3410 (zero_extend:DI 3411 (minus:SI 3412 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3413 (match_operand:SI 2 "aarch64_borrow_operation" ""))))] 3414 "" 3415 "sbc\\t%w0, %w1, wzr" 3416 [(set_attr "type" "adc_reg")] 3417) 3418 3419(define_expand "sub<mode>3_carryin" 3420 [(set (match_operand:GPI 0 "register_operand") 3421 (minus:GPI 3422 (minus:GPI 3423 (match_operand:GPI 1 "aarch64_reg_or_zero") 3424 (match_operand:GPI 2 "register_operand")) 3425 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))] 3426 "" 3427 "" 3428) 3429 3430(define_insn "*sub<mode>3_carryin" 3431 [(set (match_operand:GPI 0 "register_operand" "=r") 3432 (minus:GPI 3433 (minus:GPI 3434 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3435 (match_operand:GPI 2 "register_operand" "r")) 3436 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3437 3438 "" 3439 "sbc\\t%<w>0, %<w>1, %<w>2" 3440 [(set_attr "type" "adc_reg")] 3441) 3442 3443;; zero_extend version of the above 3444(define_insn "*subsi3_carryin_uxtw" 3445 [(set (match_operand:DI 0 "register_operand" "=r") 3446 (zero_extend:DI 3447 (minus:SI 3448 (minus:SI 3449 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3450 (match_operand:SI 2 "register_operand" "r")) 3451 (match_operand:SI 3 "aarch64_borrow_operation" ""))))] 3452 3453 "" 3454 "sbc\\t%w0, %w1, %w2" 3455 [(set_attr "type" "adc_reg")] 3456) 3457 3458(define_insn "*sub<mode>3_carryin_alt" 3459 [(set (match_operand:GPI 0 "register_operand" "=r") 3460 (minus:GPI 3461 (minus:GPI 3462 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3463 (match_operand:GPI 3 "aarch64_borrow_operation" "")) 3464 (match_operand:GPI 2 "register_operand" "r")))] 3465 "" 3466 "sbc\\t%<w>0, %<w>1, %<w>2" 3467 [(set_attr "type" "adc_reg")] 3468) 3469 3470;; zero_extend version of the above 3471(define_insn "*subsi3_carryin_alt_uxtw" 3472 [(set (match_operand:DI 0 "register_operand" "=r") 3473 (zero_extend:DI 3474 (minus:SI 3475 (minus:SI 3476 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3477 (match_operand:SI 3 "aarch64_borrow_operation" "")) 3478 (match_operand:SI 2 "register_operand" "r"))))] 3479 "" 3480 "sbc\\t%w0, %w1, %w2" 3481 [(set_attr "type" "adc_reg")] 3482) 3483 3484(define_expand "usub<GPI:mode>3_carryinC" 3485 [(parallel 3486 [(set (reg:CC CC_REGNUM) 3487 (compare:CC 3488 (zero_extend:<DWI> 3489 (match_operand:GPI 1 "aarch64_reg_or_zero")) 3490 (plus:<DWI> 3491 (zero_extend:<DWI> 3492 (match_operand:GPI 2 "register_operand")) 3493 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))) 3494 (set (match_operand:GPI 0 "register_operand") 3495 (minus:GPI 3496 (minus:GPI (match_dup 1) (match_dup 2)) 3497 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])] 3498 "" 3499) 3500 3501(define_insn "*usub<GPI:mode>3_carryinC_z1" 3502 [(set (reg:CC CC_REGNUM) 3503 (compare:CC 3504 (const_int 0) 3505 (plus:<DWI> 3506 (zero_extend:<DWI> 3507 (match_operand:GPI 1 "register_operand" "r")) 3508 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))) 3509 (set (match_operand:GPI 0 "register_operand" "=r") 3510 (minus:GPI 3511 (neg:GPI (match_dup 1)) 3512 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3513 "" 3514 "sbcs\\t%<w>0, <w>zr, %<w>1" 3515 [(set_attr "type" "adc_reg")] 3516) 3517 3518(define_insn "*usub<GPI:mode>3_carryinC_z2" 3519 [(set (reg:CC CC_REGNUM) 3520 (compare:CC 3521 (zero_extend:<DWI> 3522 (match_operand:GPI 1 "register_operand" "r")) 3523 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))) 3524 (set (match_operand:GPI 0 "register_operand" "=r") 3525 (minus:GPI 3526 (match_dup 1) 3527 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3528 "" 3529 "sbcs\\t%<w>0, %<w>1, <w>zr" 3530 [(set_attr "type" "adc_reg")] 3531) 3532 3533(define_insn "*usub<GPI:mode>3_carryinC" 3534 [(set (reg:CC CC_REGNUM) 3535 (compare:CC 3536 (zero_extend:<DWI> 3537 (match_operand:GPI 1 "register_operand" "r")) 3538 (plus:<DWI> 3539 (zero_extend:<DWI> 3540 (match_operand:GPI 2 "register_operand" "r")) 3541 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))) 3542 (set (match_operand:GPI 0 "register_operand" "=r") 3543 (minus:GPI 3544 (minus:GPI (match_dup 1) (match_dup 2)) 3545 (match_operand:GPI 4 "aarch64_borrow_operation" "")))] 3546 "" 3547 "sbcs\\t%<w>0, %<w>1, %<w>2" 3548 [(set_attr "type" "adc_reg")] 3549) 3550 3551(define_expand "sub<GPI:mode>3_carryinV" 3552 [(parallel 3553 [(set (reg:CC_V CC_REGNUM) 3554 (compare:CC_V 3555 (minus:<DWI> 3556 (sign_extend:<DWI> 3557 (match_operand:GPI 1 "aarch64_reg_or_zero")) 3558 (plus:<DWI> 3559 (sign_extend:<DWI> 3560 (match_operand:GPI 2 "register_operand")) 3561 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))) 3562 (sign_extend:<DWI> 3563 (minus:GPI (match_dup 1) 3564 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)) 3565 (match_dup 2)))))) 3566 (set (match_operand:GPI 0 "register_operand") 3567 (minus:GPI 3568 (minus:GPI (match_dup 1) (match_dup 2)) 3569 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])] 3570 "" 3571) 3572 3573(define_insn "*sub<mode>3_carryinV_z2" 3574 [(set (reg:CC_V CC_REGNUM) 3575 (compare:CC_V 3576 (minus:<DWI> 3577 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")) 3578 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")) 3579 (sign_extend:<DWI> 3580 (minus:GPI (match_dup 1) 3581 (match_operand:GPI 3 "aarch64_borrow_operation" ""))))) 3582 (set (match_operand:GPI 0 "register_operand" "=r") 3583 (minus:GPI 3584 (match_dup 1) (match_dup 3)))] 3585 "" 3586 "sbcs\\t%<w>0, %<w>1, <w>zr" 3587 [(set_attr "type" "adc_reg")] 3588) 3589 3590(define_insn "*sub<mode>3_carryinV" 3591 [(set (reg:CC_V CC_REGNUM) 3592 (compare:CC_V 3593 (minus:<DWI> 3594 (sign_extend:<DWI> 3595 (match_operand:GPI 1 "register_operand" "r")) 3596 (plus:<DWI> 3597 (sign_extend:<DWI> 3598 (match_operand:GPI 2 "register_operand" "r")) 3599 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))) 3600 (sign_extend:<DWI> 3601 (minus:GPI 3602 (match_dup 1) 3603 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "") 3604 (match_dup 2)))))) 3605 (set (match_operand:GPI 0 "register_operand" "=r") 3606 (minus:GPI 3607 (minus:GPI (match_dup 1) (match_dup 2)) 3608 (match_dup 4)))] 3609 "" 3610 "sbcs\\t%<w>0, %<w>1, %<w>2" 3611 [(set_attr "type" "adc_reg")] 3612) 3613 3614(define_insn "*sub_uxt<mode>_shift2" 3615 [(set (match_operand:GPI 0 "register_operand" "=rk") 3616 (minus:GPI (match_operand:GPI 4 "register_operand" "rk") 3617 (and:GPI 3618 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 3619 (match_operand 2 "aarch64_imm3" "Ui3")) 3620 (match_operand 3 "const_int_operand" "n"))))] 3621 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0" 3622 "* 3623 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 3624 INTVAL (operands[3]))); 3625 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";" 3626 [(set_attr "type" "alu_ext")] 3627) 3628 3629;; zero_extend version of above 3630(define_insn "*sub_uxtsi_shift2_uxtw" 3631 [(set (match_operand:DI 0 "register_operand" "=rk") 3632 (zero_extend:DI 3633 (minus:SI (match_operand:SI 4 "register_operand" "rk") 3634 (and:SI 3635 (ashift:SI (match_operand:SI 1 "register_operand" "r") 3636 (match_operand 2 "aarch64_imm3" "Ui3")) 3637 (match_operand 3 "const_int_operand" "n")))))] 3638 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0" 3639 "* 3640 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 3641 INTVAL (operands[3]))); 3642 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";" 3643 [(set_attr "type" "alu_ext")] 3644) 3645 3646(define_insn "*sub_uxt<mode>_multp2" 3647 [(set (match_operand:GPI 0 "register_operand" "=rk") 3648 (minus:GPI (match_operand:GPI 4 "register_operand" "rk") 3649 (and:GPI 3650 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3651 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 3652 (match_operand 3 "const_int_operand" "n"))))] 3653 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0" 3654 "* 3655 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), 3656 INTVAL (operands[3]))); 3657 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";" 3658 [(set_attr "type" "alu_ext")] 3659) 3660 3661;; zero_extend version of above 3662(define_insn "*sub_uxtsi_multp2_uxtw" 3663 [(set (match_operand:DI 0 "register_operand" "=rk") 3664 (zero_extend:DI 3665 (minus:SI (match_operand:SI 4 "register_operand" "rk") 3666 (and:SI 3667 (mult:SI (match_operand:SI 1 "register_operand" "r") 3668 (match_operand 2 "aarch64_pwr_imm3" "Up3")) 3669 (match_operand 3 "const_int_operand" "n")))))] 3670 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0" 3671 "* 3672 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), 3673 INTVAL (operands[3]))); 3674 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";" 3675 [(set_attr "type" "alu_ext")] 3676) 3677 3678(define_expand "abs<mode>2" 3679 [(match_operand:GPI 0 "register_operand") 3680 (match_operand:GPI 1 "register_operand")] 3681 "" 3682 { 3683 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx); 3684 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx); 3685 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1])); 3686 DONE; 3687 } 3688) 3689 3690(define_insn "neg<mode>2" 3691 [(set (match_operand:GPI 0 "register_operand" "=r,w") 3692 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))] 3693 "" 3694 "@ 3695 neg\\t%<w>0, %<w>1 3696 neg\\t%<rtn>0<vas>, %<rtn>1<vas>" 3697 [(set_attr "type" "alu_sreg, neon_neg<q>") 3698 (set_attr "arch" "*,simd")] 3699) 3700 3701;; zero_extend version of above 3702(define_insn "*negsi2_uxtw" 3703 [(set (match_operand:DI 0 "register_operand" "=r") 3704 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))] 3705 "" 3706 "neg\\t%w0, %w1" 3707 [(set_attr "type" "alu_sreg")] 3708) 3709 3710(define_insn "*ngc<mode>" 3711 [(set (match_operand:GPI 0 "register_operand" "=r") 3712 (minus:GPI 3713 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" "")) 3714 (match_operand:GPI 1 "register_operand" "r")))] 3715 "" 3716 "ngc\\t%<w>0, %<w>1" 3717 [(set_attr "type" "adc_reg")] 3718) 3719 3720(define_insn "*ngcsi_uxtw" 3721 [(set (match_operand:DI 0 "register_operand" "=r") 3722 (zero_extend:DI 3723 (minus:SI 3724 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" "")) 3725 (match_operand:SI 1 "register_operand" "r"))))] 3726 "" 3727 "ngc\\t%w0, %w1" 3728 [(set_attr "type" "adc_reg")] 3729) 3730 3731(define_insn "neg<mode>2_compare0" 3732 [(set (reg:CC_NZ CC_REGNUM) 3733 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r")) 3734 (const_int 0))) 3735 (set (match_operand:GPI 0 "register_operand" "=r") 3736 (neg:GPI (match_dup 1)))] 3737 "" 3738 "negs\\t%<w>0, %<w>1" 3739 [(set_attr "type" "alus_sreg")] 3740) 3741 3742;; zero_extend version of above 3743(define_insn "*negsi2_compare0_uxtw" 3744 [(set (reg:CC_NZ CC_REGNUM) 3745 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r")) 3746 (const_int 0))) 3747 (set (match_operand:DI 0 "register_operand" "=r") 3748 (zero_extend:DI (neg:SI (match_dup 1))))] 3749 "" 3750 "negs\\t%w0, %w1" 3751 [(set_attr "type" "alus_sreg")] 3752) 3753 3754(define_insn "*neg_<shift><mode>3_compare0" 3755 [(set (reg:CC_NZ CC_REGNUM) 3756 (compare:CC_NZ 3757 (neg:GPI (ASHIFT:GPI 3758 (match_operand:GPI 1 "register_operand" "r") 3759 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 3760 (const_int 0))) 3761 (set (match_operand:GPI 0 "register_operand" "=r") 3762 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))] 3763 "" 3764 "negs\\t%<w>0, %<w>1, <shift> %2" 3765 [(set_attr "type" "alus_shift_imm")] 3766) 3767 3768(define_insn "*neg_<shift>_<mode>2" 3769 [(set (match_operand:GPI 0 "register_operand" "=r") 3770 (neg:GPI (ASHIFT:GPI 3771 (match_operand:GPI 1 "register_operand" "r") 3772 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 3773 "" 3774 "neg\\t%<w>0, %<w>1, <shift> %2" 3775 [(set_attr "type" "alu_shift_imm")] 3776) 3777 3778;; zero_extend version of above 3779(define_insn "*neg_<shift>_si2_uxtw" 3780 [(set (match_operand:DI 0 "register_operand" "=r") 3781 (zero_extend:DI 3782 (neg:SI (ASHIFT:SI 3783 (match_operand:SI 1 "register_operand" "r") 3784 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))] 3785 "" 3786 "neg\\t%w0, %w1, <shift> %2" 3787 [(set_attr "type" "alu_shift_imm")] 3788) 3789 3790(define_insn "*neg_mul_imm_<mode>2" 3791 [(set (match_operand:GPI 0 "register_operand" "=r") 3792 (neg:GPI (mult:GPI 3793 (match_operand:GPI 1 "register_operand" "r") 3794 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))] 3795 "" 3796 "neg\\t%<w>0, %<w>1, lsl %p2" 3797 [(set_attr "type" "alu_shift_imm")] 3798) 3799 3800;; zero_extend version of above 3801(define_insn "*neg_mul_imm_si2_uxtw" 3802 [(set (match_operand:DI 0 "register_operand" "=r") 3803 (zero_extend:DI 3804 (neg:SI (mult:SI 3805 (match_operand:SI 1 "register_operand" "r") 3806 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))] 3807 "" 3808 "neg\\t%w0, %w1, lsl %p2" 3809 [(set_attr "type" "alu_shift_imm")] 3810) 3811 3812(define_insn "mul<mode>3" 3813 [(set (match_operand:GPI 0 "register_operand" "=r") 3814 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3815 (match_operand:GPI 2 "register_operand" "r")))] 3816 "" 3817 "mul\\t%<w>0, %<w>1, %<w>2" 3818 [(set_attr "type" "mul")] 3819) 3820 3821;; zero_extend version of above 3822(define_insn "*mulsi3_uxtw" 3823 [(set (match_operand:DI 0 "register_operand" "=r") 3824 (zero_extend:DI 3825 (mult:SI (match_operand:SI 1 "register_operand" "r") 3826 (match_operand:SI 2 "register_operand" "r"))))] 3827 "" 3828 "mul\\t%w0, %w1, %w2" 3829 [(set_attr "type" "mul")] 3830) 3831 3832(define_insn "madd<mode>" 3833 [(set (match_operand:GPI 0 "register_operand" "=r") 3834 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3835 (match_operand:GPI 2 "register_operand" "r")) 3836 (match_operand:GPI 3 "register_operand" "r")))] 3837 "" 3838 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3" 3839 [(set_attr "type" "mla")] 3840) 3841 3842;; zero_extend version of above 3843(define_insn "*maddsi_uxtw" 3844 [(set (match_operand:DI 0 "register_operand" "=r") 3845 (zero_extend:DI 3846 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r") 3847 (match_operand:SI 2 "register_operand" "r")) 3848 (match_operand:SI 3 "register_operand" "r"))))] 3849 "" 3850 "madd\\t%w0, %w1, %w2, %w3" 3851 [(set_attr "type" "mla")] 3852) 3853 3854(define_insn "*msub<mode>" 3855 [(set (match_operand:GPI 0 "register_operand" "=r") 3856 (minus:GPI (match_operand:GPI 3 "register_operand" "r") 3857 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3858 (match_operand:GPI 2 "register_operand" "r"))))] 3859 3860 "" 3861 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3" 3862 [(set_attr "type" "mla")] 3863) 3864 3865;; zero_extend version of above 3866(define_insn "*msubsi_uxtw" 3867 [(set (match_operand:DI 0 "register_operand" "=r") 3868 (zero_extend:DI 3869 (minus:SI (match_operand:SI 3 "register_operand" "r") 3870 (mult:SI (match_operand:SI 1 "register_operand" "r") 3871 (match_operand:SI 2 "register_operand" "r")))))] 3872 3873 "" 3874 "msub\\t%w0, %w1, %w2, %w3" 3875 [(set_attr "type" "mla")] 3876) 3877 3878(define_insn "*mul<mode>_neg" 3879 [(set (match_operand:GPI 0 "register_operand" "=r") 3880 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r")) 3881 (match_operand:GPI 2 "register_operand" "r")))] 3882 3883 "" 3884 "mneg\\t%<w>0, %<w>1, %<w>2" 3885 [(set_attr "type" "mul")] 3886) 3887 3888;; zero_extend version of above 3889(define_insn "*mulsi_neg_uxtw" 3890 [(set (match_operand:DI 0 "register_operand" "=r") 3891 (zero_extend:DI 3892 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r")) 3893 (match_operand:SI 2 "register_operand" "r"))))] 3894 3895 "" 3896 "mneg\\t%w0, %w1, %w2" 3897 [(set_attr "type" "mul")] 3898) 3899 3900(define_insn "<su_optab>mulsidi3" 3901 [(set (match_operand:DI 0 "register_operand" "=r") 3902 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3903 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))] 3904 "" 3905 "<su>mull\\t%0, %w1, %w2" 3906 [(set_attr "type" "<su>mull")] 3907) 3908 3909(define_insn "<su_optab>maddsidi4" 3910 [(set (match_operand:DI 0 "register_operand" "=r") 3911 (plus:DI (mult:DI 3912 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3913 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))) 3914 (match_operand:DI 3 "register_operand" "r")))] 3915 "" 3916 "<su>maddl\\t%0, %w1, %w2, %3" 3917 [(set_attr "type" "<su>mlal")] 3918) 3919 3920(define_insn "<su_optab>msubsidi4" 3921 [(set (match_operand:DI 0 "register_operand" "=r") 3922 (minus:DI 3923 (match_operand:DI 3 "register_operand" "r") 3924 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3925 (ANY_EXTEND:DI 3926 (match_operand:SI 2 "register_operand" "r")))))] 3927 "" 3928 "<su>msubl\\t%0, %w1, %w2, %3" 3929 [(set_attr "type" "<su>mlal")] 3930) 3931 3932(define_insn "*<su_optab>mulsidi_neg" 3933 [(set (match_operand:DI 0 "register_operand" "=r") 3934 (mult:DI (neg:DI 3935 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))) 3936 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))] 3937 "" 3938 "<su>mnegl\\t%0, %w1, %w2" 3939 [(set_attr "type" "<su>mull")] 3940) 3941 3942(define_expand "<su_optab>mulditi3" 3943 [(set (match_operand:TI 0 "register_operand") 3944 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand")) 3945 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))] 3946 "" 3947{ 3948 rtx low = gen_reg_rtx (DImode); 3949 emit_insn (gen_muldi3 (low, operands[1], operands[2])); 3950 3951 rtx high = gen_reg_rtx (DImode); 3952 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2])); 3953 3954 emit_move_insn (gen_lowpart (DImode, operands[0]), low); 3955 emit_move_insn (gen_highpart (DImode, operands[0]), high); 3956 DONE; 3957}) 3958 3959;; The default expansion of multi3 using umuldi3_highpart will perform 3960;; the additions in an order that fails to combine into two madd insns. 3961(define_expand "multi3" 3962 [(set (match_operand:TI 0 "register_operand") 3963 (mult:TI (match_operand:TI 1 "register_operand") 3964 (match_operand:TI 2 "register_operand")))] 3965 "" 3966{ 3967 rtx l0 = gen_reg_rtx (DImode); 3968 rtx l1 = gen_lowpart (DImode, operands[1]); 3969 rtx l2 = gen_lowpart (DImode, operands[2]); 3970 rtx h0 = gen_reg_rtx (DImode); 3971 rtx h1 = gen_highpart (DImode, operands[1]); 3972 rtx h2 = gen_highpart (DImode, operands[2]); 3973 3974 emit_insn (gen_muldi3 (l0, l1, l2)); 3975 emit_insn (gen_umuldi3_highpart (h0, l1, l2)); 3976 emit_insn (gen_madddi (h0, h1, l2, h0)); 3977 emit_insn (gen_madddi (h0, l1, h2, h0)); 3978 3979 emit_move_insn (gen_lowpart (DImode, operands[0]), l0); 3980 emit_move_insn (gen_highpart (DImode, operands[0]), h0); 3981 DONE; 3982}) 3983 3984(define_insn "<su>muldi3_highpart" 3985 [(set (match_operand:DI 0 "register_operand" "=r") 3986 (truncate:DI 3987 (lshiftrt:TI 3988 (mult:TI 3989 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r")) 3990 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r"))) 3991 (const_int 64))))] 3992 "" 3993 "<su>mulh\\t%0, %1, %2" 3994 [(set_attr "type" "<su>mull")] 3995) 3996 3997(define_insn "<su_optab>div<mode>3" 3998 [(set (match_operand:GPI 0 "register_operand" "=r") 3999 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r") 4000 (match_operand:GPI 2 "register_operand" "r")))] 4001 "" 4002 "<su>div\\t%<w>0, %<w>1, %<w>2" 4003 [(set_attr "type" "<su>div")] 4004) 4005 4006;; zero_extend version of above 4007(define_insn "*<su_optab>divsi3_uxtw" 4008 [(set (match_operand:DI 0 "register_operand" "=r") 4009 (zero_extend:DI 4010 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r") 4011 (match_operand:SI 2 "register_operand" "r"))))] 4012 "" 4013 "<su>div\\t%w0, %w1, %w2" 4014 [(set_attr "type" "<su>div")] 4015) 4016 4017;; ------------------------------------------------------------------- 4018;; Comparison insns 4019;; ------------------------------------------------------------------- 4020 4021(define_insn "cmp<mode>" 4022 [(set (reg:CC CC_REGNUM) 4023 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk") 4024 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))] 4025 "" 4026 "@ 4027 cmp\\t%<w>0, %<w>1 4028 cmp\\t%<w>0, %1 4029 cmn\\t%<w>0, #%n1" 4030 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 4031) 4032 4033(define_insn "fcmp<mode>" 4034 [(set (reg:CCFP CC_REGNUM) 4035 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w") 4036 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))] 4037 "TARGET_FLOAT" 4038 "@ 4039 fcmp\\t%<s>0, #0.0 4040 fcmp\\t%<s>0, %<s>1" 4041 [(set_attr "type" "fcmp<s>")] 4042) 4043 4044(define_insn "fcmpe<mode>" 4045 [(set (reg:CCFPE CC_REGNUM) 4046 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w") 4047 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))] 4048 "TARGET_FLOAT" 4049 "@ 4050 fcmpe\\t%<s>0, #0.0 4051 fcmpe\\t%<s>0, %<s>1" 4052 [(set_attr "type" "fcmp<s>")] 4053) 4054 4055(define_insn "*cmp_swp_<shift>_reg<mode>" 4056 [(set (reg:CC_SWP CC_REGNUM) 4057 (compare:CC_SWP (ASHIFT:GPI 4058 (match_operand:GPI 0 "register_operand" "r") 4059 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")) 4060 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))] 4061 "" 4062 "cmp\\t%<w>2, %<w>0, <shift> %1" 4063 [(set_attr "type" "alus_shift_imm")] 4064) 4065 4066(define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>" 4067 [(set (reg:CC_SWP CC_REGNUM) 4068 (compare:CC_SWP (ANY_EXTEND:GPI 4069 (match_operand:ALLX 0 "register_operand" "r")) 4070 (match_operand:GPI 1 "register_operand" "r")))] 4071 "" 4072 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>" 4073 [(set_attr "type" "alus_ext")] 4074) 4075 4076(define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>" 4077 [(set (reg:CC_SWP CC_REGNUM) 4078 (compare:CC_SWP (ashift:GPI 4079 (ANY_EXTEND:GPI 4080 (match_operand:ALLX 0 "register_operand" "r")) 4081 (match_operand 1 "aarch64_imm3" "Ui3")) 4082 (match_operand:GPI 2 "register_operand" "r")))] 4083 "" 4084 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1" 4085 [(set_attr "type" "alus_ext")] 4086) 4087 4088;; ------------------------------------------------------------------- 4089;; Store-flag and conditional select insns 4090;; ------------------------------------------------------------------- 4091 4092(define_expand "cstore<mode>4" 4093 [(set (match_operand:SI 0 "register_operand") 4094 (match_operator:SI 1 "aarch64_comparison_operator" 4095 [(match_operand:GPI 2 "register_operand") 4096 (match_operand:GPI 3 "aarch64_plus_operand")]))] 4097 "" 4098 " 4099 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 4100 operands[3]); 4101 operands[3] = const0_rtx; 4102 " 4103) 4104 4105(define_expand "cstorecc4" 4106 [(set (match_operand:SI 0 "register_operand") 4107 (match_operator 1 "aarch64_comparison_operator_mode" 4108 [(match_operand 2 "cc_register") 4109 (match_operand 3 "const0_operand")]))] 4110 "" 4111"{ 4112 emit_insn (gen_rtx_SET (operands[0], operands[1])); 4113 DONE; 4114}") 4115 4116 4117(define_expand "cstore<mode>4" 4118 [(set (match_operand:SI 0 "register_operand") 4119 (match_operator:SI 1 "aarch64_comparison_operator_mode" 4120 [(match_operand:GPF 2 "register_operand") 4121 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))] 4122 "" 4123 " 4124 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 4125 operands[3]); 4126 operands[3] = const0_rtx; 4127 " 4128) 4129 4130(define_insn "aarch64_cstore<mode>" 4131 [(set (match_operand:ALLI 0 "register_operand" "=r") 4132 (match_operator:ALLI 1 "aarch64_comparison_operator_mode" 4133 [(match_operand 2 "cc_register" "") (const_int 0)]))] 4134 "" 4135 "cset\\t%<w>0, %m1" 4136 [(set_attr "type" "csel")] 4137) 4138 4139;; For a 24-bit immediate CST we can optimize the compare for equality 4140;; and branch sequence from: 4141;; mov x0, #imm1 4142;; movk x0, #imm2, lsl 16 /* x0 contains CST. */ 4143;; cmp x1, x0 4144;; cset x2, <ne,eq> 4145;; into the shorter: 4146;; sub x0, x1, #(CST & 0xfff000) 4147;; subs x0, x0, #(CST & 0x000fff) 4148;; cset x2, <ne, eq>. 4149(define_insn_and_split "*compare_cstore<mode>_insn" 4150 [(set (match_operand:GPI 0 "register_operand" "=r") 4151 (EQL:GPI (match_operand:GPI 1 "register_operand" "r") 4152 (match_operand:GPI 2 "aarch64_imm24" "n"))) 4153 (clobber (reg:CC CC_REGNUM))] 4154 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode) 4155 && !aarch64_plus_operand (operands[2], <MODE>mode) 4156 && !reload_completed" 4157 "#" 4158 "&& true" 4159 [(const_int 0)] 4160 { 4161 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff; 4162 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000; 4163 rtx tmp = gen_reg_rtx (<MODE>mode); 4164 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm))); 4165 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm))); 4166 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM); 4167 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx); 4168 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg)); 4169 DONE; 4170 } 4171 [(set_attr "type" "csel")] 4172) 4173 4174;; zero_extend version of the above 4175(define_insn "*cstoresi_insn_uxtw" 4176 [(set (match_operand:DI 0 "register_operand" "=r") 4177 (zero_extend:DI 4178 (match_operator:SI 1 "aarch64_comparison_operator_mode" 4179 [(match_operand 2 "cc_register" "") (const_int 0)])))] 4180 "" 4181 "cset\\t%w0, %m1" 4182 [(set_attr "type" "csel")] 4183) 4184 4185(define_insn "cstore<mode>_neg" 4186 [(set (match_operand:ALLI 0 "register_operand" "=r") 4187 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode" 4188 [(match_operand 2 "cc_register" "") (const_int 0)])))] 4189 "" 4190 "csetm\\t%<w>0, %m1" 4191 [(set_attr "type" "csel")] 4192) 4193 4194;; zero_extend version of the above 4195(define_insn "*cstoresi_neg_uxtw" 4196 [(set (match_operand:DI 0 "register_operand" "=r") 4197 (zero_extend:DI 4198 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode" 4199 [(match_operand 2 "cc_register" "") (const_int 0)]))))] 4200 "" 4201 "csetm\\t%w0, %m1" 4202 [(set_attr "type" "csel")] 4203) 4204 4205(define_expand "cmov<mode>6" 4206 [(set (match_operand:GPI 0 "register_operand") 4207 (if_then_else:GPI 4208 (match_operator 1 "aarch64_comparison_operator" 4209 [(match_operand:GPI 2 "register_operand") 4210 (match_operand:GPI 3 "aarch64_plus_operand")]) 4211 (match_operand:GPI 4 "register_operand") 4212 (match_operand:GPI 5 "register_operand")))] 4213 "" 4214 " 4215 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 4216 operands[3]); 4217 operands[3] = const0_rtx; 4218 " 4219) 4220 4221(define_expand "cmov<mode>6" 4222 [(set (match_operand:GPF 0 "register_operand") 4223 (if_then_else:GPF 4224 (match_operator 1 "aarch64_comparison_operator" 4225 [(match_operand:GPF 2 "register_operand") 4226 (match_operand:GPF 3 "aarch64_fp_compare_operand")]) 4227 (match_operand:GPF 4 "register_operand") 4228 (match_operand:GPF 5 "register_operand")))] 4229 "" 4230 " 4231 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 4232 operands[3]); 4233 operands[3] = const0_rtx; 4234 " 4235) 4236 4237(define_insn "*cmov<mode>_insn" 4238 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r") 4239 (if_then_else:ALLI 4240 (match_operator 1 "aarch64_comparison_operator" 4241 [(match_operand 2 "cc_register" "") (const_int 0)]) 4242 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1") 4243 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))] 4244 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx) 4245 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))" 4246 ;; Final two alternatives should be unreachable, but included for completeness 4247 "@ 4248 csel\\t%<w>0, %<w>3, %<w>4, %m1 4249 csinv\\t%<w>0, %<w>3, <w>zr, %m1 4250 csinv\\t%<w>0, %<w>4, <w>zr, %M1 4251 csinc\\t%<w>0, %<w>3, <w>zr, %m1 4252 csinc\\t%<w>0, %<w>4, <w>zr, %M1 4253 mov\\t%<w>0, -1 4254 mov\\t%<w>0, 1" 4255 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")] 4256) 4257 4258;; zero_extend version of above 4259(define_insn "*cmovsi_insn_uxtw" 4260 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r") 4261 (zero_extend:DI 4262 (if_then_else:SI 4263 (match_operator 1 "aarch64_comparison_operator" 4264 [(match_operand 2 "cc_register" "") (const_int 0)]) 4265 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1") 4266 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))] 4267 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx) 4268 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))" 4269 ;; Final two alternatives should be unreachable, but included for completeness 4270 "@ 4271 csel\\t%w0, %w3, %w4, %m1 4272 csinv\\t%w0, %w3, wzr, %m1 4273 csinv\\t%w0, %w4, wzr, %M1 4274 csinc\\t%w0, %w3, wzr, %m1 4275 csinc\\t%w0, %w4, wzr, %M1 4276 mov\\t%w0, -1 4277 mov\\t%w0, 1" 4278 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")] 4279) 4280 4281(define_insn "*cmovdi_insn_uxtw" 4282 [(set (match_operand:DI 0 "register_operand" "=r") 4283 (if_then_else:DI 4284 (match_operator 1 "aarch64_comparison_operator" 4285 [(match_operand 2 "cc_register" "") (const_int 0)]) 4286 (zero_extend:DI (match_operand:SI 3 "register_operand" "r")) 4287 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))] 4288 "" 4289 "csel\\t%w0, %w3, %w4, %m1" 4290 [(set_attr "type" "csel")] 4291) 4292 4293(define_insn "*cmov<mode>_insn" 4294 [(set (match_operand:GPF 0 "register_operand" "=w") 4295 (if_then_else:GPF 4296 (match_operator 1 "aarch64_comparison_operator" 4297 [(match_operand 2 "cc_register" "") (const_int 0)]) 4298 (match_operand:GPF 3 "register_operand" "w") 4299 (match_operand:GPF 4 "register_operand" "w")))] 4300 "TARGET_FLOAT" 4301 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1" 4302 [(set_attr "type" "fcsel")] 4303) 4304 4305(define_expand "mov<mode>cc" 4306 [(set (match_operand:ALLI 0 "register_operand") 4307 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator") 4308 (match_operand:ALLI 2 "register_operand") 4309 (match_operand:ALLI 3 "register_operand")))] 4310 "" 4311 { 4312 rtx ccreg; 4313 enum rtx_code code = GET_CODE (operands[1]); 4314 4315 if (code == UNEQ || code == LTGT) 4316 FAIL; 4317 4318 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4319 XEXP (operands[1], 1)); 4320 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4321 } 4322) 4323 4324(define_expand "mov<GPF:mode><GPI:mode>cc" 4325 [(set (match_operand:GPI 0 "register_operand") 4326 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator") 4327 (match_operand:GPF 2 "register_operand") 4328 (match_operand:GPF 3 "register_operand")))] 4329 "" 4330 { 4331 rtx ccreg; 4332 enum rtx_code code = GET_CODE (operands[1]); 4333 4334 if (code == UNEQ || code == LTGT) 4335 FAIL; 4336 4337 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4338 XEXP (operands[1], 1)); 4339 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4340 } 4341) 4342 4343(define_expand "mov<mode>cc" 4344 [(set (match_operand:GPF 0 "register_operand") 4345 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator") 4346 (match_operand:GPF 2 "register_operand") 4347 (match_operand:GPF 3 "register_operand")))] 4348 "" 4349 { 4350 rtx ccreg; 4351 enum rtx_code code = GET_CODE (operands[1]); 4352 4353 if (code == UNEQ || code == LTGT) 4354 FAIL; 4355 4356 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4357 XEXP (operands[1], 1)); 4358 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4359 } 4360) 4361 4362(define_expand "<neg_not_op><mode>cc" 4363 [(set (match_operand:GPI 0 "register_operand") 4364 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator") 4365 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand")) 4366 (match_operand:GPI 3 "register_operand")))] 4367 "" 4368 { 4369 rtx ccreg; 4370 enum rtx_code code = GET_CODE (operands[1]); 4371 4372 if (code == UNEQ || code == LTGT) 4373 FAIL; 4374 4375 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4376 XEXP (operands[1], 1)); 4377 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4378 } 4379) 4380 4381;; CRC32 instructions. 4382(define_insn "aarch64_<crc_variant>" 4383 [(set (match_operand:SI 0 "register_operand" "=r") 4384 (unspec:SI [(match_operand:SI 1 "register_operand" "r") 4385 (match_operand:<crc_mode> 2 "register_operand" "r")] 4386 CRC))] 4387 "TARGET_CRC32" 4388 { 4389 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64) 4390 return "<crc_variant>\\t%w0, %w1, %x2"; 4391 else 4392 return "<crc_variant>\\t%w0, %w1, %w2"; 4393 } 4394 [(set_attr "type" "crc")] 4395) 4396 4397(define_insn "*csinc2<mode>_insn" 4398 [(set (match_operand:GPI 0 "register_operand" "=r") 4399 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "") 4400 (match_operand:GPI 1 "register_operand" "r")))] 4401 "" 4402 "cinc\\t%<w>0, %<w>1, %m2" 4403 [(set_attr "type" "csel")] 4404) 4405 4406(define_insn "csinc3<mode>_insn" 4407 [(set (match_operand:GPI 0 "register_operand" "=r") 4408 (if_then_else:GPI 4409 (match_operand 1 "aarch64_comparison_operation" "") 4410 (plus:GPI (match_operand:GPI 2 "register_operand" "r") 4411 (const_int 1)) 4412 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4413 "" 4414 "csinc\\t%<w>0, %<w>3, %<w>2, %M1" 4415 [(set_attr "type" "csel")] 4416) 4417 4418(define_insn "*csinv3<mode>_insn" 4419 [(set (match_operand:GPI 0 "register_operand" "=r") 4420 (if_then_else:GPI 4421 (match_operand 1 "aarch64_comparison_operation" "") 4422 (not:GPI (match_operand:GPI 2 "register_operand" "r")) 4423 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4424 "" 4425 "csinv\\t%<w>0, %<w>3, %<w>2, %M1" 4426 [(set_attr "type" "csel")] 4427) 4428 4429(define_insn "csneg3_uxtw_insn" 4430 [(set (match_operand:DI 0 "register_operand" "=r") 4431 (zero_extend:DI 4432 (if_then_else:SI 4433 (match_operand 1 "aarch64_comparison_operation" "") 4434 (neg:SI (match_operand:SI 2 "register_operand" "r")) 4435 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))] 4436 "" 4437 "csneg\\t%w0, %w3, %w2, %M1" 4438 [(set_attr "type" "csel")] 4439) 4440 4441(define_insn "csneg3<mode>_insn" 4442 [(set (match_operand:GPI 0 "register_operand" "=r") 4443 (if_then_else:GPI 4444 (match_operand 1 "aarch64_comparison_operation" "") 4445 (neg:GPI (match_operand:GPI 2 "register_operand" "r")) 4446 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4447 "" 4448 "csneg\\t%<w>0, %<w>3, %<w>2, %M1" 4449 [(set_attr "type" "csel")] 4450) 4451 4452;; If X can be loaded by a single CNT[BHWD] instruction, 4453;; 4454;; A = UMAX (B, X) 4455;; 4456;; is equivalent to: 4457;; 4458;; TMP = UQDEC[BHWD] (B, X) 4459;; A = TMP + X 4460;; 4461;; Defining the pattern this way means that: 4462;; 4463;; A = UMAX (B, X) - X 4464;; 4465;; becomes: 4466;; 4467;; TMP1 = UQDEC[BHWD] (B, X) 4468;; TMP2 = TMP1 + X 4469;; A = TMP2 - X 4470;; 4471;; which combine can optimize to: 4472;; 4473;; A = UQDEC[BHWD] (B, X) 4474;; 4475;; We don't use match_operand predicates because the order of the operands 4476;; can vary: the CNT[BHWD] constant will come first if the other operand is 4477;; a simpler constant (such as a CONST_INT), otherwise it will come second. 4478(define_expand "umax<mode>3" 4479 [(set (match_operand:GPI 0 "register_operand") 4480 (umax:GPI (match_operand:GPI 1 "") 4481 (match_operand:GPI 2 "")))] 4482 "TARGET_SVE" 4483 { 4484 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode)) 4485 std::swap (operands[1], operands[2]); 4486 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode)) 4487 FAIL; 4488 rtx temp = gen_reg_rtx (<MODE>mode); 4489 operands[1] = force_reg (<MODE>mode, operands[1]); 4490 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2])); 4491 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2])); 4492 DONE; 4493 } 4494) 4495 4496;; Saturating unsigned subtraction of a CNT[BHWD] immediate. 4497(define_insn "aarch64_uqdec<mode>" 4498 [(set (match_operand:GPI 0 "register_operand" "=r") 4499 (minus:GPI 4500 (umax:GPI (match_operand:GPI 1 "register_operand" "0") 4501 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv")) 4502 (match_dup 2)))] 4503 "TARGET_SVE" 4504 { 4505 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]); 4506 } 4507) 4508 4509;; ------------------------------------------------------------------- 4510;; Logical operations 4511;; ------------------------------------------------------------------- 4512 4513 4514(define_insn_and_split "*aarch64_and<mode>_imm2" 4515 [(set (match_operand:GPI 0 "register_operand" "=rk") 4516 (and:GPI (match_operand:GPI 1 "register_operand" "%r") 4517 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))] 4518 "" 4519 "#" 4520 "true" 4521 [(const_int 0)] 4522 { 4523 HOST_WIDE_INT val = INTVAL (operands[2]); 4524 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val)); 4525 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val)); 4526 4527 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1)); 4528 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2)); 4529 DONE; 4530 } 4531) 4532 4533(define_insn "<optab><mode>3" 4534 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w") 4535 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w") 4536 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))] 4537 "" 4538 "@ 4539 <logical>\\t%<w>0, %<w>1, %<w>2 4540 <logical>\\t%<w>0, %<w>1, %2 4541 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>" 4542 [(set_attr "type" "logic_reg,logic_imm,neon_logic") 4543 (set_attr "arch" "*,*,simd")] 4544) 4545 4546;; zero_extend version of above 4547(define_insn "*<optab>si3_uxtw" 4548 [(set (match_operand:DI 0 "register_operand" "=r,rk") 4549 (zero_extend:DI 4550 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r") 4551 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))] 4552 "" 4553 "@ 4554 <logical>\\t%w0, %w1, %w2 4555 <logical>\\t%w0, %w1, %2" 4556 [(set_attr "type" "logic_reg,logic_imm")] 4557) 4558 4559(define_insn "*and<mode>3_compare0" 4560 [(set (reg:CC_NZ CC_REGNUM) 4561 (compare:CC_NZ 4562 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r") 4563 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>")) 4564 (const_int 0))) 4565 (set (match_operand:GPI 0 "register_operand" "=r,r") 4566 (and:GPI (match_dup 1) (match_dup 2)))] 4567 "" 4568 "@ 4569 ands\\t%<w>0, %<w>1, %<w>2 4570 ands\\t%<w>0, %<w>1, %2" 4571 [(set_attr "type" "logics_reg,logics_imm")] 4572) 4573 4574;; zero_extend version of above 4575(define_insn "*andsi3_compare0_uxtw" 4576 [(set (reg:CC_NZ CC_REGNUM) 4577 (compare:CC_NZ 4578 (and:SI (match_operand:SI 1 "register_operand" "%r,r") 4579 (match_operand:SI 2 "aarch64_logical_operand" "r,K")) 4580 (const_int 0))) 4581 (set (match_operand:DI 0 "register_operand" "=r,r") 4582 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))] 4583 "" 4584 "@ 4585 ands\\t%w0, %w1, %w2 4586 ands\\t%w0, %w1, %2" 4587 [(set_attr "type" "logics_reg,logics_imm")] 4588) 4589 4590(define_insn "*and_<SHIFT:optab><mode>3_compare0" 4591 [(set (reg:CC_NZ CC_REGNUM) 4592 (compare:CC_NZ 4593 (and:GPI (SHIFT:GPI 4594 (match_operand:GPI 1 "register_operand" "r") 4595 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4596 (match_operand:GPI 3 "register_operand" "r")) 4597 (const_int 0))) 4598 (set (match_operand:GPI 0 "register_operand" "=r") 4599 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))] 4600 "" 4601 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4602 [(set_attr "type" "logics_shift_imm")] 4603) 4604 4605;; zero_extend version of above 4606(define_insn "*and_<SHIFT:optab>si3_compare0_uxtw" 4607 [(set (reg:CC_NZ CC_REGNUM) 4608 (compare:CC_NZ 4609 (and:SI (SHIFT:SI 4610 (match_operand:SI 1 "register_operand" "r") 4611 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4612 (match_operand:SI 3 "register_operand" "r")) 4613 (const_int 0))) 4614 (set (match_operand:DI 0 "register_operand" "=r") 4615 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2)) 4616 (match_dup 3))))] 4617 "" 4618 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4619 [(set_attr "type" "logics_shift_imm")] 4620) 4621 4622(define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3" 4623 [(set (match_operand:GPI 0 "register_operand" "=r") 4624 (LOGICAL:GPI (SHIFT:GPI 4625 (match_operand:GPI 1 "register_operand" "r") 4626 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4627 (match_operand:GPI 3 "register_operand" "r")))] 4628 "" 4629 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4630 [(set_attr "type" "logic_shift_imm")] 4631) 4632 4633(define_insn "*<optab>_rol<mode>3" 4634 [(set (match_operand:GPI 0 "register_operand" "=r") 4635 (LOGICAL:GPI (rotate:GPI 4636 (match_operand:GPI 1 "register_operand" "r") 4637 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4638 (match_operand:GPI 3 "register_operand" "r")))] 4639 "" 4640 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)" 4641 [(set_attr "type" "logic_shift_imm")] 4642) 4643 4644;; zero_extend versions of above 4645(define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw" 4646 [(set (match_operand:DI 0 "register_operand" "=r") 4647 (zero_extend:DI 4648 (LOGICAL:SI (SHIFT:SI 4649 (match_operand:SI 1 "register_operand" "r") 4650 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4651 (match_operand:SI 3 "register_operand" "r"))))] 4652 "" 4653 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4654 [(set_attr "type" "logic_shift_imm")] 4655) 4656 4657(define_insn "*<optab>_rolsi3_uxtw" 4658 [(set (match_operand:DI 0 "register_operand" "=r") 4659 (zero_extend:DI 4660 (LOGICAL:SI (rotate:SI 4661 (match_operand:SI 1 "register_operand" "r") 4662 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4663 (match_operand:SI 3 "register_operand" "r"))))] 4664 "" 4665 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)" 4666 [(set_attr "type" "logic_shift_imm")] 4667) 4668 4669(define_insn "one_cmpl<mode>2" 4670 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4671 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))] 4672 "" 4673 "@ 4674 mvn\\t%<w>0, %<w>1 4675 mvn\\t%0.8b, %1.8b" 4676 [(set_attr "type" "logic_reg,neon_logic") 4677 (set_attr "arch" "*,simd")] 4678) 4679 4680(define_insn "*one_cmpl_<optab><mode>2" 4681 [(set (match_operand:GPI 0 "register_operand" "=r") 4682 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r") 4683 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 4684 "" 4685 "mvn\\t%<w>0, %<w>1, <shift> %2" 4686 [(set_attr "type" "logic_shift_imm")] 4687) 4688 4689;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b). 4690 4691(define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3" 4692 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4693 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w")) 4694 (match_operand:GPI 2 "register_operand" "r,w")))] 4695 "" 4696 "@ 4697 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1 4698 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>" 4699 [(set_attr "type" "logic_reg,neon_logic") 4700 (set_attr "arch" "*,simd")] 4701) 4702 4703(define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze" 4704 [(set (match_operand:DI 0 "register_operand" "=r") 4705 (zero_extend:DI 4706 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r")) 4707 (match_operand:SI 2 "register_operand" "r"))))] 4708 "" 4709 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1" 4710 [(set_attr "type" "logic_reg")] 4711) 4712 4713(define_insn "*xor_one_cmplsidi3_ze" 4714 [(set (match_operand:DI 0 "register_operand" "=r") 4715 (zero_extend:DI 4716 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r") 4717 (match_operand:SI 2 "register_operand" "r")))))] 4718 "" 4719 "eon\\t%w0, %w1, %w2" 4720 [(set_attr "type" "logic_reg")] 4721) 4722 4723;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)). 4724;; eon does not operate on SIMD registers so the vector variant must be split. 4725(define_insn_and_split "*xor_one_cmpl<mode>3" 4726 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4727 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w") 4728 (match_operand:GPI 2 "register_operand" "r,w"))))] 4729 "" 4730 "@ 4731 eon\\t%<w>0, %<w>1, %<w>2 4732 #" 4733 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers. 4734 [(set (match_operand:GPI 0 "register_operand" "=w") 4735 (xor:GPI (match_operand:GPI 1 "register_operand" "w") 4736 (match_operand:GPI 2 "register_operand" "w"))) 4737 (set (match_dup 0) (not:GPI (match_dup 0)))] 4738 "" 4739 [(set_attr "type" "logic_reg,multiple") 4740 (set_attr "arch" "*,simd")] 4741) 4742 4743(define_insn "*and_one_cmpl<mode>3_compare0" 4744 [(set (reg:CC_NZ CC_REGNUM) 4745 (compare:CC_NZ 4746 (and:GPI (not:GPI 4747 (match_operand:GPI 1 "register_operand" "r")) 4748 (match_operand:GPI 2 "register_operand" "r")) 4749 (const_int 0))) 4750 (set (match_operand:GPI 0 "register_operand" "=r") 4751 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))] 4752 "" 4753 "bics\\t%<w>0, %<w>2, %<w>1" 4754 [(set_attr "type" "logics_reg")] 4755) 4756 4757;; zero_extend version of above 4758(define_insn "*and_one_cmplsi3_compare0_uxtw" 4759 [(set (reg:CC_NZ CC_REGNUM) 4760 (compare:CC_NZ 4761 (and:SI (not:SI 4762 (match_operand:SI 1 "register_operand" "r")) 4763 (match_operand:SI 2 "register_operand" "r")) 4764 (const_int 0))) 4765 (set (match_operand:DI 0 "register_operand" "=r") 4766 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))] 4767 "" 4768 "bics\\t%w0, %w2, %w1" 4769 [(set_attr "type" "logics_reg")] 4770) 4771 4772(define_insn "*and_one_cmpl<mode>3_compare0_no_reuse" 4773 [(set (reg:CC_NZ CC_REGNUM) 4774 (compare:CC_NZ 4775 (and:GPI (not:GPI 4776 (match_operand:GPI 0 "register_operand" "r")) 4777 (match_operand:GPI 1 "register_operand" "r")) 4778 (const_int 0)))] 4779 "" 4780 "bics\\t<w>zr, %<w>1, %<w>0" 4781 [(set_attr "type" "logics_reg")] 4782) 4783 4784(define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3" 4785 [(set (match_operand:GPI 0 "register_operand" "=r") 4786 (LOGICAL:GPI (not:GPI 4787 (SHIFT:GPI 4788 (match_operand:GPI 1 "register_operand" "r") 4789 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 4790 (match_operand:GPI 3 "register_operand" "r")))] 4791 "" 4792 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4793 [(set_attr "type" "logic_shift_imm")] 4794) 4795 4796(define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt" 4797 [(set (match_operand:GPI 0 "register_operand" "=r") 4798 (not:GPI (xor:GPI 4799 (SHIFT:GPI 4800 (match_operand:GPI 1 "register_operand" "r") 4801 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4802 (match_operand:GPI 3 "register_operand" "r"))))] 4803 "" 4804 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4805 [(set_attr "type" "logic_shift_imm")] 4806) 4807 4808;; Zero-extend version of the above. 4809(define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze" 4810 [(set (match_operand:DI 0 "register_operand" "=r") 4811 (zero_extend:DI 4812 (not:SI (xor:SI 4813 (SHIFT:SI 4814 (match_operand:SI 1 "register_operand" "r") 4815 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4816 (match_operand:SI 3 "register_operand" "r")))))] 4817 "" 4818 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4819 [(set_attr "type" "logic_shift_imm")] 4820) 4821 4822(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0" 4823 [(set (reg:CC_NZ CC_REGNUM) 4824 (compare:CC_NZ 4825 (and:GPI (not:GPI 4826 (SHIFT:GPI 4827 (match_operand:GPI 1 "register_operand" "r") 4828 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 4829 (match_operand:GPI 3 "register_operand" "r")) 4830 (const_int 0))) 4831 (set (match_operand:GPI 0 "register_operand" "=r") 4832 (and:GPI (not:GPI 4833 (SHIFT:GPI 4834 (match_dup 1) (match_dup 2))) (match_dup 3)))] 4835 "" 4836 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4837 [(set_attr "type" "logics_shift_imm")] 4838) 4839 4840;; zero_extend version of above 4841(define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw" 4842 [(set (reg:CC_NZ CC_REGNUM) 4843 (compare:CC_NZ 4844 (and:SI (not:SI 4845 (SHIFT:SI 4846 (match_operand:SI 1 "register_operand" "r") 4847 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))) 4848 (match_operand:SI 3 "register_operand" "r")) 4849 (const_int 0))) 4850 (set (match_operand:DI 0 "register_operand" "=r") 4851 (zero_extend:DI (and:SI 4852 (not:SI 4853 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))] 4854 "" 4855 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4856 [(set_attr "type" "logics_shift_imm")] 4857) 4858 4859(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse" 4860 [(set (reg:CC_NZ CC_REGNUM) 4861 (compare:CC_NZ 4862 (and:GPI (not:GPI 4863 (SHIFT:GPI 4864 (match_operand:GPI 0 "register_operand" "r") 4865 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))) 4866 (match_operand:GPI 2 "register_operand" "r")) 4867 (const_int 0)))] 4868 "" 4869 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1" 4870 [(set_attr "type" "logics_shift_imm")] 4871) 4872 4873(define_insn "clz<mode>2" 4874 [(set (match_operand:GPI 0 "register_operand" "=r") 4875 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))] 4876 "" 4877 "clz\\t%<w>0, %<w>1" 4878 [(set_attr "type" "clz")] 4879) 4880 4881(define_expand "ffs<mode>2" 4882 [(match_operand:GPI 0 "register_operand") 4883 (match_operand:GPI 1 "register_operand")] 4884 "" 4885 { 4886 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx); 4887 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx); 4888 4889 emit_insn (gen_rbit<mode>2 (operands[0], operands[1])); 4890 emit_insn (gen_clz<mode>2 (operands[0], operands[0])); 4891 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx)); 4892 DONE; 4893 } 4894) 4895 4896;; Pop count be done via the "CNT" instruction in AdvSIMD. 4897;; 4898;; MOV v.1d, x0 4899;; CNT v1.8b, v.8b 4900;; ADDV b2, v1.8b 4901;; MOV w0, v2.b[0] 4902 4903(define_expand "popcount<mode>2" 4904 [(match_operand:GPI 0 "register_operand") 4905 (match_operand:GPI 1 "register_operand")] 4906 "TARGET_SIMD" 4907{ 4908 rtx v = gen_reg_rtx (V8QImode); 4909 rtx v1 = gen_reg_rtx (V8QImode); 4910 rtx in = operands[1]; 4911 rtx out = operands[0]; 4912 if(<MODE>mode == SImode) 4913 { 4914 rtx tmp; 4915 tmp = gen_reg_rtx (DImode); 4916 /* If we have SImode, zero extend to DImode, pop count does 4917 not change if we have extra zeros. */ 4918 emit_insn (gen_zero_extendsidi2 (tmp, in)); 4919 in = tmp; 4920 } 4921 emit_move_insn (v, gen_lowpart (V8QImode, in)); 4922 emit_insn (gen_popcountv8qi2 (v1, v)); 4923 emit_insn (gen_aarch64_zero_extend<mode>_reduc_plus_v8qi (out, v1)); 4924 DONE; 4925}) 4926 4927(define_insn "clrsb<mode>2" 4928 [(set (match_operand:GPI 0 "register_operand" "=r") 4929 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))] 4930 "" 4931 "cls\\t%<w>0, %<w>1" 4932 [(set_attr "type" "clz")] 4933) 4934 4935(define_insn "rbit<mode>2" 4936 [(set (match_operand:GPI 0 "register_operand" "=r") 4937 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))] 4938 "" 4939 "rbit\\t%<w>0, %<w>1" 4940 [(set_attr "type" "rbit")] 4941) 4942 4943;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC 4944;; it is unlikely to fold with any other operation, so keep this as a CTZ 4945;; expression and split after reload to enable scheduling them apart if 4946;; needed. 4947 4948(define_insn_and_split "ctz<mode>2" 4949 [(set (match_operand:GPI 0 "register_operand" "=r") 4950 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))] 4951 "" 4952 "#" 4953 "reload_completed" 4954 [(const_int 0)] 4955 " 4956 emit_insn (gen_rbit<mode>2 (operands[0], operands[1])); 4957 emit_insn (gen_clz<mode>2 (operands[0], operands[0])); 4958 DONE; 4959") 4960 4961(define_insn "*and<mode>_compare0" 4962 [(set (reg:CC_NZ CC_REGNUM) 4963 (compare:CC_NZ 4964 (match_operand:SHORT 0 "register_operand" "r") 4965 (const_int 0)))] 4966 "" 4967 "tst\\t%<w>0, <short_mask>" 4968 [(set_attr "type" "alus_imm")] 4969) 4970 4971(define_insn "*ands<GPI:mode>_compare0" 4972 [(set (reg:CC_NZ CC_REGNUM) 4973 (compare:CC_NZ 4974 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r")) 4975 (const_int 0))) 4976 (set (match_operand:GPI 0 "register_operand" "=r") 4977 (zero_extend:GPI (match_dup 1)))] 4978 "" 4979 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>" 4980 [(set_attr "type" "alus_imm")] 4981) 4982 4983(define_insn "*and<mode>3nr_compare0" 4984 [(set (reg:CC_NZ CC_REGNUM) 4985 (compare:CC_NZ 4986 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r") 4987 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>")) 4988 (const_int 0)))] 4989 "" 4990 "@ 4991 tst\\t%<w>0, %<w>1 4992 tst\\t%<w>0, %1" 4993 [(set_attr "type" "logics_reg,logics_imm")] 4994) 4995 4996(define_split 4997 [(set (reg:CC_NZ CC_REGNUM) 4998 (compare:CC_NZ 4999 (and:GPI (match_operand:GPI 0 "register_operand") 5000 (match_operand:GPI 1 "aarch64_mov_imm_operand")) 5001 (const_int 0))) 5002 (clobber (match_operand:SI 2 "register_operand"))] 5003 "" 5004 [(set (match_dup 2) (match_dup 1)) 5005 (set (reg:CC_NZ CC_REGNUM) 5006 (compare:CC_NZ 5007 (and:GPI (match_dup 0) 5008 (match_dup 2)) 5009 (const_int 0)))] 5010) 5011 5012(define_insn "*and<mode>3nr_compare0_zextract" 5013 [(set (reg:CC_NZ CC_REGNUM) 5014 (compare:CC_NZ 5015 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r") 5016 (match_operand:GPI 1 "const_int_operand" "n") 5017 (match_operand:GPI 2 "const_int_operand" "n")) 5018 (const_int 0)))] 5019 "INTVAL (operands[1]) > 0 5020 && ((INTVAL (operands[1]) + INTVAL (operands[2])) 5021 <= GET_MODE_BITSIZE (<MODE>mode)) 5022 && aarch64_bitmask_imm ( 5023 UINTVAL (aarch64_mask_from_zextract_ops (operands[1], 5024 operands[2])), 5025 <MODE>mode)" 5026 { 5027 operands[1] 5028 = aarch64_mask_from_zextract_ops (operands[1], operands[2]); 5029 return "tst\\t%<w>0, %1"; 5030 } 5031 [(set_attr "type" "logics_shift_imm")] 5032) 5033 5034(define_insn "*and_<SHIFT:optab><mode>3nr_compare0" 5035 [(set (reg:CC_NZ CC_REGNUM) 5036 (compare:CC_NZ 5037 (and:GPI (SHIFT:GPI 5038 (match_operand:GPI 0 "register_operand" "r") 5039 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")) 5040 (match_operand:GPI 2 "register_operand" "r")) 5041 (const_int 0)))] 5042 "" 5043 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1" 5044 [(set_attr "type" "logics_shift_imm")] 5045) 5046 5047(define_split 5048 [(set (reg:CC_NZ CC_REGNUM) 5049 (compare:CC_NZ 5050 (and:GPI (SHIFT:GPI 5051 (match_operand:GPI 0 "register_operand") 5052 (match_operand:QI 1 "aarch64_shift_imm_<mode>")) 5053 (match_operand:GPI 2 "aarch64_mov_imm_operand")) 5054 (const_int 0))) 5055 (clobber (match_operand:SI 3 "register_operand"))] 5056 "" 5057 [(set (match_dup 3) (match_dup 2)) 5058 (set (reg:CC_NZ CC_REGNUM) 5059 (compare:CC_NZ 5060 (and:GPI (SHIFT:GPI 5061 (match_dup 0) 5062 (match_dup 1)) 5063 (match_dup 3)) 5064 (const_int 0)))] 5065) 5066 5067;; ------------------------------------------------------------------- 5068;; Shifts 5069;; ------------------------------------------------------------------- 5070 5071(define_expand "<optab><mode>3" 5072 [(set (match_operand:GPI 0 "register_operand") 5073 (ASHIFT:GPI (match_operand:GPI 1 "register_operand") 5074 (match_operand:QI 2 "aarch64_reg_or_imm")))] 5075 "" 5076 { 5077 if (CONST_INT_P (operands[2])) 5078 { 5079 operands[2] = GEN_INT (INTVAL (operands[2]) 5080 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5081 5082 if (operands[2] == const0_rtx) 5083 { 5084 emit_insn (gen_mov<mode> (operands[0], operands[1])); 5085 DONE; 5086 } 5087 } 5088 } 5089) 5090 5091(define_expand "ashl<mode>3" 5092 [(set (match_operand:SHORT 0 "register_operand") 5093 (ashift:SHORT (match_operand:SHORT 1 "register_operand") 5094 (match_operand:QI 2 "const_int_operand")))] 5095 "" 5096 { 5097 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode)); 5098 5099 if (operands[2] == const0_rtx) 5100 { 5101 emit_insn (gen_mov<mode> (operands[0], operands[1])); 5102 DONE; 5103 } 5104 } 5105) 5106 5107(define_expand "rotr<mode>3" 5108 [(set (match_operand:GPI 0 "register_operand") 5109 (rotatert:GPI (match_operand:GPI 1 "register_operand") 5110 (match_operand:QI 2 "aarch64_reg_or_imm")))] 5111 "" 5112 { 5113 if (CONST_INT_P (operands[2])) 5114 { 5115 operands[2] = GEN_INT (INTVAL (operands[2]) 5116 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5117 5118 if (operands[2] == const0_rtx) 5119 { 5120 emit_insn (gen_mov<mode> (operands[0], operands[1])); 5121 DONE; 5122 } 5123 } 5124 } 5125) 5126 5127(define_expand "rotl<mode>3" 5128 [(set (match_operand:GPI 0 "register_operand") 5129 (rotatert:GPI (match_operand:GPI 1 "register_operand") 5130 (match_operand:QI 2 "aarch64_reg_or_imm")))] 5131 "" 5132 { 5133 /* (SZ - cnt) % SZ == -cnt % SZ */ 5134 if (CONST_INT_P (operands[2])) 5135 { 5136 operands[2] = GEN_INT ((-INTVAL (operands[2])) 5137 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5138 if (operands[2] == const0_rtx) 5139 { 5140 emit_insn (gen_mov<mode> (operands[0], operands[1])); 5141 DONE; 5142 } 5143 } 5144 else 5145 operands[2] = expand_simple_unop (QImode, NEG, operands[2], 5146 NULL_RTX, 1); 5147 } 5148) 5149 5150;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments 5151;; they truncate the shift/rotate amount by the size of the registers they 5152;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away 5153;; such redundant masking instructions. GCC can do that automatically when 5154;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD 5155;; because some of the SISD shift alternatives don't perform this truncations. 5156;; So this pattern exists to catch such cases. 5157 5158(define_insn "*aarch64_<optab>_reg_<mode>3_mask1" 5159 [(set (match_operand:GPI 0 "register_operand" "=r") 5160 (SHIFT:GPI 5161 (match_operand:GPI 1 "register_operand" "r") 5162 (match_operator 4 "subreg_lowpart_operator" 5163 [(and:GPI (match_operand:GPI 2 "register_operand" "r") 5164 (match_operand 3 "const_int_operand" "n"))])))] 5165 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0" 5166 "<shift>\t%<w>0, %<w>1, %<w>2" 5167 [(set_attr "type" "shift_reg")] 5168) 5169 5170(define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2" 5171 [(set (match_operand:GPI 0 "register_operand" "=&r") 5172 (SHIFT:GPI 5173 (match_operand:GPI 1 "register_operand" "r") 5174 (match_operator 4 "subreg_lowpart_operator" 5175 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r") 5176 (match_operand 3 "const_int_operand" "n")))])))] 5177 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)" 5178 "#" 5179 "&& true" 5180 [(const_int 0)] 5181 { 5182 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5183 : lowpart_subreg (SImode, operands[0], <MODE>mode)); 5184 emit_insn (gen_negsi2 (tmp, operands[2])); 5185 5186 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]); 5187 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op, 5188 SUBREG_BYTE (operands[4])); 5189 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp)); 5190 DONE; 5191 } 5192) 5193 5194(define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask" 5195 [(set (match_operand:GPI 0 "register_operand" "=&r") 5196 (ashift:GPI 5197 (match_operand:GPI 1 "register_operand" "r") 5198 (minus:QI (match_operand 2 "const_int_operand" "n") 5199 (match_operator 5 "subreg_lowpart_operator" 5200 [(and:SI (match_operand:SI 3 "register_operand" "r") 5201 (match_operand 4 "const_int_operand" "n"))]))))] 5202 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0) 5203 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)" 5204 "#" 5205 "&& true" 5206 [(const_int 0)] 5207 { 5208 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5209 : operands[0]); 5210 5211 emit_insn (gen_negsi2 (tmp, operands[3])); 5212 5213 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]); 5214 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op, 5215 SUBREG_BYTE (operands[5])); 5216 5217 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp)); 5218 DONE; 5219 } 5220) 5221 5222(define_insn "*aarch64_<optab>_reg_di3_mask2" 5223 [(set (match_operand:DI 0 "register_operand" "=r") 5224 (SHIFT:DI 5225 (match_operand:DI 1 "register_operand" "r") 5226 (match_operator 4 "subreg_lowpart_operator" 5227 [(and:SI (match_operand:SI 2 "register_operand" "r") 5228 (match_operand 3 "const_int_operand" "n"))])))] 5229 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)" 5230{ 5231 rtx xop[3]; 5232 xop[0] = operands[0]; 5233 xop[1] = operands[1]; 5234 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]); 5235 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop); 5236 return ""; 5237} 5238 [(set_attr "type" "shift_reg")] 5239) 5240 5241(define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3" 5242 [(set (match_operand:GPI 0 "register_operand" "=&r") 5243 (ASHIFT:GPI 5244 (match_operand:GPI 1 "register_operand" "r") 5245 (minus:QI (match_operand 2 "const_int_operand" "n") 5246 (match_operand:QI 3 "register_operand" "r"))))] 5247 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)" 5248 "#" 5249 "&& true" 5250 [(const_int 0)] 5251 { 5252 rtx subreg_tmp = gen_lowpart (SImode, operands[3]); 5253 5254 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5255 : gen_lowpart (SImode, operands[0])); 5256 5257 emit_insn (gen_negsi2 (tmp, subreg_tmp)); 5258 5259 rtx and_op = gen_rtx_AND (SImode, tmp, 5260 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5261 5262 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op); 5263 5264 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2)); 5265 DONE; 5266 } 5267 [(set_attr "length" "8")] 5268) 5269 5270;; Logical left shift using SISD or Integer instruction 5271(define_insn "*aarch64_ashl_sisd_or_int_<mode>3" 5272 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w") 5273 (ashift:GPI 5274 (match_operand:GPI 1 "register_operand" "r,r,w,w") 5275 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))] 5276 "" 5277 "@ 5278 lsl\t%<w>0, %<w>1, %2 5279 lsl\t%<w>0, %<w>1, %<w>2 5280 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5281 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>" 5282 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>") 5283 (set_attr "arch" "*,*,simd,simd")] 5284) 5285 5286;; Logical right shift using SISD or Integer instruction 5287(define_insn "*aarch64_lshr_sisd_or_int_<mode>3" 5288 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w") 5289 (lshiftrt:GPI 5290 (match_operand:GPI 1 "register_operand" "r,r,w,w,w") 5291 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" 5292 "Us<cmode>,r,Us<cmode_simd>,w,0")))] 5293 "" 5294 "@ 5295 lsr\t%<w>0, %<w>1, %2 5296 lsr\t%<w>0, %<w>1, %<w>2 5297 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5298 # 5299 #" 5300 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>") 5301 (set_attr "arch" "*,*,simd,simd,simd")] 5302) 5303 5304(define_split 5305 [(set (match_operand:DI 0 "aarch64_simd_register") 5306 (lshiftrt:DI 5307 (match_operand:DI 1 "aarch64_simd_register") 5308 (match_operand:QI 2 "aarch64_simd_register")))] 5309 "TARGET_SIMD && reload_completed" 5310 [(set (match_dup 3) 5311 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5312 (set (match_dup 0) 5313 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))] 5314 { 5315 operands[3] = gen_lowpart (QImode, operands[0]); 5316 } 5317) 5318 5319(define_split 5320 [(set (match_operand:SI 0 "aarch64_simd_register") 5321 (lshiftrt:SI 5322 (match_operand:SI 1 "aarch64_simd_register") 5323 (match_operand:QI 2 "aarch64_simd_register")))] 5324 "TARGET_SIMD && reload_completed" 5325 [(set (match_dup 3) 5326 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5327 (set (match_dup 0) 5328 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))] 5329 { 5330 operands[3] = gen_lowpart (QImode, operands[0]); 5331 } 5332) 5333 5334;; Arithmetic right shift using SISD or Integer instruction 5335(define_insn "*aarch64_ashr_sisd_or_int_<mode>3" 5336 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w") 5337 (ashiftrt:GPI 5338 (match_operand:GPI 1 "register_operand" "r,r,w,w,w") 5339 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" 5340 "Us<cmode>,r,Us<cmode_simd>,w,0")))] 5341 "" 5342 "@ 5343 asr\t%<w>0, %<w>1, %2 5344 asr\t%<w>0, %<w>1, %<w>2 5345 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5346 # 5347 #" 5348 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>") 5349 (set_attr "arch" "*,*,simd,simd,simd")] 5350) 5351 5352(define_split 5353 [(set (match_operand:DI 0 "aarch64_simd_register") 5354 (ashiftrt:DI 5355 (match_operand:DI 1 "aarch64_simd_register") 5356 (match_operand:QI 2 "aarch64_simd_register")))] 5357 "TARGET_SIMD && reload_completed" 5358 [(set (match_dup 3) 5359 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5360 (set (match_dup 0) 5361 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))] 5362{ 5363 operands[3] = gen_lowpart (QImode, operands[0]); 5364} 5365) 5366 5367(define_split 5368 [(set (match_operand:SI 0 "aarch64_simd_register") 5369 (ashiftrt:SI 5370 (match_operand:SI 1 "aarch64_simd_register") 5371 (match_operand:QI 2 "aarch64_simd_register")))] 5372 "TARGET_SIMD && reload_completed" 5373 [(set (match_dup 3) 5374 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5375 (set (match_dup 0) 5376 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))] 5377{ 5378 operands[3] = gen_lowpart (QImode, operands[0]); 5379} 5380) 5381 5382(define_insn "*aarch64_sisd_ushl" 5383 [(set (match_operand:DI 0 "register_operand" "=w") 5384 (unspec:DI [(match_operand:DI 1 "register_operand" "w") 5385 (match_operand:QI 2 "register_operand" "w")] 5386 UNSPEC_SISD_USHL))] 5387 "TARGET_SIMD" 5388 "ushl\t%d0, %d1, %d2" 5389 [(set_attr "type" "neon_shift_reg")] 5390) 5391 5392(define_insn "*aarch64_ushl_2s" 5393 [(set (match_operand:SI 0 "register_operand" "=w") 5394 (unspec:SI [(match_operand:SI 1 "register_operand" "w") 5395 (match_operand:QI 2 "register_operand" "w")] 5396 UNSPEC_USHL_2S))] 5397 "TARGET_SIMD" 5398 "ushl\t%0.2s, %1.2s, %2.2s" 5399 [(set_attr "type" "neon_shift_reg")] 5400) 5401 5402(define_insn "*aarch64_sisd_sshl" 5403 [(set (match_operand:DI 0 "register_operand" "=w") 5404 (unspec:DI [(match_operand:DI 1 "register_operand" "w") 5405 (match_operand:QI 2 "register_operand" "w")] 5406 UNSPEC_SISD_SSHL))] 5407 "TARGET_SIMD" 5408 "sshl\t%d0, %d1, %d2" 5409 [(set_attr "type" "neon_shift_reg")] 5410) 5411 5412(define_insn "*aarch64_sshl_2s" 5413 [(set (match_operand:SI 0 "register_operand" "=w") 5414 (unspec:SI [(match_operand:SI 1 "register_operand" "w") 5415 (match_operand:QI 2 "register_operand" "w")] 5416 UNSPEC_SSHL_2S))] 5417 "TARGET_SIMD" 5418 "sshl\t%0.2s, %1.2s, %2.2s" 5419 [(set_attr "type" "neon_shift_reg")] 5420) 5421 5422(define_insn "*aarch64_sisd_neg_qi" 5423 [(set (match_operand:QI 0 "register_operand" "=w") 5424 (unspec:QI [(match_operand:QI 1 "register_operand" "w")] 5425 UNSPEC_SISD_NEG))] 5426 "TARGET_SIMD" 5427 "neg\t%d0, %d1" 5428 [(set_attr "type" "neon_neg")] 5429) 5430 5431;; Rotate right 5432(define_insn "*ror<mode>3_insn" 5433 [(set (match_operand:GPI 0 "register_operand" "=r,r") 5434 (rotatert:GPI 5435 (match_operand:GPI 1 "register_operand" "r,r") 5436 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))] 5437 "" 5438 "@ 5439 ror\\t%<w>0, %<w>1, %2 5440 ror\\t%<w>0, %<w>1, %<w>2" 5441 [(set_attr "type" "rotate_imm,shift_reg")] 5442) 5443 5444;; zero_extend version of above 5445(define_insn "*<optab>si3_insn_uxtw" 5446 [(set (match_operand:DI 0 "register_operand" "=r,r") 5447 (zero_extend:DI (SHIFT:SI 5448 (match_operand:SI 1 "register_operand" "r,r") 5449 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))] 5450 "" 5451 "@ 5452 <shift>\\t%w0, %w1, %2 5453 <shift>\\t%w0, %w1, %w2" 5454 [(set_attr "type" "bfx,shift_reg")] 5455) 5456 5457(define_insn "*<optab><mode>3_insn" 5458 [(set (match_operand:SHORT 0 "register_operand" "=r") 5459 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r") 5460 (match_operand 2 "const_int_operand" "n")))] 5461 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)" 5462{ 5463 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2])); 5464 return "<bfshift>\t%w0, %w1, %2, %3"; 5465} 5466 [(set_attr "type" "bfx")] 5467) 5468 5469(define_insn "*extr<mode>5_insn" 5470 [(set (match_operand:GPI 0 "register_operand" "=r") 5471 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5472 (match_operand 3 "const_int_operand" "n")) 5473 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r") 5474 (match_operand 4 "const_int_operand" "n"))))] 5475 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) && 5476 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))" 5477 "extr\\t%<w>0, %<w>1, %<w>2, %4" 5478 [(set_attr "type" "rotate_imm")] 5479) 5480 5481;; There are no canonicalisation rules for ashift and lshiftrt inside an ior 5482;; so we have to match both orderings. 5483(define_insn "*extr<mode>5_insn_alt" 5484 [(set (match_operand:GPI 0 "register_operand" "=r") 5485 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r") 5486 (match_operand 4 "const_int_operand" "n")) 5487 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5488 (match_operand 3 "const_int_operand" "n"))))] 5489 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) 5490 && (UINTVAL (operands[3]) + UINTVAL (operands[4]) 5491 == GET_MODE_BITSIZE (<MODE>mode))" 5492 "extr\\t%<w>0, %<w>1, %<w>2, %4" 5493 [(set_attr "type" "rotate_imm")] 5494) 5495 5496;; zero_extend version of the above 5497(define_insn "*extrsi5_insn_uxtw" 5498 [(set (match_operand:DI 0 "register_operand" "=r") 5499 (zero_extend:DI 5500 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r") 5501 (match_operand 3 "const_int_operand" "n")) 5502 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r") 5503 (match_operand 4 "const_int_operand" "n")))))] 5504 "UINTVAL (operands[3]) < 32 && 5505 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)" 5506 "extr\\t%w0, %w1, %w2, %4" 5507 [(set_attr "type" "rotate_imm")] 5508) 5509 5510(define_insn "*extrsi5_insn_uxtw_alt" 5511 [(set (match_operand:DI 0 "register_operand" "=r") 5512 (zero_extend:DI 5513 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r") 5514 (match_operand 4 "const_int_operand" "n")) 5515 (ashift:SI (match_operand:SI 1 "register_operand" "r") 5516 (match_operand 3 "const_int_operand" "n")))))] 5517 "UINTVAL (operands[3]) < 32 && 5518 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)" 5519 "extr\\t%w0, %w1, %w2, %4" 5520 [(set_attr "type" "rotate_imm")] 5521) 5522 5523(define_insn "*ror<mode>3_insn" 5524 [(set (match_operand:GPI 0 "register_operand" "=r") 5525 (rotate:GPI (match_operand:GPI 1 "register_operand" "r") 5526 (match_operand 2 "const_int_operand" "n")))] 5527 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)" 5528{ 5529 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2])); 5530 return "ror\\t%<w>0, %<w>1, %3"; 5531} 5532 [(set_attr "type" "rotate_imm")] 5533) 5534 5535;; zero_extend version of the above 5536(define_insn "*rorsi3_insn_uxtw" 5537 [(set (match_operand:DI 0 "register_operand" "=r") 5538 (zero_extend:DI 5539 (rotate:SI (match_operand:SI 1 "register_operand" "r") 5540 (match_operand 2 "const_int_operand" "n"))))] 5541 "UINTVAL (operands[2]) < 32" 5542{ 5543 operands[3] = GEN_INT (32 - UINTVAL (operands[2])); 5544 return "ror\\t%w0, %w1, %3"; 5545} 5546 [(set_attr "type" "rotate_imm")] 5547) 5548 5549(define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>" 5550 [(set (match_operand:GPI 0 "register_operand" "=r") 5551 (ANY_EXTEND:GPI 5552 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r") 5553 (match_operand 2 "const_int_operand" "n"))))] 5554 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5555{ 5556 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5557 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5558} 5559 [(set_attr "type" "bfx")] 5560) 5561 5562(define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>" 5563 [(set (match_operand:GPI 0 "register_operand" "=r") 5564 (zero_extend:GPI 5565 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r") 5566 (match_operand 2 "const_int_operand" "n"))))] 5567 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5568{ 5569 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5570 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5571} 5572 [(set_attr "type" "bfx")] 5573) 5574 5575(define_insn "*extend<GPI:mode>_ashr<SHORT:mode>" 5576 [(set (match_operand:GPI 0 "register_operand" "=r") 5577 (sign_extend:GPI 5578 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r") 5579 (match_operand 2 "const_int_operand" "n"))))] 5580 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5581{ 5582 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5583 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5584} 5585 [(set_attr "type" "bfx")] 5586) 5587 5588;; ------------------------------------------------------------------- 5589;; Bitfields 5590;; ------------------------------------------------------------------- 5591 5592(define_expand "<optab>" 5593 [(set (match_operand:DI 0 "register_operand") 5594 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand") 5595 (match_operand 2 5596 "aarch64_simd_shift_imm_offset_di") 5597 (match_operand 3 "aarch64_simd_shift_imm_di")))] 5598 "" 5599 { 5600 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5601 1, GET_MODE_BITSIZE (DImode) - 1)) 5602 FAIL; 5603 } 5604) 5605 5606 5607(define_insn "*<optab><mode>" 5608 [(set (match_operand:GPI 0 "register_operand" "=r") 5609 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r") 5610 (match_operand 2 5611 "aarch64_simd_shift_imm_offset_<mode>" "n") 5612 (match_operand 3 5613 "aarch64_simd_shift_imm_<mode>" "n")))] 5614 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5615 1, GET_MODE_BITSIZE (<MODE>mode) - 1)" 5616 "<su>bfx\\t%<w>0, %<w>1, %3, %2" 5617 [(set_attr "type" "bfx")] 5618) 5619 5620;; When the bit position and width add up to 32 we can use a W-reg LSR 5621;; instruction taking advantage of the implicit zero-extension of the X-reg. 5622(define_split 5623 [(set (match_operand:DI 0 "register_operand") 5624 (zero_extract:DI (match_operand:DI 1 "register_operand") 5625 (match_operand 2 5626 "aarch64_simd_shift_imm_offset_di") 5627 (match_operand 3 5628 "aarch64_simd_shift_imm_di")))] 5629 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1, 5630 GET_MODE_BITSIZE (DImode) - 1) 5631 && (INTVAL (operands[2]) + INTVAL (operands[3])) 5632 == GET_MODE_BITSIZE (SImode)" 5633 [(set (match_dup 0) 5634 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))] 5635 { 5636 operands[4] = gen_lowpart (SImode, operands[1]); 5637 } 5638) 5639 5640;; Bitfield Insert (insv) 5641(define_expand "insv<mode>" 5642 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand") 5643 (match_operand 1 "const_int_operand") 5644 (match_operand 2 "const_int_operand")) 5645 (match_operand:GPI 3 "general_operand"))] 5646 "" 5647{ 5648 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]); 5649 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]); 5650 rtx value = operands[3]; 5651 5652 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode)) 5653 FAIL; 5654 5655 if (CONST_INT_P (value)) 5656 { 5657 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1; 5658 5659 /* Prefer AND/OR for inserting all zeros or all ones. */ 5660 if ((UINTVAL (value) & mask) == 0 5661 || (UINTVAL (value) & mask) == mask) 5662 FAIL; 5663 5664 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */ 5665 if (width == 16 && (pos % 16) == 0) 5666 DONE; 5667 } 5668 operands[3] = force_reg (<MODE>mode, value); 5669}) 5670 5671(define_insn "*insv_reg<mode>" 5672 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5673 (match_operand 1 "const_int_operand" "n") 5674 (match_operand 2 "const_int_operand" "n")) 5675 (match_operand:GPI 3 "register_operand" "r"))] 5676 "!(UINTVAL (operands[1]) == 0 5677 || (UINTVAL (operands[2]) + UINTVAL (operands[1]) 5678 > GET_MODE_BITSIZE (<MODE>mode)))" 5679 "bfi\\t%<w>0, %<w>3, %2, %1" 5680 [(set_attr "type" "bfm")] 5681) 5682 5683(define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4" 5684 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5685 (match_operand 1 "const_int_operand" "n") 5686 (match_operand 2 "const_int_operand" "n")) 5687 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))] 5688 "UINTVAL (operands[1]) <= <ALLX:sizen>" 5689 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1" 5690 [(set_attr "type" "bfm")] 5691) 5692 5693;; Match a bfi instruction where the shift of OP3 means that we are 5694;; actually copying the least significant bits of OP3 into OP0 by way 5695;; of the AND masks and the IOR instruction. A similar instruction 5696;; with the two parts of the IOR swapped around was never triggered 5697;; in a bootstrap build and test of GCC so it was not included. 5698 5699(define_insn "*aarch64_bfi<GPI:mode>5_shift" 5700 [(set (match_operand:GPI 0 "register_operand" "=r") 5701 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5702 (match_operand:GPI 2 "const_int_operand" "n")) 5703 (and:GPI (ashift:GPI 5704 (match_operand:GPI 3 "register_operand" "r") 5705 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n")) 5706 (match_operand:GPI 5 "const_int_operand" "n"))))] 5707 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 5708 UINTVAL (operands[4]), 5709 UINTVAL(operands[5]))" 5710 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5" 5711 [(set_attr "type" "bfm")] 5712) 5713 5714(define_insn "*aarch64_bfi<GPI:mode>5_shift_alt" 5715 [(set (match_operand:GPI 0 "register_operand" "=r") 5716 (ior:GPI (and:GPI (ashift:GPI 5717 (match_operand:GPI 1 "register_operand" "r") 5718 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n")) 5719 (match_operand:GPI 3 "const_int_operand" "n")) 5720 (and:GPI (match_operand:GPI 4 "register_operand" "0") 5721 (match_operand:GPI 5 "const_int_operand" "n"))))] 5722 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]), 5723 UINTVAL (operands[2]), 5724 UINTVAL(operands[3]))" 5725 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3" 5726 [(set_attr "type" "bfm")] 5727) 5728 5729;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because 5730;; the shift is large enough to remove the need for an AND instruction. 5731 5732(define_insn "*aarch64_bfi<GPI:mode>4_noand" 5733 [(set (match_operand:GPI 0 "register_operand" "=r") 5734 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5735 (match_operand:GPI 2 "const_int_operand" "n")) 5736 (ashift:GPI 5737 (match_operand:GPI 3 "register_operand" "r") 5738 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))] 5739 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 5740 UINTVAL (operands[4]), 5741 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )" 5742{ 5743 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4])); 5744 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5"; 5745} 5746 [(set_attr "type" "bfm")] 5747) 5748 5749(define_insn "*aarch64_bfi<GPI:mode>4_noand_alt" 5750 [(set (match_operand:GPI 0 "register_operand" "=r") 5751 (ior:GPI (ashift:GPI 5752 (match_operand:GPI 1 "register_operand" "r") 5753 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n")) 5754 (and:GPI (match_operand:GPI 3 "register_operand" "0") 5755 (match_operand:GPI 4 "const_int_operand" "n"))))] 5756 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]), 5757 UINTVAL (operands[2]), 5758 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )" 5759{ 5760 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2])); 5761 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5"; 5762} 5763 [(set_attr "type" "bfm")] 5764) 5765 5766;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just 5767;; copying the least significant bits of OP3 to OP0. We need two versions 5768;; of the instruction to handle different checks on the constant values. 5769 5770(define_insn "*aarch64_bfi<GPI:mode>4_noshift" 5771 [(set (match_operand:GPI 0 "register_operand" "=r") 5772 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5773 (match_operand:GPI 2 "const_int_operand" "n")) 5774 (and:GPI (match_operand:GPI 3 "register_operand" "r") 5775 (match_operand:GPI 4 "const_int_operand" "n"))))] 5776 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0, 5777 UINTVAL (operands[4]))" 5778 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4" 5779 [(set_attr "type" "bfm")] 5780) 5781 5782(define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt" 5783 [(set (match_operand:GPI 0 "register_operand" "=r") 5784 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r") 5785 (match_operand:GPI 4 "const_int_operand" "n")) 5786 (and:GPI (match_operand:GPI 1 "register_operand" "0") 5787 (match_operand:GPI 2 "const_int_operand" "n"))))] 5788 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0, 5789 UINTVAL (operands[4]))" 5790 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4" 5791 [(set_attr "type" "bfm")] 5792) 5793 5794(define_insn "*extr_insv_lower_reg<mode>" 5795 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5796 (match_operand 1 "const_int_operand" "n") 5797 (const_int 0)) 5798 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r") 5799 (match_dup 1) 5800 (match_operand 3 "const_int_operand" "n")))] 5801 "!(UINTVAL (operands[1]) == 0 5802 || (UINTVAL (operands[3]) + UINTVAL (operands[1]) 5803 > GET_MODE_BITSIZE (<MODE>mode)))" 5804 "bfxil\\t%<w>0, %<w>2, %3, %1" 5805 [(set_attr "type" "bfm")] 5806) 5807 5808(define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>" 5809 [(set (match_operand:GPI 0 "register_operand" "=r") 5810 (ashift:GPI (ANY_EXTEND:GPI 5811 (match_operand:ALLX 1 "register_operand" "r")) 5812 (match_operand 2 "const_int_operand" "n")))] 5813 "UINTVAL (operands[2]) < <GPI:sizen>" 5814{ 5815 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2]))) 5816 ? GEN_INT (<ALLX:sizen>) 5817 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2])); 5818 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5819} 5820 [(set_attr "type" "bfx")] 5821) 5822 5823;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below 5824 5825(define_insn "*andim_ashift<mode>_bfiz" 5826 [(set (match_operand:GPI 0 "register_operand" "=r") 5827 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5828 (match_operand 2 "const_int_operand" "n")) 5829 (match_operand 3 "const_int_operand" "n")))] 5830 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])" 5831 "ubfiz\\t%<w>0, %<w>1, %2, %P3" 5832 [(set_attr "type" "bfx")] 5833) 5834 5835;; Match sbfiz pattern in a shift left + shift right operation. 5836 5837(define_insn "*ashift<mode>_extv_bfiz" 5838 [(set (match_operand:GPI 0 "register_operand" "=r") 5839 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r") 5840 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n") 5841 (const_int 0)) 5842 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))] 5843 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5844 1, GET_MODE_BITSIZE (<MODE>mode) - 1)" 5845 "sbfiz\\t%<w>0, %<w>1, %3, %2" 5846 [(set_attr "type" "bfx")] 5847) 5848 5849(define_insn "*ashiftsi_extvdi_bfiz" 5850 [(set (match_operand:SI 0 "register_operand" "=r") 5851 (ashift:SI 5852 (match_operator:SI 4 "subreg_lowpart_operator" 5853 [(sign_extract:DI 5854 (match_operand:DI 1 "register_operand" "r") 5855 (match_operand 2 "aarch64_simd_shift_imm_offset_si") 5856 (const_int 0))]) 5857 (match_operand 3 "aarch64_simd_shift_imm_si")))] 5858 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5859 1, GET_MODE_BITSIZE (SImode) - 1)" 5860 "sbfiz\\t%w0, %w1, %3, %2" 5861 [(set_attr "type" "bfx")] 5862) 5863 5864;; When the bit position and width of the equivalent extraction add up to 32 5865;; we can use a W-reg LSL instruction taking advantage of the implicit 5866;; zero-extension of the X-reg. 5867(define_split 5868 [(set (match_operand:DI 0 "register_operand") 5869 (and:DI (ashift:DI (match_operand:DI 1 "register_operand") 5870 (match_operand 2 "const_int_operand")) 5871 (match_operand 3 "const_int_operand")))] 5872 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2]) 5873 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3]))) 5874 == GET_MODE_BITSIZE (SImode)" 5875 [(set (match_dup 0) 5876 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))] 5877 { 5878 operands[4] = gen_lowpart (SImode, operands[1]); 5879 } 5880) 5881 5882(define_insn "bswap<mode>2" 5883 [(set (match_operand:GPI 0 "register_operand" "=r") 5884 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))] 5885 "" 5886 "rev\\t%<w>0, %<w>1" 5887 [(set_attr "type" "rev")] 5888) 5889 5890(define_insn "bswaphi2" 5891 [(set (match_operand:HI 0 "register_operand" "=r") 5892 (bswap:HI (match_operand:HI 1 "register_operand" "r")))] 5893 "" 5894 "rev16\\t%w0, %w1" 5895 [(set_attr "type" "rev")] 5896) 5897 5898(define_insn "*aarch64_bfxil<mode>" 5899 [(set (match_operand:GPI 0 "register_operand" "=r,r") 5900 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0") 5901 (match_operand:GPI 3 "const_int_operand" "n, Ulc")) 5902 (and:GPI (match_operand:GPI 2 "register_operand" "0,r") 5903 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))] 5904 "(INTVAL (operands[3]) == ~INTVAL (operands[4])) 5905 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3])) 5906 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))" 5907 { 5908 switch (which_alternative) 5909 { 5910 case 0: 5911 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3]))); 5912 return "bfxil\\t%<w>0, %<w>1, 0, %3"; 5913 case 1: 5914 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4]))); 5915 return "bfxil\\t%<w>0, %<w>2, 0, %3"; 5916 default: 5917 gcc_unreachable (); 5918 } 5919 } 5920 [(set_attr "type" "bfm")] 5921) 5922 5923; Zero-extended version of above (aarch64_bfxil) 5924(define_insn "*aarch64_bfxilsi_uxtw" 5925 [(set (match_operand:DI 0 "register_operand" "=r,r") 5926 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand" 5927 "r,0") 5928 (match_operand:SI 3 "const_int_operand" "n, Ulc")) 5929 (and:SI (match_operand:SI 2 "register_operand" "0,r") 5930 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))] 5931 "(INTVAL (operands[3]) == ~INTVAL (operands[4])) 5932 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3])) 5933 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))" 5934 { 5935 switch (which_alternative) 5936 { 5937 case 0: 5938 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3]))); 5939 return "bfxil\\t%w0, %w1, 0, %3"; 5940 case 1: 5941 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4]))); 5942 return "bfxil\\t%w0, %w2, 0, %3"; 5943 default: 5944 gcc_unreachable (); 5945 } 5946 } 5947 [(set_attr "type" "bfm")] 5948) 5949 5950;; There are no canonicalisation rules for the position of the lshiftrt, ashift 5951;; operations within an IOR/AND RTX, therefore we have two patterns matching 5952;; each valid permutation. 5953 5954(define_insn "rev16<mode>2" 5955 [(set (match_operand:GPI 0 "register_operand" "=r") 5956 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5957 (const_int 8)) 5958 (match_operand:GPI 3 "const_int_operand" "n")) 5959 (and:GPI (lshiftrt:GPI (match_dup 1) 5960 (const_int 8)) 5961 (match_operand:GPI 2 "const_int_operand" "n"))))] 5962 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode) 5963 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)" 5964 "rev16\\t%<w>0, %<w>1" 5965 [(set_attr "type" "rev")] 5966) 5967 5968(define_insn "rev16<mode>2_alt" 5969 [(set (match_operand:GPI 0 "register_operand" "=r") 5970 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r") 5971 (const_int 8)) 5972 (match_operand:GPI 2 "const_int_operand" "n")) 5973 (and:GPI (ashift:GPI (match_dup 1) 5974 (const_int 8)) 5975 (match_operand:GPI 3 "const_int_operand" "n"))))] 5976 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode) 5977 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)" 5978 "rev16\\t%<w>0, %<w>1" 5979 [(set_attr "type" "rev")] 5980) 5981 5982;; zero_extend version of above 5983(define_insn "*bswapsi2_uxtw" 5984 [(set (match_operand:DI 0 "register_operand" "=r") 5985 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))] 5986 "" 5987 "rev\\t%w0, %w1" 5988 [(set_attr "type" "rev")] 5989) 5990 5991;; ------------------------------------------------------------------- 5992;; Floating-point intrinsics 5993;; ------------------------------------------------------------------- 5994 5995;; frint floating-point round to integral standard patterns. 5996;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn. 5997 5998(define_insn "<frint_pattern><mode>2" 5999 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6000 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")] 6001 FRINT))] 6002 "TARGET_FLOAT" 6003 "frint<frint_suffix>\\t%<s>0, %<s>1" 6004 [(set_attr "type" "f_rint<stype>")] 6005) 6006 6007;; frcvt floating-point round to integer and convert standard patterns. 6008;; Expands to lbtrunc, lceil, lfloor, lround. 6009(define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2" 6010 [(set (match_operand:GPI 0 "register_operand" "=r") 6011 (FIXUORS:GPI 6012 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")] 6013 FCVT)))] 6014 "TARGET_FLOAT" 6015 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1" 6016 [(set_attr "type" "f_cvtf2i")] 6017) 6018 6019(define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult" 6020 [(set (match_operand:GPI 0 "register_operand" "=r") 6021 (FIXUORS:GPI 6022 (mult:GPF 6023 (match_operand:GPF 1 "register_operand" "w") 6024 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))] 6025 "TARGET_FLOAT 6026 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1, 6027 GET_MODE_BITSIZE (<GPI:MODE>mode))" 6028 { 6029 int fbits = aarch64_fpconst_pow_of_2 (operands[2]); 6030 char buf[64]; 6031 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits); 6032 output_asm_insn (buf, operands); 6033 return ""; 6034 } 6035 [(set_attr "type" "f_cvtf2i")] 6036) 6037 6038;; fma - expand fma into patterns with the accumulator operand first since 6039;; reusing the accumulator results in better register allocation. 6040;; The register allocator considers copy preferences in operand order, 6041;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0. 6042 6043(define_expand "fma<mode>4" 6044 [(set (match_operand:GPF_F16 0 "register_operand") 6045 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand") 6046 (match_operand:GPF_F16 2 "register_operand") 6047 (match_operand:GPF_F16 3 "register_operand")))] 6048 "TARGET_FLOAT" 6049) 6050 6051(define_insn "*aarch64_fma<mode>4" 6052 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6053 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w") 6054 (match_operand:GPF_F16 3 "register_operand" "w") 6055 (match_operand:GPF_F16 1 "register_operand" "w")))] 6056 "TARGET_FLOAT" 6057 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 6058 [(set_attr "type" "fmac<stype>")] 6059) 6060 6061(define_expand "fnma<mode>4" 6062 [(set (match_operand:GPF_F16 0 "register_operand") 6063 (fma:GPF_F16 6064 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand")) 6065 (match_operand:GPF_F16 2 "register_operand") 6066 (match_operand:GPF_F16 3 "register_operand")))] 6067 "TARGET_FLOAT" 6068) 6069 6070(define_insn "*aarch64_fnma<mode>4" 6071 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6072 (fma:GPF_F16 6073 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")) 6074 (match_operand:GPF_F16 3 "register_operand" "w") 6075 (match_operand:GPF_F16 1 "register_operand" "w")))] 6076 "TARGET_FLOAT" 6077 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1" 6078 [(set_attr "type" "fmac<stype>")] 6079) 6080 6081 6082(define_expand "fms<mode>4" 6083 [(set (match_operand:GPF 0 "register_operand") 6084 (fma:GPF (match_operand:GPF 1 "register_operand") 6085 (match_operand:GPF 2 "register_operand") 6086 (neg:GPF (match_operand:GPF 3 "register_operand"))))] 6087 "TARGET_FLOAT" 6088) 6089 6090(define_insn "*aarch64_fms<mode>4" 6091 [(set (match_operand:GPF 0 "register_operand" "=w") 6092 (fma:GPF (match_operand:GPF 2 "register_operand" "w") 6093 (match_operand:GPF 3 "register_operand" "w") 6094 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))] 6095 "TARGET_FLOAT" 6096 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1" 6097 [(set_attr "type" "fmac<s>")] 6098) 6099 6100(define_expand "fnms<mode>4" 6101 [(set (match_operand:GPF 0 "register_operand") 6102 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand")) 6103 (match_operand:GPF 2 "register_operand") 6104 (neg:GPF (match_operand:GPF 3 "register_operand"))))] 6105 "TARGET_FLOAT" 6106) 6107 6108(define_insn "*aarch64_fnms<mode>4" 6109 [(set (match_operand:GPF 0 "register_operand" "=w") 6110 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w")) 6111 (match_operand:GPF 3 "register_operand" "w") 6112 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))] 6113 "TARGET_FLOAT" 6114 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 6115 [(set_attr "type" "fmac<s>")] 6116) 6117 6118;; If signed zeros are ignored, -(a * b + c) = -a * b - c. 6119(define_insn "*aarch64_fnmadd<mode>4" 6120 [(set (match_operand:GPF 0 "register_operand" "=w") 6121 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w") 6122 (match_operand:GPF 3 "register_operand" "w") 6123 (match_operand:GPF 1 "register_operand" "w"))))] 6124 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT" 6125 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 6126 [(set_attr "type" "fmac<s>")] 6127) 6128 6129;; ------------------------------------------------------------------- 6130;; Floating-point conversions 6131;; ------------------------------------------------------------------- 6132 6133(define_insn "extendsfdf2" 6134 [(set (match_operand:DF 0 "register_operand" "=w") 6135 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))] 6136 "TARGET_FLOAT" 6137 "fcvt\\t%d0, %s1" 6138 [(set_attr "type" "f_cvt")] 6139) 6140 6141(define_insn "extendhfsf2" 6142 [(set (match_operand:SF 0 "register_operand" "=w") 6143 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))] 6144 "TARGET_FLOAT" 6145 "fcvt\\t%s0, %h1" 6146 [(set_attr "type" "f_cvt")] 6147) 6148 6149(define_insn "extendhfdf2" 6150 [(set (match_operand:DF 0 "register_operand" "=w") 6151 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))] 6152 "TARGET_FLOAT" 6153 "fcvt\\t%d0, %h1" 6154 [(set_attr "type" "f_cvt")] 6155) 6156 6157(define_insn "truncdfsf2" 6158 [(set (match_operand:SF 0 "register_operand" "=w") 6159 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))] 6160 "TARGET_FLOAT" 6161 "fcvt\\t%s0, %d1" 6162 [(set_attr "type" "f_cvt")] 6163) 6164 6165(define_insn "truncsfhf2" 6166 [(set (match_operand:HF 0 "register_operand" "=w") 6167 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))] 6168 "TARGET_FLOAT" 6169 "fcvt\\t%h0, %s1" 6170 [(set_attr "type" "f_cvt")] 6171) 6172 6173(define_insn "truncdfhf2" 6174 [(set (match_operand:HF 0 "register_operand" "=w") 6175 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))] 6176 "TARGET_FLOAT" 6177 "fcvt\\t%h0, %d1" 6178 [(set_attr "type" "f_cvt")] 6179) 6180 6181;; Convert SF -> SI or DF -> DI while preferring w = w register constraints 6182;; and making r = w more expensive 6183 6184(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2" 6185 [(set (match_operand:GPI 0 "register_operand" "=w,?r") 6186 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))] 6187 "TARGET_FLOAT" 6188 "@ 6189 fcvtz<su>\t%<s>0, %<s>1 6190 fcvtz<su>\t%<w>0, %<s>1" 6191 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")] 6192) 6193 6194;; Convert HF -> SI or DI 6195 6196(define_insn "<optab>_trunchf<GPI:mode>2" 6197 [(set (match_operand:GPI 0 "register_operand" "=r") 6198 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))] 6199 "TARGET_FP_F16INST" 6200 "fcvtz<su>\t%<w>0, %h1" 6201 [(set_attr "type" "f_cvtf2i")] 6202) 6203 6204;; Convert DF -> SI or SF -> DI which can only be accomplished with 6205;; input in a fp register and output in a integer register 6206 6207(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2" 6208 [(set (match_operand:GPI 0 "register_operand" "=r") 6209 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))] 6210 "TARGET_FLOAT" 6211 "fcvtz<su>\t%<w>0, %<fpw>1" 6212 [(set_attr "type" "f_cvtf2i")] 6213) 6214 6215(define_insn "*fix_to_zero_extend<mode>di2" 6216 [(set (match_operand:DI 0 "register_operand" "=r") 6217 (zero_extend:DI 6218 (unsigned_fix:SI 6219 (match_operand:GPF 1 "register_operand" "w"))))] 6220 "TARGET_FLOAT" 6221 "fcvtzu\t%w0, %<s>1" 6222 [(set_attr "type" "f_cvtf2i")] 6223) 6224 6225;; Equal width integer to fp and multiply combine. 6226(define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult" 6227 [(set (match_operand:GPF 0 "register_operand" "=w,w") 6228 (mult:GPF (FLOATUORS:GPF 6229 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")) 6230 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))] 6231 "TARGET_FLOAT" 6232 { 6233 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2])); 6234 switch (which_alternative) 6235 { 6236 case 0: 6237 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2"; 6238 case 1: 6239 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2"; 6240 default: 6241 gcc_unreachable (); 6242 } 6243 } 6244 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f") 6245 (set_attr "arch" "simd,fp")] 6246) 6247 6248;; Unequal width integer to fp and multiply combine. 6249(define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult" 6250 [(set (match_operand:GPF 0 "register_operand" "=w") 6251 (mult:GPF (FLOATUORS:GPF 6252 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")) 6253 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))] 6254 "TARGET_FLOAT" 6255 { 6256 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2])); 6257 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2"; 6258 } 6259 [(set_attr "type" "f_cvti2f")] 6260) 6261 6262;; Equal width integer to fp conversion. 6263(define_insn "<optab><fcvt_target><GPF:mode>2" 6264 [(set (match_operand:GPF 0 "register_operand" "=w,w") 6265 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))] 6266 "TARGET_FLOAT" 6267 "@ 6268 <su_optab>cvtf\t%<GPF:s>0, %<s>1 6269 <su_optab>cvtf\t%<GPF:s>0, %<w1>1" 6270 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f") 6271 (set_attr "arch" "simd,fp")] 6272) 6273 6274;; Unequal width integer to fp conversions. 6275(define_insn "<optab><fcvt_iesize><GPF:mode>2" 6276 [(set (match_operand:GPF 0 "register_operand" "=w") 6277 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))] 6278 "TARGET_FLOAT" 6279 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1" 6280 [(set_attr "type" "f_cvti2f")] 6281) 6282 6283;; If we do not have ARMv8.2-A 16-bit floating point extensions, the 6284;; midend will arrange for an SImode conversion to HFmode to first go 6285;; through DFmode, then to HFmode. But first it will try converting 6286;; to DImode then down, which would match our DImode pattern below and 6287;; give very poor code-generation. So, we must provide our own emulation 6288;; of the mid-end logic. 6289 6290(define_insn "aarch64_fp16_<optab><mode>hf2" 6291 [(set (match_operand:HF 0 "register_operand" "=w") 6292 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))] 6293 "TARGET_FP_F16INST" 6294 "<su_optab>cvtf\t%h0, %<w>1" 6295 [(set_attr "type" "f_cvti2f")] 6296) 6297 6298(define_expand "<optab>sihf2" 6299 [(set (match_operand:HF 0 "register_operand") 6300 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))] 6301 "TARGET_FLOAT" 6302{ 6303 if (TARGET_FP_F16INST) 6304 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1])); 6305 else 6306 { 6307 rtx convert_target = gen_reg_rtx (DFmode); 6308 emit_insn (gen_<optab>sidf2 (convert_target, operands[1])); 6309 emit_insn (gen_truncdfhf2 (operands[0], convert_target)); 6310 } 6311 DONE; 6312} 6313) 6314 6315;; For DImode there is no wide enough floating-point mode that we 6316;; can convert through natively (TFmode would work, but requires a library 6317;; call). However, we know that any value >= 65504 will be rounded 6318;; to infinity on conversion. This is well within the range of SImode, so 6319;; we can: 6320;; Saturate to SImode. 6321;; Convert from that to DFmode 6322;; Convert from that to HFmode (phew!). 6323;; Note that the saturation to SImode requires the SIMD extensions. If 6324;; we ever need to provide this pattern where the SIMD extensions are not 6325;; available, we would need a different approach. 6326 6327(define_expand "<optab>dihf2" 6328 [(set (match_operand:HF 0 "register_operand") 6329 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))] 6330 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)" 6331{ 6332 if (TARGET_FP_F16INST) 6333 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1])); 6334 else 6335 { 6336 rtx sat_target = gen_reg_rtx (SImode); 6337 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1])); 6338 emit_insn (gen_<optab>sihf2 (operands[0], sat_target)); 6339 } 6340 6341 DONE; 6342} 6343) 6344 6345;; Convert between fixed-point and floating-point (scalar modes) 6346 6347(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3" 6348 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w") 6349 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w") 6350 (match_operand:SI 2 "immediate_operand" "i, i")] 6351 FCVT_F2FIXED))] 6352 "" 6353 "@ 6354 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2 6355 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2" 6356 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>") 6357 (set_attr "arch" "fp,simd")] 6358) 6359 6360(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3" 6361 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w") 6362 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w") 6363 (match_operand:SI 2 "immediate_operand" "i, i")] 6364 FCVT_FIXED2F))] 6365 "" 6366 "@ 6367 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2 6368 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2" 6369 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>") 6370 (set_attr "arch" "fp,simd")] 6371) 6372 6373(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3" 6374 [(set (match_operand:GPI 0 "register_operand" "=r") 6375 (unspec:GPI [(match_operand:HF 1 "register_operand" "w") 6376 (match_operand:SI 2 "immediate_operand" "i")] 6377 FCVT_F2FIXED))] 6378 "TARGET_FP_F16INST" 6379 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2" 6380 [(set_attr "type" "f_cvtf2i")] 6381) 6382 6383(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3" 6384 [(set (match_operand:HF 0 "register_operand" "=w") 6385 (unspec:HF [(match_operand:GPI 1 "register_operand" "r") 6386 (match_operand:SI 2 "immediate_operand" "i")] 6387 FCVT_FIXED2F))] 6388 "TARGET_FP_F16INST" 6389 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2" 6390 [(set_attr "type" "f_cvti2f")] 6391) 6392 6393(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3" 6394 [(set (match_operand:HI 0 "register_operand" "=w") 6395 (unspec:HI [(match_operand:HF 1 "register_operand" "w") 6396 (match_operand:SI 2 "immediate_operand" "i")] 6397 FCVT_F2FIXED))] 6398 "TARGET_SIMD" 6399 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2" 6400 [(set_attr "type" "neon_fp_to_int_s")] 6401) 6402 6403(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3" 6404 [(set (match_operand:HF 0 "register_operand" "=w") 6405 (unspec:HF [(match_operand:HI 1 "register_operand" "w") 6406 (match_operand:SI 2 "immediate_operand" "i")] 6407 FCVT_FIXED2F))] 6408 "TARGET_SIMD" 6409 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2" 6410 [(set_attr "type" "neon_int_to_fp_s")] 6411) 6412 6413;; ------------------------------------------------------------------- 6414;; Floating-point arithmetic 6415;; ------------------------------------------------------------------- 6416 6417(define_insn "add<mode>3" 6418 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6419 (plus:GPF_F16 6420 (match_operand:GPF_F16 1 "register_operand" "w") 6421 (match_operand:GPF_F16 2 "register_operand" "w")))] 6422 "TARGET_FLOAT" 6423 "fadd\\t%<s>0, %<s>1, %<s>2" 6424 [(set_attr "type" "fadd<stype>")] 6425) 6426 6427(define_insn "sub<mode>3" 6428 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6429 (minus:GPF_F16 6430 (match_operand:GPF_F16 1 "register_operand" "w") 6431 (match_operand:GPF_F16 2 "register_operand" "w")))] 6432 "TARGET_FLOAT" 6433 "fsub\\t%<s>0, %<s>1, %<s>2" 6434 [(set_attr "type" "fadd<stype>")] 6435) 6436 6437(define_insn "mul<mode>3" 6438 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6439 (mult:GPF_F16 6440 (match_operand:GPF_F16 1 "register_operand" "w") 6441 (match_operand:GPF_F16 2 "register_operand" "w")))] 6442 "TARGET_FLOAT" 6443 "fmul\\t%<s>0, %<s>1, %<s>2" 6444 [(set_attr "type" "fmul<stype>")] 6445) 6446 6447(define_insn "*fnmul<mode>3" 6448 [(set (match_operand:GPF 0 "register_operand" "=w") 6449 (mult:GPF 6450 (neg:GPF (match_operand:GPF 1 "register_operand" "w")) 6451 (match_operand:GPF 2 "register_operand" "w")))] 6452 "TARGET_FLOAT && !flag_rounding_math" 6453 "fnmul\\t%<s>0, %<s>1, %<s>2" 6454 [(set_attr "type" "fmul<s>")] 6455) 6456 6457(define_insn "*fnmul<mode>3" 6458 [(set (match_operand:GPF 0 "register_operand" "=w") 6459 (neg:GPF (mult:GPF 6460 (match_operand:GPF 1 "register_operand" "w") 6461 (match_operand:GPF 2 "register_operand" "w"))))] 6462 "TARGET_FLOAT" 6463 "fnmul\\t%<s>0, %<s>1, %<s>2" 6464 [(set_attr "type" "fmul<s>")] 6465) 6466 6467(define_expand "div<mode>3" 6468 [(set (match_operand:GPF_F16 0 "register_operand") 6469 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand") 6470 (match_operand:GPF_F16 2 "register_operand")))] 6471 "TARGET_FLOAT" 6472{ 6473 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2])) 6474 DONE; 6475 6476 operands[1] = force_reg (<MODE>mode, operands[1]); 6477}) 6478 6479(define_insn "*div<mode>3" 6480 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6481 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w") 6482 (match_operand:GPF_F16 2 "register_operand" "w")))] 6483 "TARGET_FLOAT" 6484 "fdiv\\t%<s>0, %<s>1, %<s>2" 6485 [(set_attr "type" "fdiv<stype>")] 6486) 6487 6488(define_insn "neg<mode>2" 6489 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6490 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6491 "TARGET_FLOAT" 6492 "fneg\\t%<s>0, %<s>1" 6493 [(set_attr "type" "ffarith<stype>")] 6494) 6495 6496(define_expand "sqrt<mode>2" 6497 [(set (match_operand:GPF_F16 0 "register_operand") 6498 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))] 6499 "TARGET_FLOAT" 6500{ 6501 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false)) 6502 DONE; 6503}) 6504 6505(define_insn "*sqrt<mode>2" 6506 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6507 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6508 "TARGET_FLOAT" 6509 "fsqrt\\t%<s>0, %<s>1" 6510 [(set_attr "type" "fsqrt<stype>")] 6511) 6512 6513(define_insn "abs<mode>2" 6514 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6515 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6516 "TARGET_FLOAT" 6517 "fabs\\t%<s>0, %<s>1" 6518 [(set_attr "type" "ffarith<stype>")] 6519) 6520 6521;; Given that smax/smin do not specify the result when either input is NaN, 6522;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN 6523;; for smin. 6524 6525(define_insn "smax<mode>3" 6526 [(set (match_operand:GPF 0 "register_operand" "=w") 6527 (smax:GPF (match_operand:GPF 1 "register_operand" "w") 6528 (match_operand:GPF 2 "register_operand" "w")))] 6529 "TARGET_FLOAT" 6530 "fmaxnm\\t%<s>0, %<s>1, %<s>2" 6531 [(set_attr "type" "f_minmax<s>")] 6532) 6533 6534(define_insn "smin<mode>3" 6535 [(set (match_operand:GPF 0 "register_operand" "=w") 6536 (smin:GPF (match_operand:GPF 1 "register_operand" "w") 6537 (match_operand:GPF 2 "register_operand" "w")))] 6538 "TARGET_FLOAT" 6539 "fminnm\\t%<s>0, %<s>1, %<s>2" 6540 [(set_attr "type" "f_minmax<s>")] 6541) 6542 6543;; Scalar forms for fmax, fmin, fmaxnm, fminnm. 6544;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names, 6545;; which implement the IEEE fmax ()/fmin () functions. 6546(define_insn "<maxmin_uns><mode>3" 6547 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6548 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w") 6549 (match_operand:GPF_F16 2 "register_operand" "w")] 6550 FMAXMIN_UNS))] 6551 "TARGET_FLOAT" 6552 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2" 6553 [(set_attr "type" "f_minmax<stype>")] 6554) 6555 6556(define_expand "lrint<GPF:mode><GPI:mode>2" 6557 [(match_operand:GPI 0 "register_operand") 6558 (match_operand:GPF 1 "register_operand")] 6559 "TARGET_FLOAT 6560 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE) 6561 || !flag_trapping_math || flag_fp_int_builtin_inexact)" 6562{ 6563 rtx cvt = gen_reg_rtx (<GPF:MODE>mode); 6564 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1])); 6565 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt)); 6566 DONE; 6567} 6568) 6569 6570;; For copysign (x, y), we want to generate: 6571;; 6572;; LDR d2, #(1 << 63) 6573;; BSL v2.8b, [y], [x] 6574;; 6575;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because 6576;; we expect these operations to nearly always operate on 6577;; floating-point values, we do not want the operation to be 6578;; simplified into a bit-field insert operation that operates on the 6579;; integer side, since typically that would involve three inter-bank 6580;; register copies. As we do not expect copysign to be followed by 6581;; other logical operations on the result, it seems preferable to keep 6582;; this as an unspec operation, rather than exposing the underlying 6583;; logic to the compiler. 6584 6585(define_expand "copysign<GPF:mode>3" 6586 [(match_operand:GPF 0 "register_operand") 6587 (match_operand:GPF 1 "register_operand") 6588 (match_operand:GPF 2 "register_operand")] 6589 "TARGET_FLOAT && TARGET_SIMD" 6590{ 6591 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode); 6592 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U 6593 << (GET_MODE_BITSIZE (<MODE>mode) - 1))); 6594 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2], 6595 bitmask)); 6596 DONE; 6597} 6598) 6599 6600(define_insn "copysign<GPF:mode>3_insn" 6601 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r") 6602 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r") 6603 (match_operand:GPF 2 "register_operand" "w,w,0,0") 6604 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")] 6605 UNSPEC_COPYSIGN))] 6606 "TARGET_FLOAT && TARGET_SIMD" 6607 "@ 6608 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype> 6609 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype> 6610 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype> 6611 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>" 6612 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")] 6613) 6614 6615 6616;; For xorsign (x, y), we want to generate: 6617;; 6618;; LDR d2, #1<<63 6619;; AND v3.8B, v1.8B, v2.8B 6620;; EOR v0.8B, v0.8B, v3.8B 6621;; 6622 6623(define_expand "xorsign<mode>3" 6624 [(match_operand:GPF 0 "register_operand") 6625 (match_operand:GPF 1 "register_operand") 6626 (match_operand:GPF 2 "register_operand")] 6627 "TARGET_FLOAT && TARGET_SIMD" 6628{ 6629 6630 machine_mode imode = <V_INT_EQUIV>mode; 6631 rtx mask = gen_reg_rtx (imode); 6632 rtx op1x = gen_reg_rtx (imode); 6633 rtx op2x = gen_reg_rtx (imode); 6634 6635 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1; 6636 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits, 6637 imode))); 6638 6639 emit_insn (gen_and<v_int_equiv>3 (op2x, mask, 6640 lowpart_subreg (imode, operands[2], 6641 <MODE>mode))); 6642 emit_insn (gen_xor<v_int_equiv>3 (op1x, 6643 lowpart_subreg (imode, operands[1], 6644 <MODE>mode), 6645 op2x)); 6646 emit_move_insn (operands[0], 6647 lowpart_subreg (<MODE>mode, op1x, imode)); 6648 DONE; 6649} 6650) 6651 6652;; ------------------------------------------------------------------- 6653;; Reload support 6654;; ------------------------------------------------------------------- 6655;; Reload Scalar Floating point modes from constant pool. 6656;; The AArch64 port doesn't have __int128 constant move support. 6657;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook. 6658(define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>" 6659 [(set (match_operand:GPF_TF 0 "register_operand" "=w") 6660 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S"))) 6661 (clobber (match_operand:P 2 "register_operand" "=&r"))] 6662 "TARGET_FLOAT" 6663 { 6664 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0)); 6665 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2])); 6666 DONE; 6667 } 6668) 6669 6670;; Reload Vector modes from constant pool. 6671(define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>" 6672 [(set (match_operand:VALL 0 "register_operand" "=w") 6673 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S"))) 6674 (clobber (match_operand:P 2 "register_operand" "=&r"))] 6675 "TARGET_FLOAT" 6676 { 6677 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0)); 6678 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2])); 6679 DONE; 6680 } 6681) 6682 6683(define_expand "@aarch64_reload_mov<mode>" 6684 [(set (match_operand:TX 0 "register_operand" "=w") 6685 (match_operand:TX 1 "register_operand" "w")) 6686 (clobber (match_operand:DI 2 "register_operand" "=&r")) 6687 ] 6688 "TARGET_FLOAT" 6689 { 6690 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0); 6691 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0); 6692 gen_aarch64_movtilow_tilow (op0, op1); 6693 gen_aarch64_movdi_tihigh (operands[2], op1); 6694 gen_aarch64_movtihigh_di (op0, operands[2]); 6695 DONE; 6696 } 6697) 6698 6699;; The following secondary reload helpers patterns are invoked 6700;; after or during reload as we don't want these patterns to start 6701;; kicking in during the combiner. 6702 6703(define_insn "@aarch64_movdi_<mode>low" 6704 [(set (match_operand:DI 0 "register_operand" "=r") 6705 (zero_extract:DI (match_operand:TX 1 "register_operand" "w") 6706 (const_int 64) (const_int 0)))] 6707 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6708 "fmov\\t%x0, %d1" 6709 [(set_attr "type" "f_mrc") 6710 (set_attr "length" "4") 6711 ]) 6712 6713(define_insn "@aarch64_movdi_<mode>high" 6714 [(set (match_operand:DI 0 "register_operand" "=r") 6715 (zero_extract:DI (match_operand:TX 1 "register_operand" "w") 6716 (const_int 64) (const_int 64)))] 6717 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6718 "fmov\\t%x0, %1.d[1]" 6719 [(set_attr "type" "f_mrc") 6720 (set_attr "length" "4") 6721 ]) 6722 6723(define_insn "@aarch64_mov<mode>high_di" 6724 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w") 6725 (const_int 64) (const_int 64)) 6726 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))] 6727 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6728 "fmov\\t%0.d[1], %x1" 6729 [(set_attr "type" "f_mcr") 6730 (set_attr "length" "4") 6731 ]) 6732 6733(define_insn "@aarch64_mov<mode>low_di" 6734 [(set (match_operand:TX 0 "register_operand" "=w") 6735 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))] 6736 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6737 "fmov\\t%d0, %x1" 6738 [(set_attr "type" "f_mcr") 6739 (set_attr "length" "4") 6740 ]) 6741 6742(define_insn "aarch64_movtilow_tilow" 6743 [(set (match_operand:TI 0 "register_operand" "=w") 6744 (zero_extend:TI 6745 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))] 6746 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6747 "fmov\\t%d0, %d1" 6748 [(set_attr "type" "fmov") 6749 (set_attr "length" "4") 6750 ]) 6751 6752;; There is a deliberate reason why the parameters of high and lo_sum's 6753;; don't have modes for ADRP and ADD instructions. This is to allow high 6754;; and lo_sum's to be used with the labels defining the jump tables in 6755;; rodata section. 6756 6757(define_expand "add_losym" 6758 [(set (match_operand 0 "register_operand") 6759 (lo_sum (match_operand 1 "register_operand") 6760 (match_operand 2 "aarch64_valid_symref")))] 6761 "" 6762{ 6763 machine_mode mode = GET_MODE (operands[0]); 6764 6765 emit_insn ((mode == DImode 6766 ? gen_add_losym_di 6767 : gen_add_losym_si) (operands[0], 6768 operands[1], 6769 operands[2])); 6770 DONE; 6771}) 6772 6773(define_insn "add_losym_<mode>" 6774 [(set (match_operand:P 0 "register_operand" "=r") 6775 (lo_sum:P (match_operand:P 1 "register_operand" "r") 6776 (match_operand 2 "aarch64_valid_symref" "S")))] 6777 "" 6778 "add\\t%<w>0, %<w>1, :lo12:%c2" 6779 [(set_attr "type" "alu_imm")] 6780) 6781 6782(define_insn "ldr_got_small_<mode>" 6783 [(set (match_operand:PTR 0 "register_operand" "=r") 6784 (unspec:PTR [(mem:PTR (lo_sum:PTR 6785 (match_operand:PTR 1 "register_operand" "r") 6786 (match_operand:PTR 2 "aarch64_valid_symref" "S")))] 6787 UNSPEC_GOTSMALLPIC))] 6788 "" 6789 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]" 6790 [(set_attr "type" "load_<ldst_sz>")] 6791) 6792 6793(define_insn "ldr_got_small_sidi" 6794 [(set (match_operand:DI 0 "register_operand" "=r") 6795 (zero_extend:DI 6796 (unspec:SI [(mem:SI (lo_sum:DI 6797 (match_operand:DI 1 "register_operand" "r") 6798 (match_operand:DI 2 "aarch64_valid_symref" "S")))] 6799 UNSPEC_GOTSMALLPIC)))] 6800 "TARGET_ILP32" 6801 "ldr\\t%w0, [%1, #:got_lo12:%c2]" 6802 [(set_attr "type" "load_4")] 6803) 6804 6805(define_insn "ldr_got_small_28k_<mode>" 6806 [(set (match_operand:PTR 0 "register_operand" "=r") 6807 (unspec:PTR [(mem:PTR (lo_sum:PTR 6808 (match_operand:PTR 1 "register_operand" "r") 6809 (match_operand:PTR 2 "aarch64_valid_symref" "S")))] 6810 UNSPEC_GOTSMALLPIC28K))] 6811 "" 6812 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]" 6813 [(set_attr "type" "load_<ldst_sz>")] 6814) 6815 6816(define_insn "ldr_got_small_28k_sidi" 6817 [(set (match_operand:DI 0 "register_operand" "=r") 6818 (zero_extend:DI 6819 (unspec:SI [(mem:SI (lo_sum:DI 6820 (match_operand:DI 1 "register_operand" "r") 6821 (match_operand:DI 2 "aarch64_valid_symref" "S")))] 6822 UNSPEC_GOTSMALLPIC28K)))] 6823 "TARGET_ILP32" 6824 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]" 6825 [(set_attr "type" "load_4")] 6826) 6827 6828(define_insn "@ldr_got_tiny_<mode>" 6829 [(set (match_operand:PTR 0 "register_operand" "=r") 6830 (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")] 6831 UNSPEC_GOTTINYPIC))] 6832 "" 6833 "ldr\t%<w>0, %L1" 6834 [(set_attr "type" "load_<ldst_sz>")] 6835) 6836 6837(define_insn "ldr_got_tiny_sidi" 6838 [(set (match_operand:DI 0 "register_operand" "=r") 6839 (zero_extend:DI 6840 (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")] 6841 UNSPEC_GOTTINYPIC)))] 6842 "TARGET_ILP32" 6843 "ldr\t%w0, %L1" 6844 [(set_attr "type" "load_4")] 6845) 6846 6847(define_insn "aarch64_load_tp_hard" 6848 [(set (match_operand:DI 0 "register_operand" "=r") 6849 (unspec:DI [(const_int 0)] UNSPEC_TLS))] 6850 "" 6851 "mrs\\t%0, tpidr_el0" 6852 [(set_attr "type" "mrs")] 6853) 6854 6855;; The TLS ABI specifically requires that the compiler does not schedule 6856;; instructions in the TLS stubs, in order to enable linker relaxation. 6857;; Therefore we treat the stubs as an atomic sequence. 6858(define_expand "tlsgd_small_<mode>" 6859 [(parallel [(set (match_operand:PTR 0 "register_operand") 6860 (call (mem:DI (match_dup 2)) (const_int 1))) 6861 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI) 6862 (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS) 6863 (clobber (reg:DI LR_REGNUM))])] 6864 "" 6865{ 6866 operands[2] = aarch64_tls_get_addr (); 6867}) 6868 6869(define_insn "*tlsgd_small_<mode>" 6870 [(set (match_operand:PTR 0 "register_operand" "") 6871 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1))) 6872 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI) 6873 (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS) 6874 (clobber (reg:DI LR_REGNUM)) 6875 ] 6876 "" 6877 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop" 6878 [(set_attr "type" "call") 6879 (set_attr "length" "16")]) 6880 6881(define_insn "tlsie_small_<mode>" 6882 [(set (match_operand:PTR 0 "register_operand" "=r") 6883 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")] 6884 UNSPEC_GOTSMALLTLS))] 6885 "" 6886 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]" 6887 [(set_attr "type" "load_4") 6888 (set_attr "length" "8")] 6889) 6890 6891(define_insn "tlsie_small_sidi" 6892 [(set (match_operand:DI 0 "register_operand" "=r") 6893 (zero_extend:DI 6894 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")] 6895 UNSPEC_GOTSMALLTLS)))] 6896 "" 6897 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]" 6898 [(set_attr "type" "load_4") 6899 (set_attr "length" "8")] 6900) 6901 6902(define_insn "tlsie_tiny_<mode>" 6903 [(set (match_operand:PTR 0 "register_operand" "=&r") 6904 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S") 6905 (match_operand:PTR 2 "register_operand" "r")] 6906 UNSPEC_GOTTINYTLS))] 6907 "" 6908 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2" 6909 [(set_attr "type" "multiple") 6910 (set_attr "length" "8")] 6911) 6912 6913(define_insn "tlsie_tiny_sidi" 6914 [(set (match_operand:DI 0 "register_operand" "=&r") 6915 (zero_extend:DI 6916 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S") 6917 (match_operand:DI 2 "register_operand" "r") 6918 ] 6919 UNSPEC_GOTTINYTLS)))] 6920 "" 6921 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2" 6922 [(set_attr "type" "multiple") 6923 (set_attr "length" "8")] 6924) 6925 6926(define_insn "tlsle12_<mode>" 6927 [(set (match_operand:P 0 "register_operand" "=r") 6928 (unspec:P [(match_operand:P 1 "register_operand" "r") 6929 (match_operand 2 "aarch64_tls_le_symref" "S")] 6930 UNSPEC_TLSLE12))] 6931 "" 6932 "add\\t%<w>0, %<w>1, #%L2"; 6933 [(set_attr "type" "alu_sreg") 6934 (set_attr "length" "4")] 6935) 6936 6937(define_insn "tlsle24_<mode>" 6938 [(set (match_operand:P 0 "register_operand" "=r") 6939 (unspec:P [(match_operand:P 1 "register_operand" "r") 6940 (match_operand 2 "aarch64_tls_le_symref" "S")] 6941 UNSPEC_TLSLE24))] 6942 "" 6943 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2" 6944 [(set_attr "type" "multiple") 6945 (set_attr "length" "8")] 6946) 6947 6948(define_insn "tlsle32_<mode>" 6949 [(set (match_operand:P 0 "register_operand" "=r") 6950 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")] 6951 UNSPEC_TLSLE32))] 6952 "" 6953 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1" 6954 [(set_attr "type" "multiple") 6955 (set_attr "length" "8")] 6956) 6957 6958(define_insn "tlsle48_<mode>" 6959 [(set (match_operand:P 0 "register_operand" "=r") 6960 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")] 6961 UNSPEC_TLSLE48))] 6962 "" 6963 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1" 6964 [(set_attr "type" "multiple") 6965 (set_attr "length" "12")] 6966) 6967 6968(define_expand "tlsdesc_small_<mode>" 6969 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)] 6970 "TARGET_TLS_DESC" 6971 { 6972 if (TARGET_SVE) 6973 { 6974 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode); 6975 rtx_insn *call 6976 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi)); 6977 RTL_CONST_CALL_P (call) = 1; 6978 } 6979 else 6980 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0])); 6981 DONE; 6982 } 6983) 6984 6985;; tlsdesc calls preserve all core and Advanced SIMD registers except 6986;; R0 and LR. 6987(define_insn "tlsdesc_small_advsimd_<mode>" 6988 [(set (reg:PTR R0_REGNUM) 6989 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")] 6990 UNSPEC_TLSDESC)) 6991 (clobber (reg:DI LR_REGNUM)) 6992 (clobber (reg:CC CC_REGNUM)) 6993 (clobber (match_scratch:DI 1 "=r")) 6994 (use (reg:DI FP_REGNUM))] 6995 "TARGET_TLS_DESC && !TARGET_SVE" 6996 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1" 6997 [(set_attr "type" "call") 6998 (set_attr "length" "16")]) 6999 7000;; For SVE, model tlsdesc calls as normal calls, with the callee ABI 7001;; describing the extra call-preserved guarantees. This would work 7002;; for non-SVE too, but avoiding a call is probably better if we can. 7003(define_insn "tlsdesc_small_sve_<mode>" 7004 [(set (reg:PTR R0_REGNUM) 7005 (call (mem:DI (unspec:PTR 7006 [(match_operand 0 "aarch64_valid_symref")] 7007 UNSPEC_TLSDESC)) 7008 (const_int 0))) 7009 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI) 7010 (clobber (reg:DI LR_REGNUM)) 7011 (clobber (match_scratch:DI 2 "=r"))] 7012 "TARGET_TLS_DESC && TARGET_SVE" 7013 "adrp\\tx0, %A0\;ldr\\t%<w>2, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%2" 7014 [(set_attr "type" "call") 7015 (set_attr "length" "16")]) 7016 7017(define_insn "stack_tie" 7018 [(set (mem:BLK (scratch)) 7019 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk") 7020 (match_operand:DI 1 "register_operand" "rk")] 7021 UNSPEC_PRLG_STK))] 7022 "" 7023 "" 7024 [(set_attr "length" "0")] 7025) 7026 7027(define_insn "aarch64_fjcvtzs" 7028 [(set (match_operand:SI 0 "register_operand" "=r") 7029 (unspec:SI [(match_operand:DF 1 "register_operand" "w")] 7030 UNSPEC_FJCVTZS)) 7031 (clobber (reg:CC CC_REGNUM))] 7032 "TARGET_JSCVT" 7033 "fjcvtzs\\t%w0, %d1" 7034 [(set_attr "type" "f_cvtf2i")] 7035) 7036 7037;; Pointer authentication patterns are always provided. In architecture 7038;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs. 7039;; This lets the user write portable software which authenticates pointers 7040;; when run on something which implements ARMv8.3-A, and which runs 7041;; correctly, but does not authenticate pointers, where ARMv8.3-A is not 7042;; implemented. 7043 7044;; Signing/Authenticating R30 using SP as the salt. 7045 7046(define_insn "<pauth_mnem_prefix>sp" 7047 [(set (reg:DI R30_REGNUM) 7048 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))] 7049 "" 7050 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp"; 7051) 7052 7053;; Signing/Authenticating X17 using X16 as the salt. 7054 7055(define_insn "<pauth_mnem_prefix>1716" 7056 [(set (reg:DI R17_REGNUM) 7057 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))] 7058 "" 7059 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716"; 7060) 7061 7062;; Stripping the signature in R30. 7063 7064(define_insn "xpaclri" 7065 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))] 7066 "" 7067 "hint\t7 // xpaclri" 7068) 7069 7070;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and 7071;; all of memory. This blocks insns from being moved across this point. 7072 7073(define_insn "blockage" 7074 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)] 7075 "" 7076 "" 7077 [(set_attr "length" "0") 7078 (set_attr "type" "block")] 7079) 7080 7081(define_insn "probe_stack_range" 7082 [(set (match_operand:DI 0 "register_operand" "=rk") 7083 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0") 7084 (match_operand:DI 2 "register_operand" "r")] 7085 UNSPECV_PROBE_STACK_RANGE))] 7086 "" 7087{ 7088 return aarch64_output_probe_stack_range (operands[0], operands[2]); 7089} 7090 [(set_attr "length" "32")] 7091) 7092 7093;; This instruction is used to generate the stack clash stack adjustment and 7094;; probing loop. We can't change the control flow during prologue and epilogue 7095;; code generation. So we must emit a volatile unspec and expand it later on. 7096 7097(define_insn "@probe_sve_stack_clash_<mode>" 7098 [(set (match_operand:P 0 "register_operand" "=rk") 7099 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0") 7100 (match_operand:P 2 "register_operand" "r") 7101 (match_operand:P 3 "const_int_operand" "n") 7102 (match_operand:P 4 "aarch64_plus_immediate" "L")] 7103 UNSPECV_PROBE_STACK_RANGE))] 7104 "TARGET_SVE" 7105{ 7106 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2], 7107 operands[3], operands[4]); 7108} 7109 [(set_attr "length" "28")] 7110) 7111 7112;; Named pattern for expanding thread pointer reference. 7113(define_expand "get_thread_pointerdi" 7114 [(match_operand:DI 0 "register_operand")] 7115 "" 7116{ 7117 rtx tmp = aarch64_load_tp (operands[0]); 7118 if (tmp != operands[0]) 7119 emit_move_insn (operands[0], tmp); 7120 DONE; 7121}) 7122 7123;; Defined for -mstack-protector-guard=sysreg, which goes through this 7124;; pattern rather than stack_protect_combined_set. Our implementation 7125;; of the latter can handle both. 7126(define_expand "stack_protect_set" 7127 [(match_operand 0 "memory_operand") 7128 (match_operand 1 "")] 7129 "" 7130{ 7131 emit_insn (gen_stack_protect_combined_set (operands[0], operands[1])); 7132 DONE; 7133}) 7134 7135(define_expand "stack_protect_combined_set" 7136 [(match_operand 0 "memory_operand") 7137 (match_operand 1 "")] 7138 "" 7139{ 7140 machine_mode mode = GET_MODE (operands[0]); 7141 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1], 7142 AARCH64_SALT_SSP_SET); 7143 emit_insn ((mode == DImode 7144 ? gen_stack_protect_set_di 7145 : gen_stack_protect_set_si) (operands[0], operands[1])); 7146 DONE; 7147}) 7148 7149;; Operand 1 is either AARCH64_SALT_SSP_SET or AARCH64_SALT_SSP_TEST. 7150(define_insn "reg_stack_protect_address_<mode>" 7151 [(set (match_operand:PTR 0 "register_operand" "=r") 7152 (unspec:PTR [(match_operand 1 "const_int_operand")] 7153 UNSPEC_SSP_SYSREG))] 7154 "aarch64_stack_protector_guard != SSP_GLOBAL" 7155 { 7156 char buf[150]; 7157 snprintf (buf, 150, "mrs\\t%%<w>0, %s", 7158 aarch64_stack_protector_guard_reg_str); 7159 output_asm_insn (buf, operands); 7160 return ""; 7161 } 7162 [(set_attr "type" "mrs")]) 7163 7164;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the 7165;; canary value does not live beyond the life of this sequence. 7166(define_insn "stack_protect_set_<mode>" 7167 [(set (match_operand:PTR 0 "memory_operand" "=m") 7168 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")] 7169 UNSPEC_SP_SET)) 7170 (set (match_scratch:PTR 2 "=&r") (const_int 0))] 7171 "" 7172 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0" 7173 [(set_attr "length" "12") 7174 (set_attr "type" "multiple")]) 7175 7176;; Defined for -mstack-protector-guard=sysreg, which goes through this 7177;; pattern rather than stack_protect_combined_test. Our implementation 7178;; of the latter can handle both. 7179(define_expand "stack_protect_test" 7180 [(match_operand 0 "memory_operand") 7181 (match_operand 1 "") 7182 (match_operand 2)] 7183 "" 7184{ 7185 emit_insn (gen_stack_protect_combined_test (operands[0], operands[1], 7186 operands[2])); 7187 DONE; 7188}) 7189 7190(define_expand "stack_protect_combined_test" 7191 [(match_operand 0 "memory_operand") 7192 (match_operand 1 "") 7193 (match_operand 2)] 7194 "" 7195{ 7196 machine_mode mode = GET_MODE (operands[0]); 7197 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1], 7198 AARCH64_SALT_SSP_TEST); 7199 emit_insn ((mode == DImode 7200 ? gen_stack_protect_test_di 7201 : gen_stack_protect_test_si) (operands[0], operands[1])); 7202 7203 rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM); 7204 emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx), 7205 cc_reg, operands[2])); 7206 DONE; 7207}) 7208 7209;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the 7210;; canary value does not live beyond the end of this sequence. 7211(define_insn "stack_protect_test_<mode>" 7212 [(set (reg:CC CC_REGNUM) 7213 (unspec:CC [(match_operand:PTR 0 "memory_operand" "m") 7214 (match_operand:PTR 1 "memory_operand" "m")] 7215 UNSPEC_SP_TEST)) 7216 (clobber (match_scratch:PTR 2 "=&r")) 7217 (clobber (match_scratch:PTR 3 "=&r"))] 7218 "" 7219 "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0" 7220 [(set_attr "length" "16") 7221 (set_attr "type" "multiple")]) 7222 7223;; Write Floating-point Control Register. 7224(define_insn "set_fpcr" 7225 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)] 7226 "" 7227 "msr\\tfpcr, %0" 7228 [(set_attr "type" "mrs")]) 7229 7230;; Read Floating-point Control Register. 7231(define_insn "get_fpcr" 7232 [(set (match_operand:SI 0 "register_operand" "=r") 7233 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))] 7234 "" 7235 "mrs\\t%0, fpcr" 7236 [(set_attr "type" "mrs")]) 7237 7238;; Write Floating-point Status Register. 7239(define_insn "set_fpsr" 7240 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)] 7241 "" 7242 "msr\\tfpsr, %0" 7243 [(set_attr "type" "mrs")]) 7244 7245;; Read Floating-point Status Register. 7246(define_insn "get_fpsr" 7247 [(set (match_operand:SI 0 "register_operand" "=r") 7248 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))] 7249 "" 7250 "mrs\\t%0, fpsr" 7251 [(set_attr "type" "mrs")]) 7252 7253 7254;; Define the subtract-one-and-jump insns so loop.c 7255;; knows what to generate. 7256(define_expand "doloop_end" 7257 [(use (match_operand 0 "" "")) ; loop pseudo 7258 (use (match_operand 1 "" ""))] ; label 7259 "optimize > 0 && flag_modulo_sched" 7260{ 7261 rtx s0; 7262 rtx bcomp; 7263 rtx loc_ref; 7264 rtx cc_reg; 7265 rtx insn; 7266 rtx cmp; 7267 7268 /* Currently SMS relies on the do-loop pattern to recognize loops 7269 where (1) the control part consists of all insns defining and/or 7270 using a certain 'count' register and (2) the loop count can be 7271 adjusted by modifying this register prior to the loop. 7272 ??? The possible introduction of a new block to initialize the 7273 new IV can potentially affect branch optimizations. */ 7274 7275 if (GET_MODE (operands[0]) != DImode) 7276 FAIL; 7277 7278 s0 = operands [0]; 7279 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1))); 7280 7281 cmp = XVECEXP (PATTERN (insn), 0, 0); 7282 cc_reg = SET_DEST (cmp); 7283 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx); 7284 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]); 7285 emit_jump_insn (gen_rtx_SET (pc_rtx, 7286 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp, 7287 loc_ref, pc_rtx))); 7288 DONE; 7289}) 7290 7291;; Track speculation through conditional branches. We assume that 7292;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary. 7293(define_insn "speculation_tracker" 7294 [(set (reg:DI SPECULATION_TRACKER_REGNUM) 7295 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)] 7296 UNSPEC_SPECULATION_TRACKER))] 7297 "" 7298 { 7299 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7300 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands); 7301 return ""; 7302 } 7303 [(set_attr "type" "csel")] 7304) 7305 7306;; Like speculation_tracker, but track the inverse condition. 7307(define_insn "speculation_tracker_rev" 7308 [(set (reg:DI SPECULATION_TRACKER_REGNUM) 7309 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)] 7310 UNSPEC_SPECULATION_TRACKER_REV))] 7311 "" 7312 { 7313 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7314 output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands); 7315 return ""; 7316 } 7317 [(set_attr "type" "csel")] 7318) 7319 7320;; BTI <target> instructions 7321(define_insn "bti_noarg" 7322 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)] 7323 "" 7324 "hint\t32 // bti" 7325 [(set_attr "type" "no_insn")] 7326) 7327 7328(define_insn "bti_c" 7329 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)] 7330 "" 7331 "hint\t34 // bti c" 7332 [(set_attr "type" "no_insn")] 7333) 7334 7335(define_insn "bti_j" 7336 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)] 7337 "" 7338 "hint\t36 // bti j" 7339 [(set_attr "type" "no_insn")] 7340) 7341 7342(define_insn "bti_jc" 7343 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)] 7344 "" 7345 "hint\t38 // bti jc" 7346 [(set_attr "type" "no_insn")] 7347) 7348 7349;; Hard speculation barrier. 7350(define_insn "speculation_barrier" 7351 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)] 7352 "" 7353 "isb\;dsb\\tsy" 7354 [(set_attr "length" "8") 7355 (set_attr "type" "block") 7356 (set_attr "speculation_barrier" "true")] 7357) 7358 7359;; Support for __builtin_speculation_safe_value when we have speculation 7360;; tracking enabled. Use the speculation tracker to decide whether to 7361;; copy operand 1 to the target, or to copy the fail value (operand 2). 7362(define_expand "@despeculate_copy<ALLI_TI:mode>" 7363 [(set (match_operand:ALLI_TI 0 "register_operand") 7364 (unspec_volatile:ALLI_TI 7365 [(match_operand:ALLI_TI 1 "register_operand") 7366 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero") 7367 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7368 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7369 "" 7370 " 7371 { 7372 if (operands[2] == const0_rtx) 7373 { 7374 rtx tracker; 7375 if (<MODE>mode == TImode) 7376 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7377 else 7378 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM); 7379 7380 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1], 7381 tracker)); 7382 DONE; 7383 } 7384 } 7385 " 7386) 7387 7388;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the 7389;; encoding for CSDB, but will work in older versions of the assembler. 7390(define_insn "*despeculate_copy<ALLI:mode>_insn" 7391 [(set (match_operand:ALLI 0 "register_operand" "=r") 7392 (unspec_volatile:ALLI 7393 [(match_operand:ALLI 1 "register_operand" "r") 7394 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ") 7395 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7396 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7397 "" 7398 { 7399 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7400 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb", 7401 operands); 7402 return ""; 7403 } 7404 [(set_attr "length" "12") 7405 (set_attr "type" "block") 7406 (set_attr "speculation_barrier" "true")] 7407) 7408 7409;; Pattern to match despeculate_copyti 7410(define_insn "*despeculate_copyti_insn" 7411 [(set (match_operand:TI 0 "register_operand" "=r") 7412 (unspec_volatile:TI 7413 [(match_operand:TI 1 "register_operand" "r") 7414 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ") 7415 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7416 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7417 "" 7418 { 7419 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7420 output_asm_insn 7421 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb", 7422 operands); 7423 return ""; 7424 } 7425 [(set_attr "length" "16") 7426 (set_attr "type" "block") 7427 (set_attr "speculation_barrier" "true")] 7428) 7429 7430(define_insn "despeculate_simple<ALLI:mode>" 7431 [(set (match_operand:ALLI 0 "register_operand" "=r") 7432 (unspec_volatile:ALLI 7433 [(match_operand:ALLI 1 "register_operand" "r") 7434 (use (match_operand:ALLI 2 "register_operand" ""))] 7435 UNSPECV_SPECULATION_BARRIER))] 7436 "" 7437 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb" 7438 [(set_attr "type" "block") 7439 (set_attr "length" "8") 7440 (set_attr "speculation_barrier" "true")] 7441) 7442 7443(define_insn "despeculate_simpleti" 7444 [(set (match_operand:TI 0 "register_operand" "=r") 7445 (unspec_volatile:TI 7446 [(match_operand:TI 1 "register_operand" "r") 7447 (use (match_operand:DI 2 "register_operand" ""))] 7448 UNSPECV_SPECULATION_BARRIER))] 7449 "" 7450 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb" 7451 [(set_attr "type" "block") 7452 (set_attr "length" "12") 7453 (set_attr "speculation_barrier" "true")] 7454) 7455 7456(define_insn "aarch64_<frintnzs_op><mode>" 7457 [(set (match_operand:VSFDF 0 "register_operand" "=w") 7458 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")] 7459 FRINTNZX))] 7460 "TARGET_FRINT && TARGET_FLOAT 7461 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)" 7462 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>" 7463 [(set_attr "type" "f_rint<stype>")] 7464) 7465 7466;; Transactional Memory Extension (TME) instructions. 7467 7468(define_insn "tstart" 7469 [(set (match_operand:DI 0 "register_operand" "=r") 7470 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART)) 7471 (clobber (mem:BLK (scratch)))] 7472 "TARGET_TME" 7473 "tstart\\t%0" 7474 [(set_attr "type" "tme")] 7475) 7476 7477(define_insn "ttest" 7478 [(set (match_operand:DI 0 "register_operand" "=r") 7479 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST)) 7480 (clobber (mem:BLK (scratch)))] 7481 "TARGET_TME" 7482 "ttest\\t%0" 7483 [(set_attr "type" "tme")] 7484) 7485 7486(define_insn "tcommit" 7487 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT) 7488 (clobber (mem:BLK (scratch)))] 7489 "TARGET_TME" 7490 "tcommit" 7491 [(set_attr "type" "tme")] 7492) 7493 7494(define_insn "tcancel" 7495 [(unspec_volatile:BLK 7496 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL) 7497 (clobber (mem:BLK (scratch)))] 7498 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)" 7499 "tcancel\\t#%0" 7500 [(set_attr "type" "tme")] 7501) 7502 7503(define_insn "aarch64_rndr" 7504 [(set (match_operand:DI 0 "register_operand" "=r") 7505 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR)) 7506 (set (reg:CC_Z CC_REGNUM) 7507 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))] 7508 "TARGET_RNG" 7509 "mrs\t%0, RNDR" 7510 [(set_attr "type" "mrs")] 7511) 7512 7513(define_insn "aarch64_rndrrs" 7514 [(set (match_operand:DI 0 "register_operand" "=r") 7515 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS)) 7516 (set (reg:CC_Z CC_REGNUM) 7517 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))] 7518 "TARGET_RNG" 7519 "mrs\t%0, RNDRRS" 7520 [(set_attr "type" "mrs")] 7521) 7522 7523;; Memory Tagging Extension (MTE) instructions. 7524 7525(define_insn "irg" 7526 [(set (match_operand:DI 0 "register_operand" "=rk") 7527 (ior:DI 7528 (and:DI (match_operand:DI 1 "register_operand" "rk") 7529 (const_int -1080863910568919041)) ;; 0xf0ff... 7530 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")] 7531 UNSPEC_GEN_TAG_RND) 7532 (const_int 56))))] 7533 "TARGET_MEMTAG" 7534 "irg\\t%0, %1, %2" 7535 [(set_attr "type" "memtag")] 7536) 7537 7538(define_insn "gmi" 7539 [(set (match_operand:DI 0 "register_operand" "=r") 7540 (ior:DI (ashift:DI 7541 (const_int 1) 7542 (and:QI (lshiftrt:DI 7543 (match_operand:DI 1 "register_operand" "rk") 7544 (const_int 56)) (const_int 15))) 7545 (match_operand:DI 2 "register_operand" "r")))] 7546 "TARGET_MEMTAG" 7547 "gmi\\t%0, %1, %2" 7548 [(set_attr "type" "memtag")] 7549) 7550 7551(define_insn "addg" 7552 [(set (match_operand:DI 0 "register_operand" "=rk") 7553 (ior:DI 7554 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk") 7555 (match_operand:DI 2 "aarch64_granule16_uimm6" "i")) 7556 (const_int -1080863910568919041)) ;; 0xf0ff... 7557 (ashift:DI 7558 (unspec:QI 7559 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15)) 7560 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")] 7561 UNSPEC_GEN_TAG) 7562 (const_int 56))))] 7563 "TARGET_MEMTAG" 7564 "addg\\t%0, %1, #%2, #%3" 7565 [(set_attr "type" "memtag")] 7566) 7567 7568(define_insn "subp" 7569 [(set (match_operand:DI 0 "register_operand" "=r") 7570 (minus:DI 7571 (and:DI (match_operand:DI 1 "register_operand" "rk") 7572 (const_int 72057594037927935)) ;; 0x00ff... 7573 (and:DI (match_operand:DI 2 "register_operand" "rk") 7574 (const_int 72057594037927935))))] ;; 0x00ff... 7575 "TARGET_MEMTAG" 7576 "subp\\t%0, %1, %2" 7577 [(set_attr "type" "memtag")] 7578) 7579 7580;; LDG will use the 16-byte aligned value of the address. 7581(define_insn "ldg" 7582 [(set (match_operand:DI 0 "register_operand" "+r") 7583 (ior:DI 7584 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff... 7585 (ashift:DI 7586 (mem:QI (unspec:DI 7587 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk") 7588 (match_operand:DI 2 "aarch64_granule16_simm9" "i")) 7589 (const_int -16))] UNSPEC_TAG_SPACE)) 7590 (const_int 56))))] 7591 "TARGET_MEMTAG" 7592 "ldg\\t%0, [%1, #%2]" 7593 [(set_attr "type" "memtag")] 7594) 7595 7596;; STG doesn't align the address but aborts with alignment fault 7597;; when the address is not 16-byte aligned. 7598(define_insn "stg" 7599 [(set (mem:QI (unspec:DI 7600 [(plus:DI (match_operand:DI 1 "register_operand" "rk") 7601 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))] 7602 UNSPEC_TAG_SPACE)) 7603 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk") 7604 (const_int 56)) (const_int 15)))] 7605 "TARGET_MEMTAG" 7606 "stg\\t%0, [%1, #%2]" 7607 [(set_attr "type" "memtag")] 7608) 7609 7610(define_insn "patchable_area" 7611 [(unspec_volatile [(match_operand 0 "const_int_operand") 7612 (match_operand 1 "const_int_operand")] 7613 UNSPECV_PATCHABLE_AREA)] 7614 "" 7615{ 7616 aarch64_output_patchable_area (INTVAL (operands[0]), 7617 INTVAL (operands[1]) != 0); 7618 return ""; 7619} 7620 [(set (attr "length") (symbol_ref "INTVAL (operands[0])"))] 7621) 7622 7623;; AdvSIMD Stuff 7624(include "aarch64-simd.md") 7625 7626;; Atomic Operations 7627(include "atomics.md") 7628 7629;; ldp/stp peephole patterns 7630(include "aarch64-ldpstp.md") 7631 7632;; SVE. 7633(include "aarch64-sve.md") 7634 7635;; SVE2. 7636(include "aarch64-sve2.md") 7637