1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>; 203def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>; 204def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>; 205def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 206 207def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; 208 209def SVEArithUImmPat : ComplexPattern<i32, 1, "SelectSVEArithImm", []>; 210def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 211 212def SVEShiftImm64 : ComplexPattern<i32, 1, "SelectSVEShiftImm64<0, 64>", []>; 213 214class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 215 let Name = "SVEExactFPImmOperand" # Suffix; 216 let DiagnosticType = "Invalid" # Name; 217 let ParserMethod = "tryParseFPImm<false>"; 218 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 219 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 220} 221 222class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 223 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 224 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 225} 226 227def sve_fpimm_half_one 228 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 229 "AArch64ExactFPImm::one">; 230def sve_fpimm_half_two 231 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 232 "AArch64ExactFPImm::two">; 233def sve_fpimm_zero_one 234 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 235 "AArch64ExactFPImm::one">; 236 237def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 238 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 239}]> { 240 let ParserMatchClass = Imm1_16Operand; 241 let EncoderMethod = "getSVEIncDecImm"; 242 let DecoderMethod = "DecodeSVEIncDecImm"; 243} 244 245// This allows i32 immediate extraction from i64 based arithmetic. 246def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 247def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">; 248 249//===----------------------------------------------------------------------===// 250// SVE PTrue - These are used extensively throughout the pattern matching so 251// it's important we define them first. 252//===----------------------------------------------------------------------===// 253 254class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 255 ValueType vt, SDPatternOperator op> 256: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 257 asm, "\t$Pd, $pattern", 258 "", 259 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 260 bits<4> Pd; 261 bits<5> pattern; 262 let Inst{31-24} = 0b00100101; 263 let Inst{23-22} = sz8_64; 264 let Inst{21-19} = 0b011; 265 let Inst{18-17} = opc{2-1}; 266 let Inst{16} = opc{0}; 267 let Inst{15-10} = 0b111000; 268 let Inst{9-5} = pattern; 269 let Inst{4} = 0b0; 270 let Inst{3-0} = Pd; 271 272 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 273} 274 275multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 276 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 277 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 278 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 279 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 280 281 def : InstAlias<asm # "\t$Pd", 282 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 283 def : InstAlias<asm # "\t$Pd", 284 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 285 def : InstAlias<asm # "\t$Pd", 286 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 287 def : InstAlias<asm # "\t$Pd", 288 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 289} 290 291def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 292def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 293 294let Predicates = [HasSVE] in { 295 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 296 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 297} 298 299//===----------------------------------------------------------------------===// 300// SVE pattern match helpers. 301//===----------------------------------------------------------------------===// 302 303class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 304 Instruction inst> 305: Pat<(vtd (op vt1:$Op1)), 306 (inst $Op1)>; 307 308class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 309 ValueType it, ComplexPattern cpx, Instruction inst> 310 : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))), 311 (inst $Op1, i32:$imm, i32:$shift)>; 312 313class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 314 ValueType it, ComplexPattern cpx, Instruction inst> 315 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))), 316 (inst $Op1, i32:$imm, i32:$shift)>; 317 318class SVE_1_Op_Imm_Arith_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 319 ValueType it, ComplexPattern cpx, Instruction inst> 320 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))), 321 (inst $Op1, i32:$imm)>; 322 323class SVE_1_Op_Imm_Shift_Pred_Pat<ValueType vt, ValueType pt, SDPatternOperator op, 324 ZPRRegOp zprty, Operand ImmTy, Instruction inst> 325 : Pat<(vt (op (pt (AArch64ptrue 31)), (vt zprty:$Op1), (vt (AArch64dup (ImmTy:$imm))))), 326 (inst $Op1, ImmTy:$imm)>; 327 328class SVE_1_Op_Imm_Arith_Pred_Pat<ValueType vt, ValueType pt, SDPatternOperator op, 329 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 330 : Pat<(vt (op (pt (AArch64ptrue 31)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))), 331 (inst $Op1, i32:$imm)>; 332 333class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 334 ValueType it, ComplexPattern cpx, Instruction inst> 335 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))), 336 (inst $Op1, i64:$imm)>; 337 338class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 339 ValueType vt2, Instruction inst> 340: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 341 (inst $Op1, $Op2)>; 342 343class SVE_2_Op_Pat_Reduce_To_Neon<ValueType vtd, SDPatternOperator op, ValueType vt1, 344 ValueType vt2, Instruction inst, SubRegIndex sub> 345: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 346 (INSERT_SUBREG (vtd (IMPLICIT_DEF)), (inst $Op1, $Op2), sub)>; 347 348class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 349 ValueType vt2, ValueType vt3, Instruction inst> 350: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 351 (inst $Op1, $Op2, $Op3)>; 352 353class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 354 ValueType vt2, ValueType vt3, ValueType vt4, 355 Instruction inst> 356: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 357 (inst $Op1, $Op2, $Op3, $Op4)>; 358 359class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 360 ValueType vt2, Operand ImmTy, Instruction inst> 361: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 362 (inst $Op1, ImmTy:$Op2)>; 363 364class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 365 ValueType vt2, ValueType vt3, Operand ImmTy, 366 Instruction inst> 367: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 368 (inst $Op1, $Op2, ImmTy:$Op3)>; 369 370class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 371 ValueType vt2, ValueType vt3, ValueType vt4, 372 Operand ImmTy, Instruction inst> 373: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 374 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 375 376def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>; 377def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>; 378 379let AddedComplexity = 1 in { 380class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 381 ValueType vt2, ValueType vt3, Instruction inst> 382: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 383 (inst $Op1, $Op2, $Op3)>; 384 385class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 386 ValueType vt1, ValueType vt2, 387 Operand vt3, Instruction inst> 388: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 389 (inst $Op1, $Op2, vt3:$Op3)>; 390} 391 392// 393// Common but less generic patterns. 394// 395 396class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 397 Instruction inst, Instruction ptrue> 398: Pat<(vtd (op vt1:$Op1)), 399 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 400 401class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 402 ValueType vt2, Instruction inst, Instruction ptrue> 403: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 404 (inst (ptrue 31), $Op1, $Op2)>; 405 406// 407// Pseudo -> Instruction mappings 408// 409def getSVEPseudoMap : InstrMapping { 410 let FilterClass = "SVEPseudo2Instr"; 411 let RowFields = ["PseudoName"]; 412 let ColFields = ["IsInstr"]; 413 let KeyCol = ["0"]; 414 let ValueCols = [["1"]]; 415} 416 417class SVEPseudo2Instr<string name, bit instr> { 418 string PseudoName = name; 419 bit IsInstr = instr; 420} 421 422// Lookup e.g. DIV -> DIVR 423def getSVERevInstr : InstrMapping { 424 let FilterClass = "SVEInstr2Rev"; 425 let RowFields = ["InstrName"]; 426 let ColFields = ["isReverseInstr"]; 427 let KeyCol = ["0"]; 428 let ValueCols = [["1"]]; 429} 430 431// Lookup e.g. DIVR -> DIV 432def getSVENonRevInstr : InstrMapping { 433 let FilterClass = "SVEInstr2Rev"; 434 let RowFields = ["InstrName"]; 435 let ColFields = ["isReverseInstr"]; 436 let KeyCol = ["1"]; 437 let ValueCols = [["0"]]; 438} 439 440class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 441 string InstrName = !if(name1IsReverseInstr, name1, name2); 442 bit isReverseInstr = name1IsReverseInstr; 443} 444 445// 446// Pseudos for destructive operands 447// 448let hasNoSchedulingInfo = 1 in { 449 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 450 FalseLanesEnum flags = FalseLanesNone> 451 : SVEPseudo2Instr<name, 0>, 452 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 453 let FalseLanes = flags; 454 } 455 456 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 457 FalseLanesEnum flags = FalseLanesNone> 458 : SVEPseudo2Instr<name, 0>, 459 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 460 let FalseLanes = flags; 461 } 462} 463 464//===----------------------------------------------------------------------===// 465// SVE Predicate Misc Group 466//===----------------------------------------------------------------------===// 467 468class sve_int_pfalse<bits<6> opc, string asm> 469: I<(outs PPR8:$Pd), (ins), 470 asm, "\t$Pd", 471 "", 472 []>, Sched<[]> { 473 bits<4> Pd; 474 let Inst{31-24} = 0b00100101; 475 let Inst{23-22} = opc{5-4}; 476 let Inst{21-19} = 0b011; 477 let Inst{18-16} = opc{3-1}; 478 let Inst{15-10} = 0b111001; 479 let Inst{9} = opc{0}; 480 let Inst{8-4} = 0b00000; 481 let Inst{3-0} = Pd; 482} 483 484class sve_int_ptest<bits<6> opc, string asm> 485: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 486 asm, "\t$Pg, $Pn", 487 "", 488 []>, Sched<[]> { 489 bits<4> Pg; 490 bits<4> Pn; 491 let Inst{31-24} = 0b00100101; 492 let Inst{23-22} = opc{5-4}; 493 let Inst{21-19} = 0b010; 494 let Inst{18-16} = opc{3-1}; 495 let Inst{15-14} = 0b11; 496 let Inst{13-10} = Pg; 497 let Inst{9} = opc{0}; 498 let Inst{8-5} = Pn; 499 let Inst{4-0} = 0b00000; 500 501 let Defs = [NZCV]; 502} 503 504class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 505 PPRRegOp pprty> 506: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 507 asm, "\t$Pdn, $Pg, $_Pdn", 508 "", 509 []>, Sched<[]> { 510 bits<4> Pdn; 511 bits<4> Pg; 512 let Inst{31-24} = 0b00100101; 513 let Inst{23-22} = sz8_64; 514 let Inst{21-19} = 0b011; 515 let Inst{18-16} = opc{4-2}; 516 let Inst{15-11} = 0b11000; 517 let Inst{10-9} = opc{1-0}; 518 let Inst{8-5} = Pg; 519 let Inst{4} = 0; 520 let Inst{3-0} = Pdn; 521 522 let Constraints = "$Pdn = $_Pdn"; 523 let Defs = [NZCV]; 524} 525 526multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 527 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 528 529 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 530} 531 532multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 533 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 534 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 535 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 536 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 537 538 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 539 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 540 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 541 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 542} 543 544//===----------------------------------------------------------------------===// 545// SVE Predicate Count Group 546//===----------------------------------------------------------------------===// 547 548class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 549 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 550: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 551 asm, "\t$Rdn, $Pg", 552 "", 553 []>, Sched<[]> { 554 bits<5> Rdn; 555 bits<4> Pg; 556 let Inst{31-24} = 0b00100101; 557 let Inst{23-22} = sz8_64; 558 let Inst{21-19} = 0b101; 559 let Inst{18-16} = opc{4-2}; 560 let Inst{15-11} = 0b10001; 561 let Inst{10-9} = opc{1-0}; 562 let Inst{8-5} = Pg; 563 let Inst{4-0} = Rdn; 564 565 // Signed 32bit forms require their GPR operand printed. 566 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 567 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 568 !strconcat(asm, "\t$Rdn, $Pg")); 569 let Constraints = "$Rdn = $_Rdn"; 570} 571 572multiclass sve_int_count_r_s32<bits<5> opc, string asm, 573 SDPatternOperator op> { 574 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 575 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 576 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 577 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 578 579 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 580 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 581 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 582 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 583 584 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 585 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 586 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 587 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 588 589 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 590 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 591 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 592 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 593 594 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 595 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 596 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 597 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 598} 599 600multiclass sve_int_count_r_u32<bits<5> opc, string asm, 601 SDPatternOperator op> { 602 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 603 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 604 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 605 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 606 607 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 608 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 609 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 610 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 611 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 612 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 613 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 614 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 615} 616 617multiclass sve_int_count_r_x64<bits<5> opc, string asm, 618 SDPatternOperator op = null_frag> { 619 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 620 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 621 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 622 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 623 624 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 625 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 626 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 627 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 628 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 629 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 630 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 631 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 632} 633 634class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 635 ZPRRegOp zprty, PPRRegOp pprty> 636: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 637 asm, "\t$Zdn, $Pm", 638 "", 639 []>, Sched<[]> { 640 bits<4> Pm; 641 bits<5> Zdn; 642 let Inst{31-24} = 0b00100101; 643 let Inst{23-22} = sz8_64; 644 let Inst{21-19} = 0b101; 645 let Inst{18-16} = opc{4-2}; 646 let Inst{15-11} = 0b10000; 647 let Inst{10-9} = opc{1-0}; 648 let Inst{8-5} = Pm; 649 let Inst{4-0} = Zdn; 650 651 let Constraints = "$Zdn = $_Zdn"; 652 let DestructiveInstType = DestructiveOther; 653 let ElementSize = ElementSizeNone; 654} 655 656multiclass sve_int_count_v<bits<5> opc, string asm, 657 SDPatternOperator op = null_frag> { 658 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 659 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 660 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 661 662 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 663 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 664 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 665 666 def : InstAlias<asm # "\t$Zdn, $Pm", 667 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 668 def : InstAlias<asm # "\t$Zdn, $Pm", 669 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 670 def : InstAlias<asm # "\t$Zdn, $Pm", 671 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 672} 673 674class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 675 PPRRegOp pprty> 676: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 677 asm, "\t$Rd, $Pg, $Pn", 678 "", 679 []>, Sched<[]> { 680 bits<4> Pg; 681 bits<4> Pn; 682 bits<5> Rd; 683 let Inst{31-24} = 0b00100101; 684 let Inst{23-22} = sz8_64; 685 let Inst{21-19} = 0b100; 686 let Inst{18-16} = opc{3-1}; 687 let Inst{15-14} = 0b10; 688 let Inst{13-10} = Pg; 689 let Inst{9} = opc{0}; 690 let Inst{8-5} = Pn; 691 let Inst{4-0} = Rd; 692} 693 694multiclass sve_int_pcount_pred<bits<4> opc, string asm, 695 SDPatternOperator int_op> { 696 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 697 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 698 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 699 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 700 701 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 702 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 703 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 704 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 705} 706 707//===----------------------------------------------------------------------===// 708// SVE Element Count Group 709//===----------------------------------------------------------------------===// 710 711class sve_int_count<bits<3> opc, string asm> 712: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 713 asm, "\t$Rd, $pattern, mul $imm4", 714 "", 715 []>, Sched<[]> { 716 bits<5> Rd; 717 bits<4> imm4; 718 bits<5> pattern; 719 let Inst{31-24} = 0b00000100; 720 let Inst{23-22} = opc{2-1}; 721 let Inst{21-20} = 0b10; 722 let Inst{19-16} = imm4; 723 let Inst{15-11} = 0b11100; 724 let Inst{10} = opc{0}; 725 let Inst{9-5} = pattern; 726 let Inst{4-0} = Rd; 727} 728 729multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 730 def NAME : sve_int_count<opc, asm>; 731 732 def : InstAlias<asm # "\t$Rd, $pattern", 733 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 734 def : InstAlias<asm # "\t$Rd", 735 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 736 737 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), 738 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 739 740 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), 741 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 742 743 def : Pat<(i64 (op sve_pred_enum:$pattern)), 744 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 745} 746 747class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 748: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 749 asm, "\t$Zdn, $pattern, mul $imm4", 750 "", 751 []>, Sched<[]> { 752 bits<5> Zdn; 753 bits<5> pattern; 754 bits<4> imm4; 755 let Inst{31-24} = 0b00000100; 756 let Inst{23-22} = opc{4-3}; 757 let Inst{21} = 0b1; 758 let Inst{20} = opc{2}; 759 let Inst{19-16} = imm4; 760 let Inst{15-12} = 0b1100; 761 let Inst{11-10} = opc{1-0}; 762 let Inst{9-5} = pattern; 763 let Inst{4-0} = Zdn; 764 765 let Constraints = "$Zdn = $_Zdn"; 766 let DestructiveInstType = DestructiveOther; 767 let ElementSize = ElementSizeNone; 768} 769 770multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 771 SDPatternOperator op = null_frag, 772 ValueType vt = OtherVT> { 773 def NAME : sve_int_countvlv<opc, asm, zprty>; 774 775 def : InstAlias<asm # "\t$Zdn, $pattern", 776 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 777 def : InstAlias<asm # "\t$Zdn", 778 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 779 780 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 781 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 782} 783 784class sve_int_pred_pattern_a<bits<3> opc, string asm> 785: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 786 asm, "\t$Rdn, $pattern, mul $imm4", 787 "", 788 []>, Sched<[]> { 789 bits<5> Rdn; 790 bits<5> pattern; 791 bits<4> imm4; 792 let Inst{31-24} = 0b00000100; 793 let Inst{23-22} = opc{2-1}; 794 let Inst{21-20} = 0b11; 795 let Inst{19-16} = imm4; 796 let Inst{15-11} = 0b11100; 797 let Inst{10} = opc{0}; 798 let Inst{9-5} = pattern; 799 let Inst{4-0} = Rdn; 800 801 let Constraints = "$Rdn = $_Rdn"; 802} 803 804multiclass sve_int_pred_pattern_a<bits<3> opc, string asm> { 805 def NAME : sve_int_pred_pattern_a<opc, asm>; 806 807 def : InstAlias<asm # "\t$Rdn, $pattern", 808 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 809 def : InstAlias<asm # "\t$Rdn", 810 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 811} 812 813class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 814 RegisterOperand st> 815: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 816 asm, "\t$Rdn, $pattern, mul $imm4", 817 "", 818 []>, Sched<[]> { 819 bits<5> Rdn; 820 bits<5> pattern; 821 bits<4> imm4; 822 let Inst{31-24} = 0b00000100; 823 let Inst{23-22} = opc{4-3}; 824 let Inst{21} = 0b1; 825 let Inst{20} = opc{2}; 826 let Inst{19-16} = imm4; 827 let Inst{15-12} = 0b1111; 828 let Inst{11-10} = opc{1-0}; 829 let Inst{9-5} = pattern; 830 let Inst{4-0} = Rdn; 831 832 // Signed 32bit forms require their GPR operand printed. 833 let AsmString = !if(!eq(opc{2,0}, 0b00), 834 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 835 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 836 837 let Constraints = "$Rdn = $_Rdn"; 838} 839 840multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 841 SDPatternOperator op> { 842 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 843 844 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 845 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 846 def : InstAlias<asm # "\t$Rd, $Rn", 847 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 848 849 // NOTE: Register allocation doesn't like tied operands of differing register 850 // class, hence the extra INSERT_SUBREG complication. 851 852 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 853 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 854 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 855 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 856} 857 858multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 859 SDPatternOperator op> { 860 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 861 862 def : InstAlias<asm # "\t$Rdn, $pattern", 863 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 864 def : InstAlias<asm # "\t$Rdn", 865 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 866 867 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 868 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 869} 870 871multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 872 SDPatternOperator op> { 873 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 874 875 def : InstAlias<asm # "\t$Rdn, $pattern", 876 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 877 def : InstAlias<asm # "\t$Rdn", 878 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 879 880 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 881 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 882} 883 884 885//===----------------------------------------------------------------------===// 886// SVE Permute - Cross Lane Group 887//===----------------------------------------------------------------------===// 888 889class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 890 ValueType vt, RegisterClass srcRegType, 891 SDPatternOperator op> 892: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 893 asm, "\t$Zd, $Rn", 894 "", 895 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 896 bits<5> Rn; 897 bits<5> Zd; 898 let Inst{31-24} = 0b00000101; 899 let Inst{23-22} = sz8_64; 900 let Inst{21-10} = 0b100000001110; 901 let Inst{9-5} = Rn; 902 let Inst{4-0} = Zd; 903} 904 905multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 906 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 907 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 908 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 909 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 910 911 def : InstAlias<"mov $Zd, $Rn", 912 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 913 def : InstAlias<"mov $Zd, $Rn", 914 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 915 def : InstAlias<"mov $Zd, $Rn", 916 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 917 def : InstAlias<"mov $Zd, $Rn", 918 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 919} 920 921class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 922 ZPRRegOp zprty> 923: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 924 asm, "\t$Zd, $Zn$idx", 925 "", 926 []>, Sched<[]> { 927 bits<5> Zd; 928 bits<5> Zn; 929 bits<7> idx; 930 let Inst{31-24} = 0b00000101; 931 let Inst{23-22} = {?,?}; // imm3h 932 let Inst{21} = 0b1; 933 let Inst{20-16} = tsz; 934 let Inst{15-10} = 0b001000; 935 let Inst{9-5} = Zn; 936 let Inst{4-0} = Zd; 937} 938 939multiclass sve_int_perm_dup_i<string asm> { 940 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 941 let Inst{23-22} = idx{5-4}; 942 let Inst{20-17} = idx{3-0}; 943 } 944 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 945 let Inst{23-22} = idx{4-3}; 946 let Inst{20-18} = idx{2-0}; 947 } 948 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 949 let Inst{23-22} = idx{3-2}; 950 let Inst{20-19} = idx{1-0}; 951 } 952 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 953 let Inst{23-22} = idx{2-1}; 954 let Inst{20} = idx{0}; 955 } 956 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 957 let Inst{23-22} = idx{1-0}; 958 } 959 960 def : InstAlias<"mov $Zd, $Zn$idx", 961 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 962 def : InstAlias<"mov $Zd, $Zn$idx", 963 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 964 def : InstAlias<"mov $Zd, $Zn$idx", 965 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 966 def : InstAlias<"mov $Zd, $Zn$idx", 967 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 968 def : InstAlias<"mov $Zd, $Zn$idx", 969 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 970 def : InstAlias<"mov $Zd, $Bn", 971 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 972 def : InstAlias<"mov $Zd, $Hn", 973 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 974 def : InstAlias<"mov $Zd, $Sn", 975 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 976 def : InstAlias<"mov $Zd, $Dn", 977 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 978 def : InstAlias<"mov $Zd, $Qn", 979 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 980} 981 982class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, 983 ZPRRegOp zprty, RegisterOperand VecList> 984: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 985 asm, "\t$Zd, $Zn, $Zm", 986 "", 987 []>, Sched<[]> { 988 bits<5> Zd; 989 bits<5> Zm; 990 bits<5> Zn; 991 let Inst{31-24} = 0b00000101; 992 let Inst{23-22} = sz8_64; 993 let Inst{21} = 0b1; 994 let Inst{20-16} = Zm; 995 let Inst{15-13} = 0b001; 996 let Inst{12-11} = opc; 997 let Inst{10} = 0b0; 998 let Inst{9-5} = Zn; 999 let Inst{4-0} = Zd; 1000} 1001 1002multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1003 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1004 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1005 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1006 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1007 1008 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1009 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1010 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1011 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1012 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1013 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1014 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1015 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1016 1017 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1018 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1019 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1020 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1021 1022 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1023 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1024 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1025} 1026 1027multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1028 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1029 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1030 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1031 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1032 1033 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1034 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1035 nxv16i8:$Op2, zsub1), 1036 nxv16i8:$Op3))>; 1037 1038 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1039 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1040 nxv8i16:$Op2, zsub1), 1041 nxv8i16:$Op3))>; 1042 1043 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1044 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1045 nxv4i32:$Op2, zsub1), 1046 nxv4i32:$Op3))>; 1047 1048 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1049 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1050 nxv2i64:$Op2, zsub1), 1051 nxv2i64:$Op3))>; 1052 1053 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1054 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1055 nxv8f16:$Op2, zsub1), 1056 nxv8i16:$Op3))>; 1057 1058 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1059 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1060 nxv4f32:$Op2, zsub1), 1061 nxv4i32:$Op3))>; 1062 1063 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1064 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1065 nxv2f64:$Op2, zsub1), 1066 nxv2i64:$Op3))>; 1067} 1068 1069class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1070: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1071 asm, "\t$Zd, $Zn, $Zm", 1072 "", 1073 []>, Sched<[]> { 1074 bits<5> Zd; 1075 bits<5> Zm; 1076 bits<5> Zn; 1077 let Inst{31-24} = 0b00000101; 1078 let Inst{23-22} = sz8_64; 1079 let Inst{21} = 0b1; 1080 let Inst{20-16} = Zm; 1081 let Inst{15-10} = 0b001011; 1082 let Inst{9-5} = Zn; 1083 let Inst{4-0} = Zd; 1084 1085 let Constraints = "$Zd = $_Zd"; 1086} 1087 1088multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1089 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1090 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1091 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1092 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1093 1094 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1095 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1096 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1097 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1098 1099 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1100 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1101 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1102} 1103 1104class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1105: I<(outs zprty:$Zd), (ins zprty:$Zn), 1106 asm, "\t$Zd, $Zn", 1107 "", 1108 []>, Sched<[]> { 1109 bits<5> Zd; 1110 bits<5> Zn; 1111 let Inst{31-24} = 0b00000101; 1112 let Inst{23-22} = sz8_64; 1113 let Inst{21-10} = 0b111000001110; 1114 let Inst{9-5} = Zn; 1115 let Inst{4-0} = Zd; 1116} 1117 1118multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1119 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1120 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1121 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1122 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1123 1124 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1125 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1126 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1127 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1128 1129 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1130 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1131 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1132} 1133 1134class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1135: I<(outs pprty:$Pd), (ins pprty:$Pn), 1136 asm, "\t$Pd, $Pn", 1137 "", 1138 []>, Sched<[]> { 1139 bits<4> Pd; 1140 bits<4> Pn; 1141 let Inst{31-24} = 0b00000101; 1142 let Inst{23-22} = sz8_64; 1143 let Inst{21-9} = 0b1101000100000; 1144 let Inst{8-5} = Pn; 1145 let Inst{4} = 0b0; 1146 let Inst{3-0} = Pd; 1147} 1148 1149multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1150 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1151 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1152 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1153 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1154 1155 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1156 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1157 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1158 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1159} 1160 1161class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1162 ZPRRegOp zprty1, ZPRRegOp zprty2> 1163: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1164 asm, "\t$Zd, $Zn", 1165 "", []>, Sched<[]> { 1166 bits<5> Zd; 1167 bits<5> Zn; 1168 let Inst{31-24} = 0b00000101; 1169 let Inst{23-22} = sz16_64; 1170 let Inst{21-18} = 0b1100; 1171 let Inst{17-16} = opc; 1172 let Inst{15-10} = 0b001110; 1173 let Inst{9-5} = Zn; 1174 let Inst{4-0} = Zd; 1175} 1176 1177multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1178 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1179 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1180 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1181 1182 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1183 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1184 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1185} 1186 1187class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1188 RegisterClass srcRegType> 1189: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1190 asm, "\t$Zdn, $Rm", 1191 "", 1192 []>, Sched<[]> { 1193 bits<5> Rm; 1194 bits<5> Zdn; 1195 let Inst{31-24} = 0b00000101; 1196 let Inst{23-22} = sz8_64; 1197 let Inst{21-10} = 0b100100001110; 1198 let Inst{9-5} = Rm; 1199 let Inst{4-0} = Zdn; 1200 1201 let Constraints = "$Zdn = $_Zdn"; 1202 let DestructiveInstType = DestructiveOther; 1203} 1204 1205multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1206 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1207 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1208 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1209 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1210 1211 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1212 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1213 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1214 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1215} 1216 1217class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1218 RegisterClass srcRegType> 1219: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Vm), 1220 asm, "\t$Zdn, $Vm", 1221 "", 1222 []>, Sched<[]> { 1223 bits<5> Vm; 1224 bits<5> Zdn; 1225 let Inst{31-24} = 0b00000101; 1226 let Inst{23-22} = sz8_64; 1227 let Inst{21-10} = 0b110100001110; 1228 let Inst{9-5} = Vm; 1229 let Inst{4-0} = Zdn; 1230 1231 let Constraints = "$Zdn = $_Zdn"; 1232 let DestructiveInstType = DestructiveOther; 1233} 1234 1235multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1236 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8>; 1237 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16>; 1238 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32>; 1239 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64>; 1240 1241 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, f16, !cast<Instruction>(NAME # _H)>; 1242 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, f32, !cast<Instruction>(NAME # _S)>; 1243 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, f64, !cast<Instruction>(NAME # _D)>; 1244} 1245 1246//===----------------------------------------------------------------------===// 1247// SVE Permute - Extract Group 1248//===----------------------------------------------------------------------===// 1249 1250class sve_int_perm_extract_i<string asm> 1251: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1252 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1253 "", []>, Sched<[]> { 1254 bits<5> Zdn; 1255 bits<5> Zm; 1256 bits<8> imm8; 1257 let Inst{31-21} = 0b00000101001; 1258 let Inst{20-16} = imm8{7-3}; 1259 let Inst{15-13} = 0b000; 1260 let Inst{12-10} = imm8{2-0}; 1261 let Inst{9-5} = Zm; 1262 let Inst{4-0} = Zdn; 1263 1264 let Constraints = "$Zdn = $_Zdn"; 1265 let DestructiveInstType = DestructiveOther; 1266 let ElementSize = ElementSizeNone; 1267} 1268 1269multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1270 def NAME : sve_int_perm_extract_i<asm>; 1271 1272 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1273 !cast<Instruction>(NAME)>; 1274} 1275 1276class sve2_int_perm_extract_i_cons<string asm> 1277: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1278 asm, "\t$Zd, $Zn, $imm8", 1279 "", []>, Sched<[]> { 1280 bits<5> Zd; 1281 bits<5> Zn; 1282 bits<8> imm8; 1283 let Inst{31-21} = 0b00000101011; 1284 let Inst{20-16} = imm8{7-3}; 1285 let Inst{15-13} = 0b000; 1286 let Inst{12-10} = imm8{2-0}; 1287 let Inst{9-5} = Zn; 1288 let Inst{4-0} = Zd; 1289} 1290 1291//===----------------------------------------------------------------------===// 1292// SVE Vector Select Group 1293//===----------------------------------------------------------------------===// 1294 1295class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1296: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1297 asm, "\t$Zd, $Pg, $Zn, $Zm", 1298 "", 1299 []>, Sched<[]> { 1300 bits<4> Pg; 1301 bits<5> Zd; 1302 bits<5> Zm; 1303 bits<5> Zn; 1304 let Inst{31-24} = 0b00000101; 1305 let Inst{23-22} = sz8_64; 1306 let Inst{21} = 0b1; 1307 let Inst{20-16} = Zm; 1308 let Inst{15-14} = 0b11; 1309 let Inst{13-10} = Pg; 1310 let Inst{9-5} = Zn; 1311 let Inst{4-0} = Zd; 1312} 1313 1314multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1315 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1316 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1317 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1318 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1319 1320 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1321 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1322 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1323 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1324 1325 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1326 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1327 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1328 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1329 1330 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1331 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1332 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1333 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1334 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1335 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1336 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1337 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1338} 1339 1340 1341//===----------------------------------------------------------------------===// 1342// SVE Predicate Logical Operations Group 1343//===----------------------------------------------------------------------===// 1344 1345class sve_int_pred_log<bits<4> opc, string asm> 1346: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1347 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1348 "", 1349 []>, Sched<[]> { 1350 bits<4> Pd; 1351 bits<4> Pg; 1352 bits<4> Pm; 1353 bits<4> Pn; 1354 let Inst{31-24} = 0b00100101; 1355 let Inst{23-22} = opc{3-2}; 1356 let Inst{21-20} = 0b00; 1357 let Inst{19-16} = Pm; 1358 let Inst{15-14} = 0b01; 1359 let Inst{13-10} = Pg; 1360 let Inst{9} = opc{1}; 1361 let Inst{8-5} = Pn; 1362 let Inst{4} = opc{0}; 1363 let Inst{3-0} = Pd; 1364 1365 // SEL has no predication qualifier. 1366 let AsmString = !if(!eq(opc, 0b0011), 1367 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1368 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1369 1370 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1371 1372} 1373 1374multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1375 SDPatternOperator op_nopred = null_frag> { 1376 def NAME : sve_int_pred_log<opc, asm>; 1377 1378 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1379 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1380 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1381 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1382 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1383 !cast<Instruction>(NAME), PTRUE_B>; 1384 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1385 !cast<Instruction>(NAME), PTRUE_H>; 1386 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1387 !cast<Instruction>(NAME), PTRUE_S>; 1388 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1389 !cast<Instruction>(NAME), PTRUE_D>; 1390} 1391 1392 1393//===----------------------------------------------------------------------===// 1394// SVE Logical Mask Immediate Group 1395//===----------------------------------------------------------------------===// 1396 1397class sve_int_log_imm<bits<2> opc, string asm> 1398: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1399 asm, "\t$Zdn, $_Zdn, $imms13", 1400 "", []>, Sched<[]> { 1401 bits<5> Zdn; 1402 bits<13> imms13; 1403 let Inst{31-24} = 0b00000101; 1404 let Inst{23-22} = opc; 1405 let Inst{21-18} = 0b0000; 1406 let Inst{17-5} = imms13; 1407 let Inst{4-0} = Zdn; 1408 1409 let Constraints = "$Zdn = $_Zdn"; 1410 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1411 let DestructiveInstType = DestructiveOther; 1412 let ElementSize = ElementSizeNone; 1413} 1414 1415multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1416 def NAME : sve_int_log_imm<opc, asm>; 1417 1418 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1419 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1420 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1421 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1422 1423 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1424 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1425 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1426 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1427 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1428 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1429 1430 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1431 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1432 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1433 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1434 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1435 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1436 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1437 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1438} 1439 1440class sve_int_dup_mask_imm<string asm> 1441: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1442 asm, "\t$Zd, $imms", 1443 "", 1444 []>, Sched<[]> { 1445 bits<5> Zd; 1446 bits<13> imms; 1447 let Inst{31-18} = 0b00000101110000; 1448 let Inst{17-5} = imms; 1449 let Inst{4-0} = Zd; 1450 1451 let isReMaterializable = 1; 1452 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1453} 1454 1455multiclass sve_int_dup_mask_imm<string asm> { 1456 def NAME : sve_int_dup_mask_imm<asm>; 1457 1458 def : InstAlias<"dupm $Zd, $imm", 1459 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1460 def : InstAlias<"dupm $Zd, $imm", 1461 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1462 def : InstAlias<"dupm $Zd, $imm", 1463 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1464 1465 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1466 def : InstAlias<"mov $Zd, $imm", 1467 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1468 def : InstAlias<"mov $Zd, $imm", 1469 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1470 def : InstAlias<"mov $Zd, $imm", 1471 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1472} 1473 1474//===----------------------------------------------------------------------===// 1475// SVE Integer Arithmetic - Unpredicated Group. 1476//===----------------------------------------------------------------------===// 1477 1478class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1479 ZPRRegOp zprty> 1480: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1481 asm, "\t$Zd, $Zn, $Zm", 1482 "", []>, Sched<[]> { 1483 bits<5> Zd; 1484 bits<5> Zm; 1485 bits<5> Zn; 1486 let Inst{31-24} = 0b00000100; 1487 let Inst{23-22} = sz8_64; 1488 let Inst{21} = 0b1; 1489 let Inst{20-16} = Zm; 1490 let Inst{15-13} = 0b000; 1491 let Inst{12-10} = opc; 1492 let Inst{9-5} = Zn; 1493 let Inst{4-0} = Zd; 1494} 1495 1496multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, 1497 SDPatternOperator op, SDPatternOperator int_op> { 1498 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1499 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1500 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1501 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1502 1503 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1504 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1505 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1506 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1507 1508 // Intrinsic version 1509 def : SVE_2_Op_Pat<nxv16i8, int_op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1510 def : SVE_2_Op_Pat<nxv8i16, int_op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1511 def : SVE_2_Op_Pat<nxv4i32, int_op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1512 def : SVE_2_Op_Pat<nxv2i64, int_op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1513} 1514 1515//===----------------------------------------------------------------------===// 1516// SVE Floating Point Arithmetic - Predicated Group 1517//===----------------------------------------------------------------------===// 1518 1519class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1520 ZPRRegOp zprty, 1521 Operand imm_ty> 1522: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1523 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1524 "", 1525 []>, Sched<[]> { 1526 bits<3> Pg; 1527 bits<5> Zdn; 1528 bit i1; 1529 let Inst{31-24} = 0b01100101; 1530 let Inst{23-22} = sz; 1531 let Inst{21-19} = 0b011; 1532 let Inst{18-16} = opc; 1533 let Inst{15-13} = 0b100; 1534 let Inst{12-10} = Pg; 1535 let Inst{9-6} = 0b0000; 1536 let Inst{5} = i1; 1537 let Inst{4-0} = Zdn; 1538 1539 let Constraints = "$Zdn = $_Zdn"; 1540 let DestructiveInstType = DestructiveOther; 1541 let ElementSize = zprty.ElementSize; 1542} 1543 1544multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, Operand imm_ty> { 1545 def _H : sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1546 def _S : sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1547 def _D : sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1548} 1549 1550class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1551 ZPRRegOp zprty> 1552: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1553 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1554 "", 1555 []>, Sched<[]> { 1556 bits<3> Pg; 1557 bits<5> Zdn; 1558 bits<5> Zm; 1559 let Inst{31-24} = 0b01100101; 1560 let Inst{23-22} = sz; 1561 let Inst{21-20} = 0b00; 1562 let Inst{19-16} = opc; 1563 let Inst{15-13} = 0b100; 1564 let Inst{12-10} = Pg; 1565 let Inst{9-5} = Zm; 1566 let Inst{4-0} = Zdn; 1567 1568 let Constraints = "$Zdn = $_Zdn"; 1569 let DestructiveInstType = DestructiveOther; 1570 let ElementSize = zprty.ElementSize; 1571} 1572 1573multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1574 SDPatternOperator op, DestructiveInstTypeEnum flags, 1575 string revname="", bit isReverseInstr=0> { 1576 let DestructiveInstType = flags in { 1577 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1578 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1579 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1580 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1581 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1582 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1583 } 1584 1585 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1586 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1587 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1588} 1589 1590multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1591 SDPatternOperator op> { 1592 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1593 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1594 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1595 1596 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1597 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1598 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1599} 1600 1601multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1602 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1603 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1604 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1605 1606 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1607 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1608 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1609} 1610 1611class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1612: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, imm32_0_7:$imm3), 1613 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1614 "", 1615 []>, Sched<[]> { 1616 bits<5> Zdn; 1617 bits<5> Zm; 1618 bits<3> imm3; 1619 let Inst{31-24} = 0b01100101; 1620 let Inst{23-22} = sz; 1621 let Inst{21-19} = 0b010; 1622 let Inst{18-16} = imm3; 1623 let Inst{15-10} = 0b100000; 1624 let Inst{9-5} = Zm; 1625 let Inst{4-0} = Zdn; 1626 1627 let Constraints = "$Zdn = $_Zdn"; 1628 let DestructiveInstType = DestructiveOther; 1629 let ElementSize = ElementSizeNone; 1630} 1631 1632multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1633 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1634 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1635 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1636 1637 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 imm32_0_7:$imm))), 1638 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, imm32_0_7:$imm)>; 1639 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 imm32_0_7:$imm))), 1640 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, imm32_0_7:$imm)>; 1641 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 imm32_0_7:$imm))), 1642 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, imm32_0_7:$imm)>; 1643} 1644 1645 1646//===----------------------------------------------------------------------===// 1647// SVE Floating Point Arithmetic - Unpredicated Group 1648//===----------------------------------------------------------------------===// 1649 1650class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 1651: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1652 asm, "\t$Zd, $Zn, $Zm", 1653 "", 1654 []>, Sched<[]> { 1655 bits<5> Zd; 1656 bits<5> Zm; 1657 bits<5> Zn; 1658 let Inst{31-24} = 0b01100101; 1659 let Inst{23-22} = sz; 1660 let Inst{21} = 0b0; 1661 let Inst{20-16} = Zm; 1662 let Inst{15-13} = 0b000; 1663 let Inst{12-10} = opc; 1664 let Inst{9-5} = Zn; 1665 let Inst{4-0} = Zd; 1666} 1667 1668multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 1669 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1670 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1671 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1672 1673 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1674 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1675 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1676 1677} 1678 1679multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 1680 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1681 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1682 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1683 1684 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1685 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1686 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1687} 1688 1689//===----------------------------------------------------------------------===// 1690// SVE Floating Point Fused Multiply-Add Group 1691//===----------------------------------------------------------------------===// 1692 1693class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 1694: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 1695 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 1696 "", 1697 []>, Sched<[]> { 1698 bits<3> Pg; 1699 bits<5> Zda; 1700 bits<5> Zm; 1701 bits<5> Zn; 1702 let Inst{31-24} = 0b01100101; 1703 let Inst{23-22} = sz; 1704 let Inst{21} = 0b1; 1705 let Inst{20-16} = Zm; 1706 let Inst{15} = 0b0; 1707 let Inst{14-13} = opc; 1708 let Inst{12-10} = Pg; 1709 let Inst{9-5} = Zn; 1710 let Inst{4-0} = Zda; 1711 1712 let Constraints = "$Zda = $_Zda"; 1713 let DestructiveInstType = DestructiveOther; 1714 let ElementSize = zprty.ElementSize; 1715} 1716 1717multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, SDPatternOperator op> { 1718 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>; 1719 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>; 1720 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>; 1721 1722 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1723 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1724 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1725} 1726 1727class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 1728 ZPRRegOp zprty> 1729: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 1730 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 1731 "", 1732 []>, Sched<[]> { 1733 bits<3> Pg; 1734 bits<5> Za; 1735 bits<5> Zdn; 1736 bits<5> Zm; 1737 let Inst{31-24} = 0b01100101; 1738 let Inst{23-22} = sz; 1739 let Inst{21} = 0b1; 1740 let Inst{20-16} = Za; 1741 let Inst{15} = 0b1; 1742 let Inst{14-13} = opc; 1743 let Inst{12-10} = Pg; 1744 let Inst{9-5} = Zm; 1745 let Inst{4-0} = Zdn; 1746 1747 let Constraints = "$Zdn = $_Zdn"; 1748 let DestructiveInstType = DestructiveOther; 1749 let ElementSize = zprty.ElementSize; 1750} 1751 1752multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op> { 1753 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>; 1754 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>; 1755 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>; 1756 1757 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1758 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1759 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1760} 1761 1762//===----------------------------------------------------------------------===// 1763// SVE Floating Point Multiply-Add - Indexed Group 1764//===----------------------------------------------------------------------===// 1765 1766class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 1767 ZPRRegOp zprty1, 1768 ZPRRegOp zprty2, Operand itype> 1769: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 1770 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 1771 bits<5> Zda; 1772 bits<5> Zn; 1773 let Inst{31-24} = 0b01100100; 1774 let Inst{23-22} = sz; 1775 let Inst{21} = 0b1; 1776 let Inst{15-11} = 0; 1777 let Inst{10} = opc; 1778 let Inst{9-5} = Zn; 1779 let Inst{4-0} = Zda; 1780 1781 let Constraints = "$Zda = $_Zda"; 1782 let DestructiveInstType = DestructiveOther; 1783 let ElementSize = ElementSizeNone; 1784} 1785 1786multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 1787 SDPatternOperator op> { 1788 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 1789 bits<3> Zm; 1790 bits<3> iop; 1791 let Inst{22} = iop{2}; 1792 let Inst{20-19} = iop{1-0}; 1793 let Inst{18-16} = Zm; 1794 } 1795 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 1796 bits<3> Zm; 1797 bits<2> iop; 1798 let Inst{20-19} = iop; 1799 let Inst{18-16} = Zm; 1800 } 1801 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 1802 bits<4> Zm; 1803 bit iop; 1804 let Inst{20} = iop; 1805 let Inst{19-16} = Zm; 1806 } 1807 1808 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 1809 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 1810 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 1811 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 1812 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 1813 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 1814} 1815 1816 1817//===----------------------------------------------------------------------===// 1818// SVE Floating Point Multiply - Indexed Group 1819//===----------------------------------------------------------------------===// 1820 1821class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 1822 ZPRRegOp zprty2, Operand itype> 1823: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 1824 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 1825 bits<5> Zd; 1826 bits<5> Zn; 1827 let Inst{31-24} = 0b01100100; 1828 let Inst{23-22} = sz; 1829 let Inst{21} = 0b1; 1830 let Inst{15-10} = 0b001000; 1831 let Inst{9-5} = Zn; 1832 let Inst{4-0} = Zd; 1833} 1834 1835multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 1836 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 1837 bits<3> Zm; 1838 bits<3> iop; 1839 let Inst{22} = iop{2}; 1840 let Inst{20-19} = iop{1-0}; 1841 let Inst{18-16} = Zm; 1842 } 1843 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 1844 bits<3> Zm; 1845 bits<2> iop; 1846 let Inst{20-19} = iop; 1847 let Inst{18-16} = Zm; 1848 } 1849 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 1850 bits<4> Zm; 1851 bit iop; 1852 let Inst{20} = iop; 1853 let Inst{19-16} = Zm; 1854 } 1855 1856 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 1857 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 1858 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 1859 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 1860 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 1861 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 1862} 1863 1864//===----------------------------------------------------------------------===// 1865// SVE Floating Point Complex Multiply-Add Group 1866//===----------------------------------------------------------------------===// 1867 1868class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 1869: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 1870 complexrotateop:$imm), 1871 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 1872 "", []>, Sched<[]> { 1873 bits<5> Zda; 1874 bits<3> Pg; 1875 bits<5> Zn; 1876 bits<5> Zm; 1877 bits<2> imm; 1878 let Inst{31-24} = 0b01100100; 1879 let Inst{23-22} = sz; 1880 let Inst{21} = 0; 1881 let Inst{20-16} = Zm; 1882 let Inst{15} = 0; 1883 let Inst{14-13} = imm; 1884 let Inst{12-10} = Pg; 1885 let Inst{9-5} = Zn; 1886 let Inst{4-0} = Zda; 1887 1888 let Constraints = "$Zda = $_Zda"; 1889 let DestructiveInstType = DestructiveOther; 1890 let ElementSize = zprty.ElementSize; 1891} 1892 1893multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 1894 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 1895 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 1896 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 1897 1898 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 1899 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1900 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 1901 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1902 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 1903 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1904} 1905 1906//===----------------------------------------------------------------------===// 1907// SVE Floating Point Complex Multiply-Add - Indexed Group 1908//===----------------------------------------------------------------------===// 1909 1910class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 1911 ZPRRegOp zprty, 1912 ZPRRegOp zprty2, Operand itype> 1913: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 1914 complexrotateop:$imm), 1915 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 1916 "", []>, Sched<[]> { 1917 bits<5> Zda; 1918 bits<5> Zn; 1919 bits<2> imm; 1920 let Inst{31-24} = 0b01100100; 1921 let Inst{23-22} = sz; 1922 let Inst{21} = 0b1; 1923 let Inst{15-12} = 0b0001; 1924 let Inst{11-10} = imm; 1925 let Inst{9-5} = Zn; 1926 let Inst{4-0} = Zda; 1927 1928 let Constraints = "$Zda = $_Zda"; 1929 let DestructiveInstType = DestructiveOther; 1930 let ElementSize = ElementSizeNone; 1931} 1932 1933multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 1934 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 1935 bits<3> Zm; 1936 bits<2> iop; 1937 let Inst{20-19} = iop; 1938 let Inst{18-16} = Zm; 1939 } 1940 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 1941 bits<4> Zm; 1942 bits<1> iop; 1943 let Inst{20} = iop; 1944 let Inst{19-16} = Zm; 1945 } 1946 1947 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 1948 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 1949 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 1950 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 1951} 1952 1953//===----------------------------------------------------------------------===// 1954// SVE Floating Point Complex Addition Group 1955//===----------------------------------------------------------------------===// 1956 1957class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 1958: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 1959 complexrotateopodd:$imm), 1960 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 1961 "", 1962 []>, Sched<[]> { 1963 bits<5> Zdn; 1964 bits<5> Zm; 1965 bits<3> Pg; 1966 bit imm; 1967 let Inst{31-24} = 0b01100100; 1968 let Inst{23-22} = sz; 1969 let Inst{21-17} = 0; 1970 let Inst{16} = imm; 1971 let Inst{15-13} = 0b100; 1972 let Inst{12-10} = Pg; 1973 let Inst{9-5} = Zm; 1974 let Inst{4-0} = Zdn; 1975 1976 let Constraints = "$Zdn = $_Zdn"; 1977 let DestructiveInstType = DestructiveOther; 1978 let ElementSize = zprty.ElementSize; 1979} 1980 1981multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 1982 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 1983 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 1984 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 1985 1986 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 1987 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 1988 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 1989 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 1990 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 1991 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 1992} 1993 1994//===----------------------------------------------------------------------===// 1995// SVE2 Floating Point Convert Group 1996//===----------------------------------------------------------------------===// 1997 1998class sve2_fp_convert_precision<bits<4> opc, string asm, 1999 ZPRRegOp zprty1, ZPRRegOp zprty2> 2000: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2001 asm, "\t$Zd, $Pg/m, $Zn", 2002 "", 2003 []>, Sched<[]> { 2004 bits<5> Zd; 2005 bits<5> Zn; 2006 bits<3> Pg; 2007 let Inst{31-24} = 0b01100100; 2008 let Inst{23-22} = opc{3-2}; 2009 let Inst{21-18} = 0b0010; 2010 let Inst{17-16} = opc{1-0}; 2011 let Inst{15-13} = 0b101; 2012 let Inst{12-10} = Pg; 2013 let Inst{9-5} = Zn; 2014 let Inst{4-0} = Zd; 2015 2016 let Constraints = "$Zd = $_Zd"; 2017} 2018 2019multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2020 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2021 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2022 2023 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2024 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2025} 2026 2027multiclass sve2_fp_convert_up_long<string asm, string op> { 2028 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2029 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2030 2031 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2032 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2033} 2034 2035multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2036 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2037 2038 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2039} 2040 2041//===----------------------------------------------------------------------===// 2042// SVE2 Floating Point Pairwise Group 2043//===----------------------------------------------------------------------===// 2044 2045class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2046 ZPRRegOp zprty> 2047: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2048 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2049 "", 2050 []>, Sched<[]> { 2051 bits<3> Pg; 2052 bits<5> Zm; 2053 bits<5> Zdn; 2054 let Inst{31-24} = 0b01100100; 2055 let Inst{23-22} = sz; 2056 let Inst{21-19} = 0b010; 2057 let Inst{18-16} = opc; 2058 let Inst{15-13} = 0b100; 2059 let Inst{12-10} = Pg; 2060 let Inst{9-5} = Zm; 2061 let Inst{4-0} = Zdn; 2062 2063 let Constraints = "$Zdn = $_Zdn"; 2064 let DestructiveInstType = DestructiveOther; 2065 let ElementSize = zprty.ElementSize; 2066} 2067 2068multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, SDPatternOperator op> { 2069 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2070 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2071 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2072 2073 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2074 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2075 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2076} 2077 2078//===----------------------------------------------------------------------===// 2079// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2080//===----------------------------------------------------------------------===// 2081 2082class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2083: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2084 VectorIndexH32b:$iop), 2085 asm, "\t$Zda, $Zn, $Zm$iop", 2086 "", 2087 []>, Sched<[]> { 2088 bits<5> Zda; 2089 bits<5> Zn; 2090 bits<3> Zm; 2091 bits<3> iop; 2092 let Inst{31-21} = 0b01100100101; 2093 let Inst{20-19} = iop{2-1}; 2094 let Inst{18-16} = Zm; 2095 let Inst{15-14} = 0b01; 2096 let Inst{13} = opc{1}; 2097 let Inst{12} = 0b0; 2098 let Inst{11} = iop{0}; 2099 let Inst{10} = opc{0}; 2100 let Inst{9-5} = Zn; 2101 let Inst{4-0} = Zda; 2102 2103 let Constraints = "$Zda = $_Zda"; 2104 let DestructiveInstType = DestructiveOther; 2105 let ElementSize = ElementSizeNone; 2106} 2107 2108multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2109 SDPatternOperator op> { 2110 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2111 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2112} 2113 2114//===----------------------------------------------------------------------===// 2115// SVE2 Floating Point Widening Multiply-Add Group 2116//===----------------------------------------------------------------------===// 2117 2118class sve2_fp_mla_long<bits<2> opc, string asm> 2119: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2120 asm, "\t$Zda, $Zn, $Zm", 2121 "", 2122 []>, Sched<[]> { 2123 bits<5> Zda; 2124 bits<5> Zn; 2125 bits<5> Zm; 2126 let Inst{31-21} = 0b01100100101; 2127 let Inst{20-16} = Zm; 2128 let Inst{15-14} = 0b10; 2129 let Inst{13} = opc{1}; 2130 let Inst{12-11} = 0b00; 2131 let Inst{10} = opc{0}; 2132 let Inst{9-5} = Zn; 2133 let Inst{4-0} = Zda; 2134 2135 let Constraints = "$Zda = $_Zda"; 2136 let DestructiveInstType = DestructiveOther; 2137 let ElementSize = ElementSizeNone; 2138} 2139 2140multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2141 def NAME : sve2_fp_mla_long<opc, asm>; 2142 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2143} 2144 2145//===----------------------------------------------------------------------===// 2146// SVE Stack Allocation Group 2147//===----------------------------------------------------------------------===// 2148 2149class sve_int_arith_vl<bit opc, string asm> 2150: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2151 asm, "\t$Rd, $Rn, $imm6", 2152 "", 2153 []>, Sched<[]> { 2154 bits<5> Rd; 2155 bits<5> Rn; 2156 bits<6> imm6; 2157 let Inst{31-23} = 0b000001000; 2158 let Inst{22} = opc; 2159 let Inst{21} = 0b1; 2160 let Inst{20-16} = Rn; 2161 let Inst{15-11} = 0b01010; 2162 let Inst{10-5} = imm6; 2163 let Inst{4-0} = Rd; 2164} 2165 2166class sve_int_read_vl_a<bit op, bits<5> opc2, string asm> 2167: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2168 asm, "\t$Rd, $imm6", 2169 "", 2170 []>, Sched<[]> { 2171 bits<5> Rd; 2172 bits<6> imm6; 2173 let Inst{31-23} = 0b000001001; 2174 let Inst{22} = op; 2175 let Inst{21} = 0b1; 2176 let Inst{20-16} = opc2{4-0}; 2177 let Inst{15-11} = 0b01010; 2178 let Inst{10-5} = imm6; 2179 let Inst{4-0} = Rd; 2180} 2181 2182//===----------------------------------------------------------------------===// 2183// SVE Permute - In Lane Group 2184//===----------------------------------------------------------------------===// 2185 2186class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2187 ZPRRegOp zprty> 2188: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2189 asm, "\t$Zd, $Zn, $Zm", 2190 "", 2191 []>, Sched<[]> { 2192 bits<5> Zd; 2193 bits<5> Zm; 2194 bits<5> Zn; 2195 let Inst{31-24} = 0b00000101; 2196 let Inst{23-22} = sz8_64; 2197 let Inst{21} = 0b1; 2198 let Inst{20-16} = Zm; 2199 let Inst{15-13} = 0b011; 2200 let Inst{12-10} = opc; 2201 let Inst{9-5} = Zn; 2202 let Inst{4-0} = Zd; 2203} 2204 2205multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2206 SDPatternOperator op> { 2207 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2208 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2209 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2210 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2211 2212 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2213 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2214 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2215 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2216 2217 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2218 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2219 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2220 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2221} 2222 2223//===----------------------------------------------------------------------===// 2224// SVE Floating Point Unary Operations Group 2225//===----------------------------------------------------------------------===// 2226 2227class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2228 RegisterOperand o_zprtype, ElementSizeEnum size> 2229: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2230 asm, "\t$Zd, $Pg/m, $Zn", 2231 "", 2232 []>, Sched<[]> { 2233 bits<3> Pg; 2234 bits<5> Zd; 2235 bits<5> Zn; 2236 let Inst{31-24} = 0b01100101; 2237 let Inst{23-22} = opc{6-5}; 2238 let Inst{21} = 0b0; 2239 let Inst{20-16} = opc{4-0}; 2240 let Inst{15-13} = 0b101; 2241 let Inst{12-10} = Pg; 2242 let Inst{9-5} = Zn; 2243 let Inst{4-0} = Zd; 2244 2245 let Constraints = "$Zd = $_Zd"; 2246 let DestructiveInstType = DestructiveOther; 2247 let ElementSize = size; 2248} 2249 2250multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2251 RegisterOperand i_zprtype, 2252 RegisterOperand o_zprtype, 2253 SDPatternOperator op, ValueType vt1, 2254 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2255 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2256 2257 def : SVE_3_Op_Pat<vt1, op, vt1, vt2, vt3, !cast<Instruction>(NAME)>; 2258} 2259 2260multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2261 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>; 2262 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>; 2263 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>; 2264 2265 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2266 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2267 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2268} 2269 2270multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2271 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2272 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2273 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2274 2275 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2276 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2277 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2278} 2279 2280multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2281 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2282 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2283} 2284 2285//===----------------------------------------------------------------------===// 2286// SVE Floating Point Unary Operations - Unpredicated Group 2287//===----------------------------------------------------------------------===// 2288 2289class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2290 ZPRRegOp zprty> 2291: I<(outs zprty:$Zd), (ins zprty:$Zn), 2292 asm, "\t$Zd, $Zn", 2293 "", 2294 []>, Sched<[]> { 2295 bits<5> Zd; 2296 bits<5> Zn; 2297 let Inst{31-24} = 0b01100101; 2298 let Inst{23-22} = sz; 2299 let Inst{21-19} = 0b001; 2300 let Inst{18-16} = opc; 2301 let Inst{15-10} = 0b001100; 2302 let Inst{9-5} = Zn; 2303 let Inst{4-0} = Zd; 2304} 2305 2306multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2307 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2308 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2309 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2310 2311 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2312 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2313 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2314} 2315 2316//===----------------------------------------------------------------------===// 2317// SVE Integer Arithmetic - Binary Predicated Group 2318//===----------------------------------------------------------------------===// 2319 2320class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2321 string asm, ZPRRegOp zprty> 2322: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2323 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2324 bits<3> Pg; 2325 bits<5> Zdn; 2326 bits<5> Zm; 2327 let Inst{31-24} = 0b00000100; 2328 let Inst{23-22} = sz8_64; 2329 let Inst{21} = 0b0; 2330 let Inst{20-19} = fmt; 2331 let Inst{18-16} = opc; 2332 let Inst{15-13} = 0b000; 2333 let Inst{12-10} = Pg; 2334 let Inst{9-5} = Zm; 2335 let Inst{4-0} = Zdn; 2336 2337 let Constraints = "$Zdn = $_Zdn"; 2338 let DestructiveInstType = DestructiveOther; 2339 let ElementSize = zprty.ElementSize; 2340} 2341 2342multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> { 2343 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>; 2344 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>; 2345 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>; 2346 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>; 2347 2348 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2349 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2350 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2351 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2352} 2353 2354multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2355 SDPatternOperator op, 2356 DestructiveInstTypeEnum flags, 2357 string revname="", bit isReverseInstr=0> { 2358 let DestructiveInstType = flags in { 2359 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2360 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2361 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2362 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2363 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2364 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2365 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2366 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2367 } 2368 2369 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2370 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2371 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2372 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2373} 2374 2375multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, SDPatternOperator op> { 2376 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>; 2377 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>; 2378 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>; 2379 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>; 2380 2381 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2382 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2383 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2384 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2385} 2386 2387multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, SDPatternOperator op> { 2388 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>; 2389 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>; 2390 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>; 2391 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>; 2392 2393 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2394 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2395 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2396 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2397} 2398 2399// Special case for divides which are not defined for 8b/16b elements. 2400multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2401 SDPatternOperator op, 2402 DestructiveInstTypeEnum flags, 2403 string revname="", bit isReverseInstr=0> { 2404 let DestructiveInstType = flags in { 2405 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2406 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2407 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2408 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2409 } 2410 2411 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2412 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2413} 2414 2415//===----------------------------------------------------------------------===// 2416// SVE Integer Multiply-Add Group 2417//===----------------------------------------------------------------------===// 2418 2419class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2420 ZPRRegOp zprty> 2421: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2422 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2423 "", 2424 []>, Sched<[]> { 2425 bits<3> Pg; 2426 bits<5> Zdn; 2427 bits<5> Za; 2428 bits<5> Zm; 2429 let Inst{31-24} = 0b00000100; 2430 let Inst{23-22} = sz8_64; 2431 let Inst{21} = 0b0; 2432 let Inst{20-16} = Zm; 2433 let Inst{15-14} = 0b11; 2434 let Inst{13} = opc; 2435 let Inst{12-10} = Pg; 2436 let Inst{9-5} = Za; 2437 let Inst{4-0} = Zdn; 2438 2439 let Constraints = "$Zdn = $_Zdn"; 2440 let DestructiveInstType = DestructiveOther; 2441 let ElementSize = zprty.ElementSize; 2442} 2443 2444multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2445 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2446 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2447 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2448 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2449 2450 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2451 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2452 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2453 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2454} 2455 2456class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2457 ZPRRegOp zprty> 2458: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2459 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2460 "", 2461 []>, Sched<[]> { 2462 bits<3> Pg; 2463 bits<5> Zda; 2464 bits<5> Zm; 2465 bits<5> Zn; 2466 let Inst{31-24} = 0b00000100; 2467 let Inst{23-22} = sz8_64; 2468 let Inst{21} = 0b0; 2469 let Inst{20-16} = Zm; 2470 let Inst{15-14} = 0b01; 2471 let Inst{13} = opc; 2472 let Inst{12-10} = Pg; 2473 let Inst{9-5} = Zn; 2474 let Inst{4-0} = Zda; 2475 2476 let Constraints = "$Zda = $_Zda"; 2477 let DestructiveInstType = DestructiveOther; 2478 let ElementSize = zprty.ElementSize; 2479} 2480 2481multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2482 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2483 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2484 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2485 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2486 2487 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2488 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2489 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2490 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2491} 2492 2493//===----------------------------------------------------------------------===// 2494// SVE2 Integer Multiply-Add - Unpredicated Group 2495//===----------------------------------------------------------------------===// 2496 2497class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2498 ZPRRegOp zprty1, ZPRRegOp zprty2> 2499: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2500 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2501 bits<5> Zda; 2502 bits<5> Zn; 2503 bits<5> Zm; 2504 let Inst{31-24} = 0b01000100; 2505 let Inst{23-22} = sz; 2506 let Inst{21} = 0b0; 2507 let Inst{20-16} = Zm; 2508 let Inst{15} = 0b0; 2509 let Inst{14-10} = opc; 2510 let Inst{9-5} = Zn; 2511 let Inst{4-0} = Zda; 2512 2513 let Constraints = "$Zda = $_Zda"; 2514 let DestructiveInstType = DestructiveOther; 2515 let ElementSize = ElementSizeNone; 2516} 2517 2518multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 2519 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 2520 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 2521 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 2522 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 2523 2524 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2525 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2526 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2527 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2528} 2529 2530multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 2531 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 2532 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 2533 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 2534 2535 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 2536 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 2537 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 2538} 2539 2540//===----------------------------------------------------------------------===// 2541// SVE2 Integer Multiply-Add - Indexed Group 2542//===----------------------------------------------------------------------===// 2543 2544class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 2545 ZPRRegOp zprty1, ZPRRegOp zprty2, 2546 ZPRRegOp zprty3, Operand itype> 2547: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2548 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2549 bits<5> Zda; 2550 bits<5> Zn; 2551 let Inst{31-24} = 0b01000100; 2552 let Inst{23-22} = sz; 2553 let Inst{21} = 0b1; 2554 let Inst{15-10} = opc; 2555 let Inst{9-5} = Zn; 2556 let Inst{4-0} = Zda; 2557 2558 let Constraints = "$Zda = $_Zda"; 2559 let DestructiveInstType = DestructiveOther; 2560 let ElementSize = ElementSizeNone; 2561} 2562 2563multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 2564 SDPatternOperator op> { 2565 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 2566 bits<3> Zm; 2567 bits<3> iop; 2568 let Inst{22} = iop{2}; 2569 let Inst{20-19} = iop{1-0}; 2570 let Inst{18-16} = Zm; 2571 } 2572 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 2573 bits<3> Zm; 2574 bits<2> iop; 2575 let Inst{20-19} = iop; 2576 let Inst{18-16} = Zm; 2577 } 2578 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 2579 bits<4> Zm; 2580 bit iop; 2581 let Inst{20} = iop; 2582 let Inst{19-16} = Zm; 2583 } 2584 2585 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 2586 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2587 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2588} 2589 2590//===----------------------------------------------------------------------===// 2591// SVE2 Integer Multiply-Add Long - Indexed Group 2592//===----------------------------------------------------------------------===// 2593 2594multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, SDPatternOperator op> { 2595 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2596 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 2597 bits<3> Zm; 2598 bits<3> iop; 2599 let Inst{20-19} = iop{2-1}; 2600 let Inst{18-16} = Zm; 2601 let Inst{11} = iop{0}; 2602 } 2603 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2604 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 2605 bits<4> Zm; 2606 bits<2> iop; 2607 let Inst{20} = iop{1}; 2608 let Inst{19-16} = Zm; 2609 let Inst{11} = iop{0}; 2610 } 2611 2612 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 2613 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 2614} 2615 2616//===----------------------------------------------------------------------===// 2617// SVE Integer Dot Product Group 2618//===----------------------------------------------------------------------===// 2619 2620class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 2621 ZPRRegOp zprty2> 2622: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 2623 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2624 bits<5> Zda; 2625 bits<5> Zn; 2626 bits<5> Zm; 2627 let Inst{31-23} = 0b010001001; 2628 let Inst{22} = sz; 2629 let Inst{21} = 0; 2630 let Inst{20-16} = Zm; 2631 let Inst{15-11} = 0; 2632 let Inst{10} = U; 2633 let Inst{9-5} = Zn; 2634 let Inst{4-0} = Zda; 2635 2636 let Constraints = "$Zda = $_Zda"; 2637 let DestructiveInstType = DestructiveOther; 2638} 2639 2640multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 2641 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 2642 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 2643 2644 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 2645 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 2646} 2647 2648//===----------------------------------------------------------------------===// 2649// SVE Integer Dot Product Group - Indexed Group 2650//===----------------------------------------------------------------------===// 2651 2652class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 2653 ZPRRegOp zprty1, ZPRRegOp zprty2, 2654 ZPRRegOp zprty3, Operand itype> 2655: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2656 asm, "\t$Zda, $Zn, $Zm$iop", 2657 "", []>, Sched<[]> { 2658 bits<5> Zda; 2659 bits<5> Zn; 2660 let Inst{31-23} = 0b010001001; 2661 let Inst{22} = sz; 2662 let Inst{21} = 0b1; 2663 let Inst{15-11} = 0; 2664 let Inst{10} = U; 2665 let Inst{9-5} = Zn; 2666 let Inst{4-0} = Zda; 2667 2668 let Constraints = "$Zda = $_Zda"; 2669 let DestructiveInstType = DestructiveOther; 2670} 2671 2672multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 2673 SDPatternOperator op> { 2674 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 2675 bits<2> iop; 2676 bits<3> Zm; 2677 let Inst{20-19} = iop; 2678 let Inst{18-16} = Zm; 2679 } 2680 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 2681 bits<1> iop; 2682 bits<4> Zm; 2683 let Inst{20} = iop; 2684 let Inst{19-16} = Zm; 2685 } 2686 2687 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv16i8:$Op2, nxv16i8:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2688 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2689 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv8i16:$Op2, nxv8i16:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2690 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2691} 2692 2693//===----------------------------------------------------------------------===// 2694// SVE2 Complex Integer Dot Product Group 2695//===----------------------------------------------------------------------===// 2696 2697class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 2698 ZPRRegOp zprty1, ZPRRegOp zprty2> 2699: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 2700 complexrotateop:$rot), 2701 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 2702 bits<5> Zda; 2703 bits<5> Zn; 2704 bits<5> Zm; 2705 bits<2> rot; 2706 let Inst{31-24} = 0b01000100; 2707 let Inst{23-22} = sz; 2708 let Inst{21} = 0b0; 2709 let Inst{20-16} = Zm; 2710 let Inst{15-12} = opc; 2711 let Inst{11-10} = rot; 2712 let Inst{9-5} = Zn; 2713 let Inst{4-0} = Zda; 2714 2715 let Constraints = "$Zda = $_Zda"; 2716 let DestructiveInstType = DestructiveOther; 2717 let ElementSize = ElementSizeNone; 2718} 2719 2720multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 2721 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 2722 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 2723 2724 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 2725 (i32 complexrotateop:$imm))), 2726 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 2727 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2728 (i32 complexrotateop:$imm))), 2729 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 2730} 2731 2732//===----------------------------------------------------------------------===// 2733// SVE2 Complex Multiply-Add Group 2734//===----------------------------------------------------------------------===// 2735 2736multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 2737 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 2738 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 2739 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 2740 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 2741 2742 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 2743 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 2744 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 2745 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 2746} 2747 2748//===----------------------------------------------------------------------===// 2749// SVE2 Complex Integer Dot Product - Indexed Group 2750//===----------------------------------------------------------------------===// 2751 2752class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 2753 ZPRRegOp zprty1, ZPRRegOp zprty2, 2754 ZPRRegOp zprty3, Operand itype> 2755: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 2756 complexrotateop:$rot), 2757 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 2758 bits<5> Zda; 2759 bits<5> Zn; 2760 bits<2> rot; 2761 let Inst{31-24} = 0b01000100; 2762 let Inst{23-22} = sz; 2763 let Inst{21} = 0b1; 2764 let Inst{15-12} = opc; 2765 let Inst{11-10} = rot; 2766 let Inst{9-5} = Zn; 2767 let Inst{4-0} = Zda; 2768 2769 let Constraints = "$Zda = $_Zda"; 2770 let DestructiveInstType = DestructiveOther; 2771 let ElementSize = ElementSizeNone; 2772} 2773 2774multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 2775 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 2776 bits<2> iop; 2777 bits<3> Zm; 2778 let Inst{20-19} = iop; 2779 let Inst{18-16} = Zm; 2780 } 2781 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 2782 bit iop; 2783 bits<4> Zm; 2784 let Inst{20} = iop; 2785 let Inst{19-16} = Zm; 2786 } 2787 2788 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 2789 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2790 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2791 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2792 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2793 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2794} 2795 2796//===----------------------------------------------------------------------===// 2797// SVE2 Complex Multiply-Add - Indexed Group 2798//===----------------------------------------------------------------------===// 2799 2800multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 2801 SDPatternOperator op> { 2802 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 2803 bits<2> iop; 2804 bits<3> Zm; 2805 let Inst{20-19} = iop; 2806 let Inst{18-16} = Zm; 2807 } 2808 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 2809 bit iop; 2810 bits<4> Zm; 2811 let Inst{20} = iop; 2812 let Inst{19-16} = Zm; 2813 } 2814 2815 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2816 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2817 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2818 2819 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 2820 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2821 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2822} 2823 2824//===----------------------------------------------------------------------===// 2825// SVE2 Integer Multiply - Unpredicated Group 2826//===----------------------------------------------------------------------===// 2827 2828class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2829: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2830 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 2831 bits<5> Zd; 2832 bits<5> Zm; 2833 bits<5> Zn; 2834 let Inst{31-24} = 0b00000100; 2835 let Inst{23-22} = sz; 2836 let Inst{21} = 0b1; 2837 let Inst{20-16} = Zm; 2838 let Inst{15-13} = 0b011; 2839 let Inst{12-10} = opc; 2840 let Inst{9-5} = Zn; 2841 let Inst{4-0} = Zd; 2842} 2843 2844multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op> { 2845 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 2846 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 2847 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 2848 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 2849 2850 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2851 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2852 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2853 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2854} 2855 2856multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 2857 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 2858 2859 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2860} 2861 2862//===----------------------------------------------------------------------===// 2863// SVE2 Integer Multiply - Indexed Group 2864//===----------------------------------------------------------------------===// 2865 2866class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 2867 ZPRRegOp zprty1, ZPRRegOp zprty2, 2868 ZPRRegOp zprty3, Operand itype> 2869: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 2870 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2871 bits<5> Zd; 2872 bits<5> Zn; 2873 let Inst{31-24} = 0b01000100; 2874 let Inst{23-22} = sz; 2875 let Inst{21} = 0b1; 2876 let Inst{15-14} = 0b11; 2877 let Inst{13-10} = opc; 2878 let Inst{9-5} = Zn; 2879 let Inst{4-0} = Zd; 2880} 2881 2882multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 2883 SDPatternOperator op> { 2884 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 2885 bits<3> Zm; 2886 bits<3> iop; 2887 let Inst{22} = iop{2}; 2888 let Inst{20-19} = iop{1-0}; 2889 let Inst{18-16} = Zm; 2890 } 2891 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 2892 bits<3> Zm; 2893 bits<2> iop; 2894 let Inst{20-19} = iop; 2895 let Inst{18-16} = Zm; 2896 } 2897 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 2898 bits<4> Zm; 2899 bit iop; 2900 let Inst{20} = iop; 2901 let Inst{19-16} = Zm; 2902 } 2903 2904 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 2905 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2906 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2907} 2908 2909multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 2910 SDPatternOperator op> { 2911 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 2912 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 2913 bits<3> Zm; 2914 bits<3> iop; 2915 let Inst{20-19} = iop{2-1}; 2916 let Inst{18-16} = Zm; 2917 let Inst{11} = iop{0}; 2918 } 2919 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 2920 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 2921 bits<4> Zm; 2922 bits<2> iop; 2923 let Inst{20} = iop{1}; 2924 let Inst{19-16} = Zm; 2925 let Inst{11} = iop{0}; 2926 } 2927 2928 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 2929 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 2930} 2931 2932//===----------------------------------------------------------------------===// 2933// SVE2 Integer - Predicated Group 2934//===----------------------------------------------------------------------===// 2935 2936class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 2937 ZPRRegOp zprty> 2938: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2939 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2940 bits<3> Pg; 2941 bits<5> Zm; 2942 bits<5> Zdn; 2943 let Inst{31-24} = 0b01000100; 2944 let Inst{23-22} = sz; 2945 let Inst{21-20} = 0b01; 2946 let Inst{20-16} = opc{5-1}; 2947 let Inst{15-14} = 0b10; 2948 let Inst{13} = opc{0}; 2949 let Inst{12-10} = Pg; 2950 let Inst{9-5} = Zm; 2951 let Inst{4-0} = Zdn; 2952 2953 let Constraints = "$Zdn = $_Zdn"; 2954 let DestructiveInstType = DestructiveOther; 2955 let ElementSize = zprty.ElementSize; 2956} 2957 2958multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op> { 2959 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>; 2960 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>; 2961 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>; 2962 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>; 2963 2964 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2965 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2966 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2967 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2968} 2969 2970class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 2971 ZPRRegOp zprty1, ZPRRegOp zprty2> 2972: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 2973 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 2974 bits<3> Pg; 2975 bits<5> Zn; 2976 bits<5> Zda; 2977 let Inst{31-24} = 0b01000100; 2978 let Inst{23-22} = sz; 2979 let Inst{21-17} = 0b00010; 2980 let Inst{16} = U; 2981 let Inst{15-13} = 0b101; 2982 let Inst{12-10} = Pg; 2983 let Inst{9-5} = Zn; 2984 let Inst{4-0} = Zda; 2985 2986 let Constraints = "$Zda = $_Zda"; 2987 let DestructiveInstType = DestructiveOther; 2988 let ElementSize = zprty1.ElementSize; 2989} 2990 2991multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 2992 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 2993 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 2994 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 2995 2996 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 2997 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 2998 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 2999} 3000 3001class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3002 string asm, ZPRRegOp zprty> 3003: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3004 asm, "\t$Zd, $Pg/m, $Zn", 3005 "", 3006 []>, Sched<[]> { 3007 bits<3> Pg; 3008 bits<5> Zd; 3009 bits<5> Zn; 3010 let Inst{31-24} = 0b01000100; 3011 let Inst{23-22} = sz; 3012 let Inst{21-20} = 0b00; 3013 let Inst{19} = Q; 3014 let Inst{18} = 0b0; 3015 let Inst{17-16} = opc; 3016 let Inst{15-13} = 0b101; 3017 let Inst{12-10} = Pg; 3018 let Inst{9-5} = Zn; 3019 let Inst{4-0} = Zd; 3020 3021 let Constraints = "$Zd = $_Zd"; 3022 let DestructiveInstType = DestructiveOther; 3023 let ElementSize = zprty.ElementSize; 3024} 3025 3026multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3027 SDPatternOperator op> { 3028 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; 3029 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3030} 3031 3032multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3033 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>; 3034 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>; 3035 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; 3036 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>; 3037 3038 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3039 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3040 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3041 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3042} 3043 3044//===----------------------------------------------------------------------===// 3045// SVE2 Widening Integer Arithmetic Group 3046//===----------------------------------------------------------------------===// 3047 3048class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3049 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3050: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3051 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3052 bits<5> Zd; 3053 bits<5> Zn; 3054 bits<5> Zm; 3055 let Inst{31-24} = 0b01000101; 3056 let Inst{23-22} = sz; 3057 let Inst{21} = 0b0; 3058 let Inst{20-16} = Zm; 3059 let Inst{15} = 0b0; 3060 let Inst{14-10} = opc; 3061 let Inst{9-5} = Zn; 3062 let Inst{4-0} = Zd; 3063} 3064 3065multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3066 SDPatternOperator op> { 3067 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3068 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3069 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3070 3071 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3072 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3073 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3074} 3075 3076multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3077 SDPatternOperator op> { 3078 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3079 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3080 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3081 3082 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3083 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3084 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3085} 3086 3087multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3088 SDPatternOperator op> { 3089 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3090 3091 // To avoid using 128 bit elements in the IR, the pattern below works with 3092 // llvm intrinsics with the _pair suffix, to reflect that 3093 // _Q is implemented as a pair of _D. 3094 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3095} 3096 3097multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3098 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3099 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3100 3101 // To avoid using 128 bit elements in the IR, the patterns below work with 3102 // llvm intrinsics with the _pair suffix, to reflect that 3103 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3104 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3105 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3106} 3107 3108//===----------------------------------------------------------------------===// 3109// SVE2 Misc Group 3110//===----------------------------------------------------------------------===// 3111 3112class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3113 ZPRRegOp zprty1, ZPRRegOp zprty2> 3114: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3115 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3116 bits<5> Zd; 3117 bits<5> Zn; 3118 bits<5> Zm; 3119 let Inst{31-24} = 0b01000101; 3120 let Inst{23-22} = sz; 3121 let Inst{21} = 0b0; 3122 let Inst{20-16} = Zm; 3123 let Inst{15-14} = 0b10; 3124 let Inst{13-10} = opc; 3125 let Inst{9-5} = Zn; 3126 let Inst{4-0} = Zd; 3127} 3128 3129multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3130 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3131 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3132 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3133 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3134 3135 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3136 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3137 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3138 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3139} 3140 3141multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3142 SDPatternOperator op> { 3143 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3144 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3145 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3146 3147 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3148 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3149 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3150} 3151 3152class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3153 ZPRRegOp zprty1, ZPRRegOp zprty2> 3154: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3155 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3156 bits<5> Zd; 3157 bits<5> Zn; 3158 bits<5> Zm; 3159 let Inst{31-24} = 0b01000101; 3160 let Inst{23-22} = sz; 3161 let Inst{21} = 0b0; 3162 let Inst{20-16} = Zm; 3163 let Inst{15-11} = 0b10010; 3164 let Inst{10} = opc; 3165 let Inst{9-5} = Zn; 3166 let Inst{4-0} = Zd; 3167 3168 let Constraints = "$Zd = $_Zd"; 3169 let DestructiveInstType = DestructiveOther; 3170 let ElementSize = ElementSizeNone; 3171} 3172 3173multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3174 SDPatternOperator op> { 3175 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3176 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3177 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3178 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3179 3180 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3181 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3182 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3183 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3184} 3185 3186class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3187 ZPRRegOp zprty1, ZPRRegOp zprty2, 3188 Operand immtype> 3189: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3190 asm, "\t$Zd, $Zn, $imm", 3191 "", []>, Sched<[]> { 3192 bits<5> Zd; 3193 bits<5> Zn; 3194 bits<5> imm; 3195 let Inst{31-23} = 0b010001010; 3196 let Inst{22} = tsz8_64{2}; 3197 let Inst{21} = 0b0; 3198 let Inst{20-19} = tsz8_64{1-0}; 3199 let Inst{18-16} = imm{2-0}; // imm3 3200 let Inst{15-12} = 0b1010; 3201 let Inst{11-10} = opc; 3202 let Inst{9-5} = Zn; 3203 let Inst{4-0} = Zd; 3204} 3205 3206multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3207 SDPatternOperator op> { 3208 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3209 ZPR16, ZPR8, vecshiftL8>; 3210 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3211 ZPR32, ZPR16, vecshiftL16> { 3212 let Inst{19} = imm{3}; 3213 } 3214 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3215 ZPR64, ZPR32, vecshiftL32> { 3216 let Inst{20-19} = imm{4-3}; 3217 } 3218 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3219 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3220 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3221} 3222 3223//===----------------------------------------------------------------------===// 3224// SVE2 Accumulate Group 3225//===----------------------------------------------------------------------===// 3226 3227class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3228 ZPRRegOp zprty, Operand immtype> 3229: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3230 asm, "\t$Zd, $Zn, $imm", 3231 "", []>, Sched<[]> { 3232 bits<5> Zd; 3233 bits<5> Zn; 3234 bits<6> imm; 3235 let Inst{31-24} = 0b01000101; 3236 let Inst{23-22} = tsz8_64{3-2}; 3237 let Inst{21} = 0b0; 3238 let Inst{20-19} = tsz8_64{1-0}; 3239 let Inst{18-16} = imm{2-0}; // imm3 3240 let Inst{15-11} = 0b11110; 3241 let Inst{10} = opc; 3242 let Inst{9-5} = Zn; 3243 let Inst{4-0} = Zd; 3244 3245 let Constraints = "$Zd = $_Zd"; 3246} 3247 3248multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3249 SDPatternOperator op> { 3250 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3251 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3252 let Inst{19} = imm{3}; 3253 } 3254 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3255 let Inst{20-19} = imm{4-3}; 3256 } 3257 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3258 let Inst{22} = imm{5}; 3259 let Inst{20-19} = imm{4-3}; 3260 } 3261 3262 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3263 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3264 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3265 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3266} 3267 3268multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3269 SDPatternOperator op> { 3270 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3271 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3272 let Inst{19} = imm{3}; 3273 } 3274 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3275 let Inst{20-19} = imm{4-3}; 3276 } 3277 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3278 let Inst{22} = imm{5}; 3279 let Inst{20-19} = imm{4-3}; 3280 } 3281 3282 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3283 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3284 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3285 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3286} 3287 3288class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3289 ZPRRegOp zprty, Operand immtype> 3290: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3291 asm, "\t$Zda, $Zn, $imm", 3292 "", []>, Sched<[]> { 3293 bits<5> Zda; 3294 bits<5> Zn; 3295 bits<6> imm; 3296 let Inst{31-24} = 0b01000101; 3297 let Inst{23-22} = tsz8_64{3-2}; 3298 let Inst{21} = 0b0; 3299 let Inst{20-19} = tsz8_64{1-0}; 3300 let Inst{18-16} = imm{2-0}; // imm3 3301 let Inst{15-12} = 0b1110; 3302 let Inst{11-10} = opc; 3303 let Inst{9-5} = Zn; 3304 let Inst{4-0} = Zda; 3305 3306 let Constraints = "$Zda = $_Zda"; 3307 let DestructiveInstType = DestructiveOther; 3308 let ElementSize = ElementSizeNone; 3309} 3310 3311multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3312 SDPatternOperator op> { 3313 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3314 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3315 let Inst{19} = imm{3}; 3316 } 3317 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3318 let Inst{20-19} = imm{4-3}; 3319 } 3320 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3321 let Inst{22} = imm{5}; 3322 let Inst{20-19} = imm{4-3}; 3323 } 3324 3325 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3326 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3327 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3328 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3329} 3330 3331class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3332: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3333 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3334 bits<5> Zdn; 3335 bits<5> Zm; 3336 bit rot; 3337 let Inst{31-24} = 0b01000101; 3338 let Inst{23-22} = sz; 3339 let Inst{21-17} = 0b00000; 3340 let Inst{16} = opc; 3341 let Inst{15-11} = 0b11011; 3342 let Inst{10} = rot; 3343 let Inst{9-5} = Zm; 3344 let Inst{4-0} = Zdn; 3345 3346 let Constraints = "$Zdn = $_Zdn"; 3347 let DestructiveInstType = DestructiveOther; 3348 let ElementSize = ElementSizeNone; 3349} 3350 3351multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3352 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3353 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3354 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3355 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3356 3357 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3358 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3359 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3360 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3361} 3362 3363class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3364 ZPRRegOp zprty1, ZPRRegOp zprty2> 3365: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3366 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3367 bits<5> Zda; 3368 bits<5> Zn; 3369 bits<5> Zm; 3370 let Inst{31-24} = 0b01000101; 3371 let Inst{23-22} = sz; 3372 let Inst{21} = 0b0; 3373 let Inst{20-16} = Zm; 3374 let Inst{15-14} = 0b11; 3375 let Inst{13-10} = opc; 3376 let Inst{9-5} = Zn; 3377 let Inst{4-0} = Zda; 3378 3379 let Constraints = "$Zda = $_Zda"; 3380 let DestructiveInstType = DestructiveOther; 3381 let ElementSize = ElementSizeNone; 3382} 3383 3384multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3385 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3386 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3387 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3388 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3389 3390 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3391 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3392 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3393 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3394} 3395 3396multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3397 SDPatternOperator op> { 3398 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3399 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3400 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3401 3402 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3403 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3404 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3405} 3406 3407multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, SDPatternOperator op> { 3408 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3409 ZPR32, ZPR32>; 3410 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3411 ZPR64, ZPR64>; 3412 3413 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3414 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3415} 3416 3417//===----------------------------------------------------------------------===// 3418// SVE2 Narrowing Group 3419//===----------------------------------------------------------------------===// 3420 3421class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3422 string asm, ZPRRegOp zprty1, 3423 ZPRRegOp zprty2, Operand immtype> 3424: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3425 asm, "\t$Zd, $Zn, $imm", 3426 "", []>, Sched<[]> { 3427 bits<5> Zd; 3428 bits<5> Zn; 3429 bits<5> imm; 3430 let Inst{31-23} = 0b010001010; 3431 let Inst{22} = tsz8_64{2}; 3432 let Inst{21} = 0b1; 3433 let Inst{20-19} = tsz8_64{1-0}; 3434 let Inst{18-16} = imm{2-0}; // imm3 3435 let Inst{15-14} = 0b00; 3436 let Inst{13-11} = opc; 3437 let Inst{10} = 0b0; 3438 let Inst{9-5} = Zn; 3439 let Inst{4-0} = Zd; 3440} 3441 3442multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3443 SDPatternOperator op> { 3444 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3445 tvecshiftR8>; 3446 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3447 tvecshiftR16> { 3448 let Inst{19} = imm{3}; 3449 } 3450 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3451 vecshiftR32> { 3452 let Inst{20-19} = imm{4-3}; 3453 } 3454 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3455 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3456 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3457} 3458 3459class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3460 string asm, ZPRRegOp zprty1, 3461 ZPRRegOp zprty2, Operand immtype> 3462: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3463 asm, "\t$Zd, $Zn, $imm", 3464 "", []>, Sched<[]> { 3465 bits<5> Zd; 3466 bits<5> Zn; 3467 bits<5> imm; 3468 let Inst{31-23} = 0b010001010; 3469 let Inst{22} = tsz8_64{2}; 3470 let Inst{21} = 0b1; 3471 let Inst{20-19} = tsz8_64{1-0}; 3472 let Inst{18-16} = imm{2-0}; // imm3 3473 let Inst{15-14} = 0b00; 3474 let Inst{13-11} = opc; 3475 let Inst{10} = 0b1; 3476 let Inst{9-5} = Zn; 3477 let Inst{4-0} = Zd; 3478 3479 let Constraints = "$Zd = $_Zd"; 3480} 3481 3482multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 3483 SDPatternOperator op> { 3484 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 3485 tvecshiftR8>; 3486 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 3487 tvecshiftR16> { 3488 let Inst{19} = imm{3}; 3489 } 3490 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 3491 vecshiftR32> { 3492 let Inst{20-19} = imm{4-3}; 3493 } 3494 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3495 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3496 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3497} 3498 3499class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 3500 ZPRRegOp zprty1, ZPRRegOp zprty2> 3501: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3502 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3503 bits<5> Zd; 3504 bits<5> Zn; 3505 bits<5> Zm; 3506 let Inst{31-24} = 0b01000101; 3507 let Inst{23-22} = sz; 3508 let Inst{21} = 0b1; 3509 let Inst{20-16} = Zm; 3510 let Inst{15-13} = 0b011; 3511 let Inst{12-11} = opc; // S, R 3512 let Inst{10} = 0b0; // Top 3513 let Inst{9-5} = Zn; 3514 let Inst{4-0} = Zd; 3515} 3516 3517multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 3518 SDPatternOperator op> { 3519 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 3520 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 3521 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 3522 3523 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3524 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3525 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3526} 3527 3528class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 3529 ZPRRegOp zprty1, ZPRRegOp zprty2> 3530: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3531 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3532 bits<5> Zd; 3533 bits<5> Zn; 3534 bits<5> Zm; 3535 let Inst{31-24} = 0b01000101; 3536 let Inst{23-22} = sz; 3537 let Inst{21} = 0b1; 3538 let Inst{20-16} = Zm; 3539 let Inst{15-13} = 0b011; 3540 let Inst{12-11} = opc; // S, R 3541 let Inst{10} = 0b1; // Top 3542 let Inst{9-5} = Zn; 3543 let Inst{4-0} = Zd; 3544 3545 let Constraints = "$Zd = $_Zd"; 3546} 3547 3548multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 3549 SDPatternOperator op> { 3550 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 3551 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 3552 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 3553 3554 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3555 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3556 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3557} 3558 3559class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 3560 ZPRRegOp zprty1, ZPRRegOp zprty2> 3561: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 3562 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3563 bits<5> Zd; 3564 bits<5> Zn; 3565 let Inst{31-23} = 0b010001010; 3566 let Inst{22} = tsz8_64{2}; 3567 let Inst{21} = 0b1; 3568 let Inst{20-19} = tsz8_64{1-0}; 3569 let Inst{18-13} = 0b000010; 3570 let Inst{12-11} = opc; 3571 let Inst{10} = 0b0; 3572 let Inst{9-5} = Zn; 3573 let Inst{4-0} = Zd; 3574} 3575 3576multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 3577 SDPatternOperator op> { 3578 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 3579 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 3580 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 3581 3582 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 3583 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 3584 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 3585} 3586 3587class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 3588 ZPRRegOp zprty1, ZPRRegOp zprty2> 3589: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 3590 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3591 bits<5> Zd; 3592 bits<5> Zn; 3593 let Inst{31-23} = 0b010001010; 3594 let Inst{22} = tsz8_64{2}; 3595 let Inst{21} = 0b1; 3596 let Inst{20-19} = tsz8_64{1-0}; 3597 let Inst{18-13} = 0b000010; 3598 let Inst{12-11} = opc; 3599 let Inst{10} = 0b1; 3600 let Inst{9-5} = Zn; 3601 let Inst{4-0} = Zd; 3602 3603 let Constraints = "$Zd = $_Zd"; 3604} 3605 3606multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 3607 SDPatternOperator op> { 3608 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 3609 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 3610 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 3611 3612 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 3613 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 3614 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 3615} 3616 3617//===----------------------------------------------------------------------===// 3618// SVE Integer Arithmetic - Unary Predicated Group 3619//===----------------------------------------------------------------------===// 3620 3621class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 3622 string asm, ZPRRegOp zprty> 3623: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3624 asm, "\t$Zd, $Pg/m, $Zn", 3625 "", 3626 []>, Sched<[]> { 3627 bits<3> Pg; 3628 bits<5> Zd; 3629 bits<5> Zn; 3630 let Inst{31-24} = 0b00000100; 3631 let Inst{23-22} = sz8_64; 3632 let Inst{21-20} = 0b01; 3633 let Inst{19} = opc{0}; 3634 let Inst{18-16} = opc{3-1}; 3635 let Inst{15-13} = 0b101; 3636 let Inst{12-10} = Pg; 3637 let Inst{9-5} = Zn; 3638 let Inst{4-0} = Zd; 3639 3640 let Constraints = "$Zd = $_Zd"; 3641 let DestructiveInstType = DestructiveOther; 3642 let ElementSize = zprty.ElementSize; 3643} 3644 3645multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 3646 SDPatternOperator op> { 3647 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>; 3648 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>; 3649 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3650 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3651 3652 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3653 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3654 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3655 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3656} 3657 3658multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 3659 SDPatternOperator op> { 3660 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>; 3661 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3662 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3663 3664 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3665 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3666 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3667} 3668 3669multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 3670 SDPatternOperator op> { 3671 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3672 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3673 3674 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3675 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3676} 3677 3678multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 3679 SDPatternOperator op> { 3680 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3681 3682 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3683} 3684 3685multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 3686 SDPatternOperator op> { 3687 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>; 3688 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>; 3689 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>; 3690 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>; 3691 3692 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3693 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3694 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3695 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3696 3697 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 3698 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 3699 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 3700} 3701 3702multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, 3703 SDPatternOperator op> { 3704 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>; 3705 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>; 3706 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>; 3707 3708 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 3709 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 3710 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 3711} 3712 3713//===----------------------------------------------------------------------===// 3714// SVE Integer Wide Immediate - Unpredicated Group 3715//===----------------------------------------------------------------------===// 3716class sve_int_dup_imm<bits<2> sz8_64, string asm, 3717 ZPRRegOp zprty, Operand immtype> 3718: I<(outs zprty:$Zd), (ins immtype:$imm), 3719 asm, "\t$Zd, $imm", 3720 "", 3721 []>, Sched<[]> { 3722 bits<5> Zd; 3723 bits<9> imm; 3724 let Inst{31-24} = 0b00100101; 3725 let Inst{23-22} = sz8_64; 3726 let Inst{21-14} = 0b11100011; 3727 let Inst{13} = imm{8}; // sh 3728 let Inst{12-5} = imm{7-0}; // imm8 3729 let Inst{4-0} = Zd; 3730 3731 let isReMaterializable = 1; 3732} 3733 3734multiclass sve_int_dup_imm<string asm> { 3735 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 3736 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 3737 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 3738 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 3739 3740 def : InstAlias<"mov $Zd, $imm", 3741 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 3742 def : InstAlias<"mov $Zd, $imm", 3743 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 3744 def : InstAlias<"mov $Zd, $imm", 3745 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 3746 def : InstAlias<"mov $Zd, $imm", 3747 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 3748 3749 def : InstAlias<"fmov $Zd, #0.0", 3750 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 3751 def : InstAlias<"fmov $Zd, #0.0", 3752 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 3753 def : InstAlias<"fmov $Zd, #0.0", 3754 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 3755} 3756 3757class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 3758 string asm, ZPRRegOp zprty> 3759: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 3760 asm, "\t$Zd, $imm8", 3761 "", 3762 []>, Sched<[]> { 3763 bits<5> Zd; 3764 bits<8> imm8; 3765 let Inst{31-24} = 0b00100101; 3766 let Inst{23-22} = sz8_64; 3767 let Inst{21-14} = 0b11100111; 3768 let Inst{13} = 0b0; 3769 let Inst{12-5} = imm8; 3770 let Inst{4-0} = Zd; 3771 3772 let isReMaterializable = 1; 3773} 3774 3775multiclass sve_int_dup_fpimm<string asm> { 3776 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 3777 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 3778 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 3779 3780 def : InstAlias<"fmov $Zd, $imm8", 3781 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 3782 def : InstAlias<"fmov $Zd, $imm8", 3783 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 3784 def : InstAlias<"fmov $Zd, $imm8", 3785 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 3786} 3787 3788class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 3789 ZPRRegOp zprty, Operand immtype> 3790: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 3791 asm, "\t$Zdn, $_Zdn, $imm", 3792 "", 3793 []>, Sched<[]> { 3794 bits<5> Zdn; 3795 bits<9> imm; 3796 let Inst{31-24} = 0b00100101; 3797 let Inst{23-22} = sz8_64; 3798 let Inst{21-19} = 0b100; 3799 let Inst{18-16} = opc; 3800 let Inst{15-14} = 0b11; 3801 let Inst{13} = imm{8}; // sh 3802 let Inst{12-5} = imm{7-0}; // imm8 3803 let Inst{4-0} = Zdn; 3804 3805 let Constraints = "$Zdn = $_Zdn"; 3806 let DestructiveInstType = DestructiveOther; 3807 let ElementSize = ElementSizeNone; 3808} 3809 3810multiclass sve_int_arith_imm0<bits<3> opc, string asm, 3811 SDPatternOperator op, SDPatternOperator int_op> { 3812 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 3813 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 3814 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 3815 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 3816 3817 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3818 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3819 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3820 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3821 3822 // Intrinsic version 3823 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, int_op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3824 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, int_op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3825 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, int_op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3826 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, int_op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3827} 3828 3829multiclass sve_int_arith_imm0_subr<bits<3> opc, string asm, SDPatternOperator op> { 3830 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 3831 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 3832 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 3833 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 3834 3835 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3836 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3837 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3838 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3839} 3840 3841class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 3842 ZPRRegOp zprty, Operand immtype> 3843: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 3844 asm, "\t$Zdn, $_Zdn, $imm", 3845 "", 3846 []>, Sched<[]> { 3847 bits<5> Zdn; 3848 bits<8> imm; 3849 let Inst{31-24} = 0b00100101; 3850 let Inst{23-22} = sz8_64; 3851 let Inst{21-16} = opc; 3852 let Inst{15-13} = 0b110; 3853 let Inst{12-5} = imm; 3854 let Inst{4-0} = Zdn; 3855 3856 let Constraints = "$Zdn = $_Zdn"; 3857 let DestructiveInstType = DestructiveOther; 3858 let ElementSize = ElementSizeNone; 3859} 3860 3861multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 3862 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 3863 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 3864 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 3865 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 3866 3867 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 3868 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 3869 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 3870 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 3871} 3872 3873multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 3874 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 3875 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 3876 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 3877 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 3878 3879 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _B)>; 3880 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _H)>; 3881 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImmPat, !cast<Instruction>(NAME # _S)>; 3882 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImmPat, !cast<Instruction>(NAME # _D)>; 3883} 3884 3885multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 3886 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 3887 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 3888 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 3889 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 3890 3891 def : SVE_1_Op_Imm_Arith_Pat<nxv16i8, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 3892 def : SVE_1_Op_Imm_Arith_Pat<nxv8i16, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 3893 def : SVE_1_Op_Imm_Arith_Pat<nxv4i32, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 3894 def : SVE_1_Op_Imm_Arith_Pat<nxv2i64, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 3895} 3896 3897//===----------------------------------------------------------------------===// 3898// SVE Bitwise Logical - Unpredicated Group 3899//===----------------------------------------------------------------------===// 3900 3901class sve_int_bin_cons_log<bits<2> opc, string asm> 3902: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 3903 asm, "\t$Zd, $Zn, $Zm", 3904 "", 3905 []>, Sched<[]> { 3906 bits<5> Zd; 3907 bits<5> Zm; 3908 bits<5> Zn; 3909 let Inst{31-24} = 0b00000100; 3910 let Inst{23-22} = opc{1-0}; 3911 let Inst{21} = 0b1; 3912 let Inst{20-16} = Zm; 3913 let Inst{15-10} = 0b001100; 3914 let Inst{9-5} = Zn; 3915 let Inst{4-0} = Zd; 3916} 3917 3918multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 3919 def NAME : sve_int_bin_cons_log<opc, asm>; 3920 3921 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 3922 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 3923 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 3924 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3925 3926 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 3927 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 3928 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 3929 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 3930 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 3931 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 3932} 3933 3934class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 3935: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 3936 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 3937 "", 3938 []>, Sched<[]> { 3939 bits<5> Zdn; 3940 bits<5> Zk; 3941 bits<5> Zm; 3942 let Inst{31-24} = 0b00000100; 3943 let Inst{23-22} = opc{2-1}; 3944 let Inst{21} = 0b1; 3945 let Inst{20-16} = Zm; 3946 let Inst{15-11} = 0b00111; 3947 let Inst{10} = opc{0}; 3948 let Inst{9-5} = Zk; 3949 let Inst{4-0} = Zdn; 3950 3951 let Constraints = "$Zdn = $_Zdn"; 3952 let DestructiveInstType = DestructiveOther; 3953 let ElementSize = ElementSizeNone; 3954} 3955 3956multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> { 3957 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 3958 3959 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 3960 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 3961 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 3962 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 3963 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 3964 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 3965 3966 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 3967 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 3968 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 3969 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3970} 3971 3972class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 3973 ZPRRegOp zprty, Operand immtype> 3974: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 3975 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 3976 "", 3977 []>, Sched<[]> { 3978 bits<5> Zdn; 3979 bits<5> Zm; 3980 bits<6> imm; 3981 let Inst{31-24} = 0b00000100; 3982 let Inst{23-22} = tsz8_64{3-2}; 3983 let Inst{21} = 0b1; 3984 let Inst{20-19} = tsz8_64{1-0}; 3985 let Inst{18-16} = imm{2-0}; // imm3 3986 let Inst{15-10} = 0b001101; 3987 let Inst{9-5} = Zm; 3988 let Inst{4-0} = Zdn; 3989 3990 let Constraints = "$Zdn = $_Zdn"; 3991 let DestructiveInstType = DestructiveOther; 3992 let ElementSize = ElementSizeNone; 3993} 3994 3995multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 3996 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 3997 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 3998 let Inst{19} = imm{3}; 3999 } 4000 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4001 let Inst{20-19} = imm{4-3}; 4002 } 4003 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4004 let Inst{22} = imm{5}; 4005 let Inst{20-19} = imm{4-3}; 4006 } 4007 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4008 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4009 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4010 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4011} 4012 4013//===----------------------------------------------------------------------===// 4014// SVE Integer Wide Immediate - Predicated Group 4015//===----------------------------------------------------------------------===// 4016 4017class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4018 string asm, ZPRRegOp zprty> 4019: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4020 asm, "\t$Zd, $Pg/m, $imm8", 4021 "", 4022 []>, Sched<[]> { 4023 bits<4> Pg; 4024 bits<5> Zd; 4025 bits<8> imm8; 4026 let Inst{31-24} = 0b00000101; 4027 let Inst{23-22} = sz; 4028 let Inst{21-20} = 0b01; 4029 let Inst{19-16} = Pg; 4030 let Inst{15-13} = 0b110; 4031 let Inst{12-5} = imm8; 4032 let Inst{4-0} = Zd; 4033 4034 let Constraints = "$Zd = $_Zd"; 4035 let DestructiveInstType = DestructiveOther; 4036 let ElementSize = zprty.ElementSize; 4037} 4038 4039multiclass sve_int_dup_fpimm_pred<string asm> { 4040 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4041 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4042 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4043 4044 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4045 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4046 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4047 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4048 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4049 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4050} 4051 4052class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4053 ZPRRegOp zprty, string pred_qual, dag iops> 4054: I<(outs zprty:$Zd), iops, 4055 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4056 "", []>, Sched<[]> { 4057 bits<5> Zd; 4058 bits<4> Pg; 4059 bits<9> imm; 4060 let Inst{31-24} = 0b00000101; 4061 let Inst{23-22} = sz8_64; 4062 let Inst{21-20} = 0b01; 4063 let Inst{19-16} = Pg; 4064 let Inst{15} = 0b0; 4065 let Inst{14} = m; 4066 let Inst{13} = imm{8}; // sh 4067 let Inst{12-5} = imm{7-0}; // imm8 4068 let Inst{4-0} = Zd; 4069 4070 let DestructiveInstType = DestructiveOther; 4071 let ElementSize = zprty.ElementSize; 4072} 4073 4074multiclass sve_int_dup_imm_pred_merge_inst< 4075 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4076 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4077 let Constraints = "$Zd = $_Zd" in 4078 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4079 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4080 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4081 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4082 def : Pat<(intty 4083 (vselect predty:$Pg, 4084 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4085 intty:$Zd)), 4086 (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; 4087} 4088 4089multiclass sve_int_dup_imm_pred_merge<string asm> { 4090 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4091 i32, cpy_imm8_opt_lsl_i8>; 4092 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4093 i32, cpy_imm8_opt_lsl_i16>; 4094 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4095 i32, cpy_imm8_opt_lsl_i32>; 4096 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4097 i64, cpy_imm8_opt_lsl_i64>; 4098 4099 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4100 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4101 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4102 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4103 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4104 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4105} 4106 4107multiclass sve_int_dup_imm_pred_zero_inst< 4108 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4109 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4110 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4111 (ins PPRAny:$Pg, cpyimm:$imm)>; 4112 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4113 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4114 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4115 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4116 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4117 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4118 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4119 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4120 def : Pat<(intty 4121 (vselect predty:$Pg, 4122 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4123 (intty (AArch64dup (scalarty 0))))), 4124 (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>; 4125} 4126 4127multiclass sve_int_dup_imm_pred_zero<string asm> { 4128 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4129 i32, cpy_imm8_opt_lsl_i8>; 4130 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4131 i32, cpy_imm8_opt_lsl_i16>; 4132 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4133 i32, cpy_imm8_opt_lsl_i32>; 4134 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4135 i64, cpy_imm8_opt_lsl_i64>; 4136} 4137 4138//===----------------------------------------------------------------------===// 4139// SVE Integer Compare - Vectors Group 4140//===----------------------------------------------------------------------===// 4141 4142class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4143 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4144: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4145 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4146 "", 4147 []>, Sched<[]> { 4148 bits<4> Pd; 4149 bits<3> Pg; 4150 bits<5> Zm; 4151 bits<5> Zn; 4152 let Inst{31-24} = 0b00100100; 4153 let Inst{23-22} = sz8_64; 4154 let Inst{21} = 0b0; 4155 let Inst{20-16} = Zm; 4156 let Inst{15} = opc{2}; 4157 let Inst{14} = cmp_1; 4158 let Inst{13} = opc{1}; 4159 let Inst{12-10} = Pg; 4160 let Inst{9-5} = Zn; 4161 let Inst{4} = opc{0}; 4162 let Inst{3-0} = Pd; 4163 4164 let Defs = [NZCV]; 4165} 4166 4167multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4168 ValueType intvt, sve_int_cmp cmp> { 4169 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4170 (cmp $Op1, $Op2, $Op3)>; 4171 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4172 (cmp $Op1, $Op3, $Op2)>; 4173} 4174 4175multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4176 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4177 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4178 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4179 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4180 4181 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4182 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4183 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4184 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4185} 4186 4187multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4188 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4189 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4190 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4191 4192 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4193 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4194 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4195} 4196 4197multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4198 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4199 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4200 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4201 4202 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4203 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4204 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4205} 4206 4207 4208//===----------------------------------------------------------------------===// 4209// SVE Integer Compare - Signed Immediate Group 4210//===----------------------------------------------------------------------===// 4211 4212class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4213 ZPRRegOp zprty, 4214 Operand immtype> 4215: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4216 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4217 "", 4218 []>, Sched<[]> { 4219 bits<4> Pd; 4220 bits<3> Pg; 4221 bits<5> Zn; 4222 bits<5> imm5; 4223 let Inst{31-24} = 0b00100101; 4224 let Inst{23-22} = sz8_64; 4225 let Inst{21} = 0b0; 4226 let Inst{20-16} = imm5; 4227 let Inst{15} = opc{2}; 4228 let Inst{14} = 0b0; 4229 let Inst{13} = opc{1}; 4230 let Inst{12-10} = Pg; 4231 let Inst{9-5} = Zn; 4232 let Inst{4} = opc{0}; 4233 let Inst{3-0} = Pd; 4234 4235 let Defs = [NZCV]; 4236 let ElementSize = pprty.ElementSize; 4237} 4238 4239multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4240 ValueType predvt, ValueType intvt, 4241 Operand immtype, Instruction cmp> { 4242 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4243 (intvt ZPR:$Zs1), 4244 (intvt (AArch64dup (immtype:$imm))), 4245 cc)), 4246 (cmp $Pg, $Zs1, immtype:$imm)>; 4247 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4248 (intvt (AArch64dup (immtype:$imm))), 4249 (intvt ZPR:$Zs1), 4250 commuted_cc)), 4251 (cmp $Pg, $Zs1, immtype:$imm)>; 4252} 4253 4254multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4255 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4256 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4257 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4258 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4259 4260 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4261 !cast<Instruction>(NAME # _B)>; 4262 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4263 !cast<Instruction>(NAME # _H)>; 4264 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4265 !cast<Instruction>(NAME # _S)>; 4266 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4267 !cast<Instruction>(NAME # _D)>; 4268} 4269 4270 4271//===----------------------------------------------------------------------===// 4272// SVE Integer Compare - Unsigned Immediate Group 4273//===----------------------------------------------------------------------===// 4274 4275class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4276 ZPRRegOp zprty, Operand immtype> 4277: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4278 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4279 "", 4280 []>, Sched<[]> { 4281 bits<4> Pd; 4282 bits<3> Pg; 4283 bits<5> Zn; 4284 bits<7> imm7; 4285 let Inst{31-24} = 0b00100100; 4286 let Inst{23-22} = sz8_64; 4287 let Inst{21} = 1; 4288 let Inst{20-14} = imm7; 4289 let Inst{13} = opc{1}; 4290 let Inst{12-10} = Pg; 4291 let Inst{9-5} = Zn; 4292 let Inst{4} = opc{0}; 4293 let Inst{3-0} = Pd; 4294 4295 let Defs = [NZCV]; 4296} 4297 4298multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4299 CondCode commuted_cc> { 4300 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4301 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4302 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4303 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4304 4305 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4306 !cast<Instruction>(NAME # _B)>; 4307 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4308 !cast<Instruction>(NAME # _H)>; 4309 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4310 !cast<Instruction>(NAME # _S)>; 4311 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4312 !cast<Instruction>(NAME # _D)>; 4313} 4314 4315 4316//===----------------------------------------------------------------------===// 4317// SVE Integer Compare - Scalars Group 4318//===----------------------------------------------------------------------===// 4319 4320class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4321: I<(outs), (ins rt:$Rn, rt:$Rm), 4322 asm, "\t$Rn, $Rm", 4323 "", 4324 []>, Sched<[]> { 4325 bits<5> Rm; 4326 bits<5> Rn; 4327 let Inst{31-23} = 0b001001011; 4328 let Inst{22} = sz; 4329 let Inst{21} = 0b1; 4330 let Inst{20-16} = Rm; 4331 let Inst{15-10} = 0b001000; 4332 let Inst{9-5} = Rn; 4333 let Inst{4} = opc; 4334 let Inst{3-0} = 0b0000; 4335 4336 let Defs = [NZCV]; 4337} 4338 4339class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4340 RegisterClass gprty, PPRRegOp pprty, 4341 ValueType vt, SDPatternOperator op> 4342: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4343 asm, "\t$Pd, $Rn, $Rm", 4344 "", []>, Sched<[]> { 4345 bits<4> Pd; 4346 bits<5> Rm; 4347 bits<5> Rn; 4348 let Inst{31-24} = 0b00100101; 4349 let Inst{23-22} = sz8_64; 4350 let Inst{21} = 0b1; 4351 let Inst{20-16} = Rm; 4352 let Inst{15-13} = 0b000; 4353 let Inst{12-10} = opc{3-1}; 4354 let Inst{9-5} = Rn; 4355 let Inst{4} = opc{0}; 4356 let Inst{3-0} = Pd; 4357 4358 let Defs = [NZCV]; 4359} 4360 4361multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4362 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8, nxv16i1, op>; 4363 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16, nxv8i1, op>; 4364 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32, nxv4i1, op>; 4365 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64, nxv2i1, op>; 4366 4367 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4368 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4369 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4370 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4371} 4372 4373multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4374 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8, nxv16i1, op>; 4375 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16, nxv8i1, op>; 4376 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32, nxv4i1, op>; 4377 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64, nxv2i1, op>; 4378 4379 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 4380 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 4381 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 4382 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4383} 4384 4385class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 4386 PPRRegOp pprty> 4387: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 4388 asm, "\t$Pd, $Rn, $Rm", 4389 "", []>, Sched<[]> { 4390 bits<4> Pd; 4391 bits<5> Rm; 4392 bits<5> Rn; 4393 let Inst{31-24} = 0b00100101; 4394 let Inst{23-22} = sz8_64; 4395 let Inst{21} = 0b1; 4396 let Inst{20-16} = Rm; 4397 let Inst{15-10} = 0b001100; 4398 let Inst{9-5} = Rn; 4399 let Inst{4} = rw; 4400 let Inst{3-0} = Pd; 4401 4402 let Defs = [NZCV]; 4403} 4404 4405multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 4406 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 4407 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 4408 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 4409 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 4410 4411 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 4412 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 4413 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 4414 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 4415 4416} 4417 4418//===----------------------------------------------------------------------===// 4419// SVE Floating Point Fast Reduction Group 4420//===----------------------------------------------------------------------===// 4421 4422class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 4423 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4424: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 4425 asm, "\t$Vd, $Pg, $Zn", 4426 "", 4427 []>, Sched<[]> { 4428 bits<5> Zn; 4429 bits<5> Vd; 4430 bits<3> Pg; 4431 let Inst{31-24} = 0b01100101; 4432 let Inst{23-22} = sz; 4433 let Inst{21-19} = 0b000; 4434 let Inst{18-16} = opc; 4435 let Inst{15-13} = 0b001; 4436 let Inst{12-10} = Pg; 4437 let Inst{9-5} = Zn; 4438 let Inst{4-0} = Vd; 4439} 4440 4441multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 4442 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 4443 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 4444 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 4445 4446 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4447 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4448 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4449} 4450 4451 4452//===----------------------------------------------------------------------===// 4453// SVE Floating Point Accumulating Reduction Group 4454//===----------------------------------------------------------------------===// 4455 4456class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 4457 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4458: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 4459 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 4460 "", 4461 []>, 4462 Sched<[]> { 4463 bits<3> Pg; 4464 bits<5> Vdn; 4465 bits<5> Zm; 4466 let Inst{31-24} = 0b01100101; 4467 let Inst{23-22} = sz; 4468 let Inst{21-19} = 0b011; 4469 let Inst{18-16} = opc; 4470 let Inst{15-13} = 0b001; 4471 let Inst{12-10} = Pg; 4472 let Inst{9-5} = Zm; 4473 let Inst{4-0} = Vdn; 4474 4475 let Constraints = "$Vdn = $_Vdn"; 4476} 4477 4478multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 4479 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 4480 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 4481 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 4482 4483 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4484 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4485 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4486} 4487 4488//===----------------------------------------------------------------------===// 4489// SVE Floating Point Compare - Vectors Group 4490//===----------------------------------------------------------------------===// 4491 4492class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4493 ZPRRegOp zprty> 4494: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 4495 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4496 "", 4497 []>, Sched<[]> { 4498 bits<4> Pd; 4499 bits<3> Pg; 4500 bits<5> Zm; 4501 bits<5> Zn; 4502 let Inst{31-24} = 0b01100101; 4503 let Inst{23-22} = sz; 4504 let Inst{21} = 0b0; 4505 let Inst{20-16} = Zm; 4506 let Inst{15} = opc{2}; 4507 let Inst{14} = 0b1; 4508 let Inst{13} = opc{1}; 4509 let Inst{12-10} = Pg; 4510 let Inst{9-5} = Zn; 4511 let Inst{4} = opc{0}; 4512 let Inst{3-0} = Pd; 4513} 4514 4515multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 4516 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4517 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4518 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4519 4520 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4521 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4522 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4523} 4524 4525multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, SDPatternOperator op, 4526 SDPatternOperator op_nopred> 4527: sve_fp_3op_p_pd<opc, asm, op> { 4528 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8f16, nxv8f16, 4529 !cast<Instruction>(NAME # _H), PTRUE_H>; 4530 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f16, nxv4f16, 4531 !cast<Instruction>(NAME # _H), PTRUE_S>; 4532 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f16, nxv2f16, 4533 !cast<Instruction>(NAME # _H), PTRUE_D>; 4534 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f32, nxv4f32, 4535 !cast<Instruction>(NAME # _S), PTRUE_S>; 4536 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f32, nxv2f32, 4537 !cast<Instruction>(NAME # _S), PTRUE_D>; 4538 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f64, nxv2f64, 4539 !cast<Instruction>(NAME # _D), PTRUE_D>; 4540} 4541 4542//===----------------------------------------------------------------------===// 4543// SVE Floating Point Compare - with Zero Group 4544//===----------------------------------------------------------------------===// 4545 4546class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4547 ZPRRegOp zprty> 4548: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 4549 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 4550 "", 4551 []>, Sched<[]> { 4552 bits<4> Pd; 4553 bits<3> Pg; 4554 bits<5> Zn; 4555 let Inst{31-24} = 0b01100101; 4556 let Inst{23-22} = sz; 4557 let Inst{21-18} = 0b0100; 4558 let Inst{17-16} = opc{2-1}; 4559 let Inst{15-13} = 0b001; 4560 let Inst{12-10} = Pg; 4561 let Inst{9-5} = Zn; 4562 let Inst{4} = opc{0}; 4563 let Inst{3-0} = Pd; 4564} 4565 4566multiclass sve_fp_2op_p_pd<bits<3> opc, string asm> { 4567 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4568 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4569 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4570} 4571 4572 4573//===----------------------------------------------------------------------===// 4574//SVE Index Generation Group 4575//===----------------------------------------------------------------------===// 4576 4577class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4578 Operand imm_ty> 4579: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 4580 asm, "\t$Zd, $imm5, $imm5b", 4581 "", []>, Sched<[]> { 4582 bits<5> Zd; 4583 bits<5> imm5; 4584 bits<5> imm5b; 4585 let Inst{31-24} = 0b00000100; 4586 let Inst{23-22} = sz8_64; 4587 let Inst{21} = 0b1; 4588 let Inst{20-16} = imm5b; 4589 let Inst{15-10} = 0b010000; 4590 let Inst{9-5} = imm5; 4591 let Inst{4-0} = Zd; 4592} 4593 4594multiclass sve_int_index_ii<string asm, SDPatternOperator op> { 4595 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 4596 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 4597 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 4598 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 4599 4600 def : Pat<(nxv16i8 (op simm5_8b:$imm5, simm5_8b:$imm5b)), 4601 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, simm5_8b:$imm5b)>; 4602 def : Pat<(nxv8i16 (op simm5_16b:$imm5, simm5_16b:$imm5b)), 4603 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, simm5_16b:$imm5b)>; 4604 def : Pat<(nxv4i32 (op simm5_32b:$imm5, simm5_32b:$imm5b)), 4605 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 4606 def : Pat<(nxv2i64 (op simm5_64b:$imm5, simm5_64b:$imm5b)), 4607 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 4608} 4609 4610class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4611 RegisterClass srcRegType, Operand imm_ty> 4612: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 4613 asm, "\t$Zd, $imm5, $Rm", 4614 "", []>, Sched<[]> { 4615 bits<5> Rm; 4616 bits<5> Zd; 4617 bits<5> imm5; 4618 let Inst{31-24} = 0b00000100; 4619 let Inst{23-22} = sz8_64; 4620 let Inst{21} = 0b1; 4621 let Inst{20-16} = Rm; 4622 let Inst{15-10} = 0b010010; 4623 let Inst{9-5} = imm5; 4624 let Inst{4-0} = Zd; 4625} 4626 4627multiclass sve_int_index_ir<string asm, SDPatternOperator op> { 4628 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 4629 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 4630 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 4631 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 4632 4633 def : Pat<(nxv16i8 (op simm5_8b:$imm5, GPR32:$Rm)), 4634 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 4635 def : Pat<(nxv8i16 (op simm5_16b:$imm5, GPR32:$Rm)), 4636 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 4637 def : Pat<(nxv4i32 (op simm5_32b:$imm5, GPR32:$Rm)), 4638 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 4639 def : Pat<(nxv2i64 (op simm5_64b:$imm5, GPR64:$Rm)), 4640 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 4641} 4642 4643class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4644 RegisterClass srcRegType, Operand imm_ty> 4645: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 4646 asm, "\t$Zd, $Rn, $imm5", 4647 "", []>, Sched<[]> { 4648 bits<5> Rn; 4649 bits<5> Zd; 4650 bits<5> imm5; 4651 let Inst{31-24} = 0b00000100; 4652 let Inst{23-22} = sz8_64; 4653 let Inst{21} = 0b1; 4654 let Inst{20-16} = imm5; 4655 let Inst{15-10} = 0b010001; 4656 let Inst{9-5} = Rn; 4657 let Inst{4-0} = Zd; 4658} 4659 4660multiclass sve_int_index_ri<string asm, SDPatternOperator op> { 4661 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 4662 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 4663 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 4664 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 4665 4666 def : Pat<(nxv16i8 (op GPR32:$Rm, simm5_8b:$imm5)), 4667 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, simm5_8b:$imm5)>; 4668 def : Pat<(nxv8i16 (op GPR32:$Rm, simm5_16b:$imm5)), 4669 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, simm5_16b:$imm5)>; 4670 def : Pat<(nxv4i32 (op GPR32:$Rm, simm5_32b:$imm5)), 4671 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 4672 def : Pat<(nxv2i64 (op GPR64:$Rm, simm5_64b:$imm5)), 4673 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 4674} 4675 4676class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4677 RegisterClass srcRegType> 4678: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 4679 asm, "\t$Zd, $Rn, $Rm", 4680 "", []>, Sched<[]> { 4681 bits<5> Zd; 4682 bits<5> Rm; 4683 bits<5> Rn; 4684 let Inst{31-24} = 0b00000100; 4685 let Inst{23-22} = sz8_64; 4686 let Inst{21} = 0b1; 4687 let Inst{20-16} = Rm; 4688 let Inst{15-10} = 0b010011; 4689 let Inst{9-5} = Rn; 4690 let Inst{4-0} = Zd; 4691} 4692 4693multiclass sve_int_index_rr<string asm, SDPatternOperator op> { 4694 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 4695 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 4696 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 4697 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 4698 4699 def : SVE_2_Op_Pat<nxv16i8, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4700 def : SVE_2_Op_Pat<nxv8i16, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4701 def : SVE_2_Op_Pat<nxv4i32, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4702 def : SVE_2_Op_Pat<nxv2i64, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4703} 4704// 4705//===----------------------------------------------------------------------===// 4706// SVE Bitwise Shift - Predicated Group 4707//===----------------------------------------------------------------------===// 4708class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 4709 ZPRRegOp zprty, Operand immtype> 4710: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 4711 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 4712 "", 4713 []>, Sched<[]> { 4714 bits<3> Pg; 4715 bits<5> Zdn; 4716 bits<6> imm; 4717 let Inst{31-24} = 0b00000100; 4718 let Inst{23-22} = tsz8_64{3-2}; 4719 let Inst{21-20} = 0b00; 4720 let Inst{19-16} = opc; 4721 let Inst{15-13} = 0b100; 4722 let Inst{12-10} = Pg; 4723 let Inst{9-8} = tsz8_64{1-0}; 4724 let Inst{7-5} = imm{2-0}; // imm3 4725 let Inst{4-0} = Zdn; 4726 4727 let Constraints = "$Zdn = $_Zdn"; 4728 let DestructiveInstType = DestructiveBinaryImm; 4729 let ElementSize = zprty.ElementSize; 4730} 4731 4732multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string psName=""> { 4733 def _B : SVEPseudo2Instr<psName # _B, 1>, 4734 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 4735 def _H : SVEPseudo2Instr<psName # _H, 1>, 4736 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 4737 let Inst{8} = imm{3}; 4738 } 4739 def _S : SVEPseudo2Instr<psName # _S, 1>, 4740 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 4741 let Inst{9-8} = imm{4-3}; 4742 } 4743 def _D : SVEPseudo2Instr<psName # _D, 1>, 4744 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 4745 let Inst{22} = imm{5}; 4746 let Inst{9-8} = imm{4-3}; 4747 } 4748} 4749 4750multiclass sve2_int_bin_pred_shift_imm_left<bits<4> opc, string asm, 4751 string psName, 4752 SDPatternOperator op> { 4753 4754 def _B : SVEPseudo2Instr<psName # _B, 1>, sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 4755 def _H : SVEPseudo2Instr<psName # _H, 1>, 4756 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 4757 let Inst{8} = imm{3}; 4758 } 4759 def _S : SVEPseudo2Instr<psName # _S, 1>, 4760 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 4761 let Inst{9-8} = imm{4-3}; 4762 } 4763 def _D : SVEPseudo2Instr<psName # _D, 1>, 4764 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 4765 let Inst{22} = imm{5}; 4766 let Inst{9-8} = imm{4-3}; 4767 } 4768 4769 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 4770 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 4771 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 4772 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 4773} 4774 4775multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 4776 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 4777 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 4778 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 4779 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 4780 4781 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 4782 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 4783 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 4784 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 4785} 4786 4787multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 4788 SDPatternOperator op = null_frag> { 4789 def _B : SVEPseudo2Instr<Ps # _B, 1>, 4790 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 4791 def _H : SVEPseudo2Instr<Ps # _H, 1>, 4792 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 4793 let Inst{8} = imm{3}; 4794 } 4795 def _S : SVEPseudo2Instr<Ps # _S, 1>, 4796 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 4797 let Inst{9-8} = imm{4-3}; 4798 } 4799 def _D : SVEPseudo2Instr<Ps # _D, 1>, 4800 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 4801 let Inst{22} = imm{5}; 4802 let Inst{9-8} = imm{4-3}; 4803 } 4804 4805 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4806 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4807 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4808 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4809} 4810 4811multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 4812 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 4813 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 4814 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 4815 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 4816 4817 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 4818 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 4819 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 4820 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 4821} 4822 4823class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 4824 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 4825: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 4826 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 4827 "", 4828 []>, Sched<[]> { 4829 bits<3> Pg; 4830 bits<5> Zdn; 4831 bits<5> Zm; 4832 let Inst{31-24} = 0b00000100; 4833 let Inst{23-22} = sz8_64; 4834 let Inst{21-20} = 0b01; 4835 let Inst{19} = wide; 4836 let Inst{18-16} = opc; 4837 let Inst{15-13} = 0b100; 4838 let Inst{12-10} = Pg; 4839 let Inst{9-5} = Zm; 4840 let Inst{4-0} = Zdn; 4841 4842 let Constraints = "$Zdn = $_Zdn"; 4843 let DestructiveInstType = DestructiveOther; 4844 let ElementSize = zprty.ElementSize; 4845} 4846 4847multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 4848 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 4849 let DestructiveInstType = DestructiveBinaryCommWithRev in { 4850 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 4851 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 4852 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 4853 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 4854 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 4855 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 4856 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 4857 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 4858 } 4859 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 4860 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 4861 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 4862 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 4863} 4864 4865multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 4866 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 4867 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 4868 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 4869 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 4870 4871 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 4872 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 4873 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 4874 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 4875} 4876 4877multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 4878 SDPatternOperator op> { 4879 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 4880 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 4881 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 4882 4883 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4884 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4885 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4886} 4887 4888//===----------------------------------------------------------------------===// 4889// SVE Shift - Unpredicated Group 4890//===----------------------------------------------------------------------===// 4891 4892class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 4893 ZPRRegOp zprty> 4894: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 4895 asm, "\t$Zd, $Zn, $Zm", 4896 "", 4897 []>, Sched<[]> { 4898 bits<5> Zd; 4899 bits<5> Zm; 4900 bits<5> Zn; 4901 let Inst{31-24} = 0b00000100; 4902 let Inst{23-22} = sz8_64; 4903 let Inst{21} = 0b1; 4904 let Inst{20-16} = Zm; 4905 let Inst{15-12} = 0b1000; 4906 let Inst{11-10} = opc; 4907 let Inst{9-5} = Zn; 4908 let Inst{4-0} = Zd; 4909} 4910 4911multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm> { 4912 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 4913 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 4914 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 4915} 4916 4917class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 4918 ZPRRegOp zprty, Operand immtype> 4919: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 4920 asm, "\t$Zd, $Zn, $imm", 4921 "", 4922 []>, Sched<[]> { 4923 bits<5> Zd; 4924 bits<5> Zn; 4925 bits<6> imm; 4926 let Inst{31-24} = 0b00000100; 4927 let Inst{23-22} = tsz8_64{3-2}; 4928 let Inst{21} = 0b1; 4929 let Inst{20-19} = tsz8_64{1-0}; 4930 let Inst{18-16} = imm{2-0}; // imm3 4931 let Inst{15-12} = 0b1001; 4932 let Inst{11-10} = opc; 4933 let Inst{9-5} = Zn; 4934 let Inst{4-0} = Zd; 4935} 4936 4937multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 4938 SDPatternOperator op> { 4939 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 4940 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 4941 let Inst{19} = imm{3}; 4942 } 4943 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 4944 let Inst{20-19} = imm{4-3}; 4945 } 4946 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 4947 let Inst{22} = imm{5}; 4948 let Inst{20-19} = imm{4-3}; 4949 } 4950 4951 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, vecshiftL8, !cast<Instruction>(NAME # _B)>; 4952 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, vecshiftL16, !cast<Instruction>(NAME # _H)>; 4953 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, vecshiftL32, !cast<Instruction>(NAME # _S)>; 4954 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEShiftImm64, !cast<Instruction>(NAME # _D)>; 4955} 4956 4957multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 4958 SDPatternOperator op> { 4959 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 4960 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 4961 let Inst{19} = imm{3}; 4962 } 4963 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 4964 let Inst{20-19} = imm{4-3}; 4965 } 4966 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 4967 let Inst{22} = imm{5}; 4968 let Inst{20-19} = imm{4-3}; 4969 } 4970 4971 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, vecshiftR8, !cast<Instruction>(NAME # _B)>; 4972 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, vecshiftR16, !cast<Instruction>(NAME # _H)>; 4973 def : SVE_1_Op_Imm_Shift_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, vecshiftR32, !cast<Instruction>(NAME # _S)>; 4974 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEShiftImm64, !cast<Instruction>(NAME # _D)>; 4975} 4976//===----------------------------------------------------------------------===// 4977// SVE Memory - Store Group 4978//===----------------------------------------------------------------------===// 4979 4980class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 4981 RegisterOperand VecList> 4982: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 4983 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 4984 "", 4985 []>, Sched<[]> { 4986 bits<3> Pg; 4987 bits<5> Rn; 4988 bits<5> Zt; 4989 bits<4> imm4; 4990 let Inst{31-25} = 0b1110010; 4991 let Inst{24-23} = msz; 4992 let Inst{22-21} = esz; 4993 let Inst{20} = 0; 4994 let Inst{19-16} = imm4; 4995 let Inst{15-13} = 0b111; 4996 let Inst{12-10} = Pg; 4997 let Inst{9-5} = Rn; 4998 let Inst{4-0} = Zt; 4999 5000 let mayStore = 1; 5001} 5002 5003multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5004 RegisterOperand listty, ZPRRegOp zprty> 5005{ 5006 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5007 5008 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5009 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5010 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5011 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5012 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5013 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5014} 5015 5016class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5017 string asm, Operand immtype> 5018: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5019 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5020 "", 5021 []>, Sched<[]> { 5022 bits<3> Pg; 5023 bits<5> Rn; 5024 bits<5> Zt; 5025 bits<4> imm4; 5026 let Inst{31-25} = 0b1110010; 5027 let Inst{24-23} = sz; 5028 let Inst{22-21} = nregs; 5029 let Inst{20} = 1; 5030 let Inst{19-16} = imm4; 5031 let Inst{15-13} = 0b111; 5032 let Inst{12-10} = Pg; 5033 let Inst{9-5} = Rn; 5034 let Inst{4-0} = Zt; 5035 5036 let mayStore = 1; 5037} 5038 5039multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5040 string asm, Operand immtype> { 5041 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5042 5043 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5044 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5045} 5046 5047class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5048 string asm, RegisterOperand gprty> 5049: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5050 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5051 "", 5052 []>, Sched<[]> { 5053 bits<3> Pg; 5054 bits<5> Rm; 5055 bits<5> Rn; 5056 bits<5> Zt; 5057 let Inst{31-25} = 0b1110010; 5058 let Inst{24-23} = sz; 5059 let Inst{22-21} = nregs; 5060 let Inst{20-16} = Rm; 5061 let Inst{15-13} = 0b011; 5062 let Inst{12-10} = Pg; 5063 let Inst{9-5} = Rn; 5064 let Inst{4-0} = Zt; 5065 5066 let mayStore = 1; 5067} 5068 5069class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5070 RegisterOperand listty, RegisterOperand gprty> 5071: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5072 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5073 "", 5074 []>, Sched<[]> { 5075 bits<3> Pg; 5076 bits<5> Rm; 5077 bits<5> Rn; 5078 bits<5> Zt; 5079 let Inst{31-25} = 0b1110010; 5080 let Inst{24-21} = dtype; 5081 let Inst{20-16} = Rm; 5082 let Inst{15-13} = 0b010; 5083 let Inst{12-10} = Pg; 5084 let Inst{9-5} = Rn; 5085 let Inst{4-0} = Zt; 5086 5087 let mayStore = 1; 5088} 5089 5090multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5091 RegisterOperand listty, ZPRRegOp zprty, 5092 RegisterOperand gprty> { 5093 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5094 5095 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5096 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5097} 5098 5099class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5100: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5101 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5102 "", 5103 []>, Sched<[]> { 5104 bits<3> Pg; 5105 bits<5> Rn; 5106 bits<5> Zt; 5107 bits<4> imm4; 5108 let Inst{31-25} = 0b1110010; 5109 let Inst{24-23} = msz; 5110 let Inst{22-20} = 0b001; 5111 let Inst{19-16} = imm4; 5112 let Inst{15-13} = 0b111; 5113 let Inst{12-10} = Pg; 5114 let Inst{9-5} = Rn; 5115 let Inst{4-0} = Zt; 5116 5117 let mayStore = 1; 5118} 5119 5120multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5121 ZPRRegOp zprty> { 5122 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5123 5124 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5125 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5126 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5127 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5128 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5129 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5130} 5131 5132class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5133 RegisterOperand gprty> 5134: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5135 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5136 "", 5137 []>, Sched<[]> { 5138 bits<3> Pg; 5139 bits<5> Rm; 5140 bits<5> Rn; 5141 bits<5> Zt; 5142 let Inst{31-25} = 0b1110010; 5143 let Inst{24-23} = msz; 5144 let Inst{22-21} = 0b00; 5145 let Inst{20-16} = Rm; 5146 let Inst{15-13} = 0b011; 5147 let Inst{12-10} = Pg; 5148 let Inst{9-5} = Rn; 5149 let Inst{4-0} = Zt; 5150 5151 let mayStore = 1; 5152} 5153 5154multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5155 ZPRRegOp zprty, RegisterOperand gprty> { 5156 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5157 5158 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5159 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5160} 5161 5162class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5163 RegisterOperand listty, ZPRRegOp zprty> 5164: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5165 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5166 "", 5167 []>, Sched<[]> { 5168 bits<3> Pg; 5169 bits<5> Rm; 5170 bits<5> Zn; 5171 bits<5> Zt; 5172 let Inst{31-25} = 0b1110010; 5173 let Inst{24-22} = opc; 5174 let Inst{21} = 0b0; 5175 let Inst{20-16} = Rm; 5176 let Inst{15-13} = 0b001; 5177 let Inst{12-10} = Pg; 5178 let Inst{9-5} = Zn; 5179 let Inst{4-0} = Zt; 5180 5181 let mayStore = 1; 5182} 5183 5184multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5185 SDPatternOperator op, 5186 ValueType vt> { 5187 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5188 5189 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5190 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5191 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5192 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5193 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5194 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5195 5196 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5197 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5198} 5199 5200multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5201 SDPatternOperator op, 5202 ValueType vt> { 5203 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5204 5205 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5206 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5207 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5208 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5209 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5210 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5211 5212 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5213 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5214} 5215 5216class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5217 RegisterOperand VecList, RegisterOperand zprext> 5218: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5219 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5220 "", 5221 []>, Sched<[]> { 5222 bits<3> Pg; 5223 bits<5> Rn; 5224 bits<5> Zm; 5225 bits<5> Zt; 5226 let Inst{31-25} = 0b1110010; 5227 let Inst{24-22} = opc; 5228 let Inst{21} = scaled; 5229 let Inst{20-16} = Zm; 5230 let Inst{15} = 0b1; 5231 let Inst{14} = xs; 5232 let Inst{13} = 0; 5233 let Inst{12-10} = Pg; 5234 let Inst{9-5} = Rn; 5235 let Inst{4-0} = Zt; 5236 5237 let mayStore = 1; 5238} 5239 5240multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5241 SDPatternOperator sxtw_op, 5242 SDPatternOperator uxtw_op, 5243 RegisterOperand sxtw_opnd, 5244 RegisterOperand uxtw_opnd, 5245 ValueType vt > { 5246 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5247 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5248 5249 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5250 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5251 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5252 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5253 5254 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5255 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5256 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5257 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5258} 5259 5260multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5261 SDPatternOperator sxtw_op, 5262 SDPatternOperator uxtw_op, 5263 RegisterOperand sxtw_opnd, 5264 RegisterOperand uxtw_opnd, 5265 ValueType vt > { 5266 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5267 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5268 5269 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5270 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5271 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5272 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5273 5274 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5275 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5276 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5277 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5278} 5279 5280multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 5281 SDPatternOperator sxtw_op, 5282 SDPatternOperator uxtw_op, 5283 RegisterOperand sxtw_opnd, 5284 RegisterOperand uxtw_opnd, 5285 ValueType vt> { 5286 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 5287 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 5288 5289 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5290 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5291 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5292 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5293 5294 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5295 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5296 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5297 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5298} 5299 5300multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 5301 SDPatternOperator sxtw_op, 5302 SDPatternOperator uxtw_op, 5303 RegisterOperand sxtw_opnd, 5304 RegisterOperand uxtw_opnd, 5305 ValueType vt> { 5306 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 5307 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 5308 5309 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5310 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5311 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5312 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5313 5314 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5315 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5316 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5317 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5318} 5319 5320class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 5321 RegisterOperand zprext> 5322: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5323 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5324 "", 5325 []>, Sched<[]> { 5326 bits<3> Pg; 5327 bits<5> Rn; 5328 bits<5> Zm; 5329 bits<5> Zt; 5330 let Inst{31-25} = 0b1110010; 5331 let Inst{24-23} = msz; 5332 let Inst{22} = 0b0; 5333 let Inst{21} = scaled; 5334 let Inst{20-16} = Zm; 5335 let Inst{15-13} = 0b101; 5336 let Inst{12-10} = Pg; 5337 let Inst{9-5} = Rn; 5338 let Inst{4-0} = Zt; 5339 5340 let mayStore = 1; 5341} 5342 5343multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 5344 SDPatternOperator op, 5345 RegisterOperand zprext, 5346 ValueType vt> { 5347 def _SCALED_REAL : sve_mem_sst_sv2<msz, 1, asm, zprext>; 5348 5349 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5350 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 5351 5352 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 5353 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 5354} 5355 5356multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 5357 SDPatternOperator op, 5358 ValueType vt> { 5359 def _REAL : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 5360 5361 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5362 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 5363 5364 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5365 (!cast<Instruction>(NAME # _REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5366} 5367 5368class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 5369 RegisterOperand VecList, Operand imm_ty> 5370: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 5371 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 5372 "", 5373 []>, Sched<[]> { 5374 bits<3> Pg; 5375 bits<5> imm5; 5376 bits<5> Zn; 5377 bits<5> Zt; 5378 let Inst{31-25} = 0b1110010; 5379 let Inst{24-23} = opc{2-1}; 5380 let Inst{22} = 0b1; 5381 let Inst{21} = opc{0}; 5382 let Inst{20-16} = imm5; 5383 let Inst{15-13} = 0b101; 5384 let Inst{12-10} = Pg; 5385 let Inst{9-5} = Zn; 5386 let Inst{4-0} = Zt; 5387 5388 let mayStore = 1; 5389} 5390 5391multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 5392 Operand imm_ty, 5393 SDPatternOperator op, 5394 ValueType vt> { 5395 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 5396 5397 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5398 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 5399 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5400 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 5401 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5402 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 5403 5404 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 5405 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5406} 5407 5408multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 5409 Operand imm_ty, 5410 SDPatternOperator op, 5411 ValueType vt> { 5412 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 5413 5414 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5415 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 5416 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5417 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 5418 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5419 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 5420 5421 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 5422 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5423} 5424 5425class sve_mem_z_spill<string asm> 5426: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 5427 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 5428 "", 5429 []>, Sched<[]> { 5430 bits<5> Rn; 5431 bits<5> Zt; 5432 bits<9> imm9; 5433 let Inst{31-22} = 0b1110010110; 5434 let Inst{21-16} = imm9{8-3}; 5435 let Inst{15-13} = 0b010; 5436 let Inst{12-10} = imm9{2-0}; 5437 let Inst{9-5} = Rn; 5438 let Inst{4-0} = Zt; 5439 5440 let mayStore = 1; 5441} 5442 5443multiclass sve_mem_z_spill<string asm> { 5444 def NAME : sve_mem_z_spill<asm>; 5445 5446 def : InstAlias<asm # "\t$Zt, [$Rn]", 5447 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 5448} 5449 5450class sve_mem_p_spill<string asm> 5451: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 5452 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 5453 "", 5454 []>, Sched<[]> { 5455 bits<4> Pt; 5456 bits<5> Rn; 5457 bits<9> imm9; 5458 let Inst{31-22} = 0b1110010110; 5459 let Inst{21-16} = imm9{8-3}; 5460 let Inst{15-13} = 0b000; 5461 let Inst{12-10} = imm9{2-0}; 5462 let Inst{9-5} = Rn; 5463 let Inst{4} = 0b0; 5464 let Inst{3-0} = Pt; 5465 5466 let mayStore = 1; 5467} 5468 5469multiclass sve_mem_p_spill<string asm> { 5470 def NAME : sve_mem_p_spill<asm>; 5471 5472 def : InstAlias<asm # "\t$Pt, [$Rn]", 5473 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 5474} 5475 5476//===----------------------------------------------------------------------===// 5477// SVE Permute - Predicates Group 5478//===----------------------------------------------------------------------===// 5479 5480class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 5481 PPRRegOp pprty> 5482: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 5483 asm, "\t$Pd, $Pn, $Pm", 5484 "", 5485 []>, Sched<[]> { 5486 bits<4> Pd; 5487 bits<4> Pm; 5488 bits<4> Pn; 5489 let Inst{31-24} = 0b00000101; 5490 let Inst{23-22} = sz8_64; 5491 let Inst{21-20} = 0b10; 5492 let Inst{19-16} = Pm; 5493 let Inst{15-13} = 0b010; 5494 let Inst{12-10} = opc; 5495 let Inst{9} = 0b0; 5496 let Inst{8-5} = Pn; 5497 let Inst{4} = 0b0; 5498 let Inst{3-0} = Pd; 5499} 5500 5501multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 5502 SDPatternOperator op> { 5503 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 5504 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 5505 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 5506 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 5507 5508 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 5509 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 5510 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 5511 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 5512} 5513 5514class sve_int_perm_punpk<bit opc, string asm> 5515: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 5516 asm, "\t$Pd, $Pn", 5517 "", 5518 []>, Sched<[]> { 5519 bits<4> Pd; 5520 bits<4> Pn; 5521 let Inst{31-17} = 0b000001010011000; 5522 let Inst{16} = opc; 5523 let Inst{15-9} = 0b0100000; 5524 let Inst{8-5} = Pn; 5525 let Inst{4} = 0b0; 5526 let Inst{3-0} = Pd; 5527} 5528 5529multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 5530 def NAME : sve_int_perm_punpk<opc, asm>; 5531 5532 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 5533 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 5534 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 5535} 5536 5537class sve_int_rdffr_pred<bit s, string asm> 5538: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 5539 asm, "\t$Pd, $Pg/z", 5540 "", 5541 []>, Sched<[]> { 5542 bits<4> Pd; 5543 bits<4> Pg; 5544 let Inst{31-23} = 0b001001010; 5545 let Inst{22} = s; 5546 let Inst{21-9} = 0b0110001111000; 5547 let Inst{8-5} = Pg; 5548 let Inst{4} = 0; 5549 let Inst{3-0} = Pd; 5550 5551 let Defs = !if(!eq (s, 1), [NZCV], []); 5552 let Uses = [FFR]; 5553} 5554 5555multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 5556 def _REAL : sve_int_rdffr_pred<s, asm>; 5557 5558 // We need a layer of indirection because early machine code passes balk at 5559 // physical register (i.e. FFR) uses that have no previous definition. 5560 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 5561 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 5562 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 5563 } 5564} 5565 5566class sve_int_rdffr_unpred<string asm> : I< 5567 (outs PPR8:$Pd), (ins), 5568 asm, "\t$Pd", 5569 "", 5570 []>, Sched<[]> { 5571 bits<4> Pd; 5572 let Inst{31-4} = 0b0010010100011001111100000000; 5573 let Inst{3-0} = Pd; 5574 5575 let Uses = [FFR]; 5576} 5577 5578multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 5579 def _REAL : sve_int_rdffr_unpred<asm>; 5580 5581 // We need a layer of indirection because early machine code passes balk at 5582 // physical register (i.e. FFR) uses that have no previous definition. 5583 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 5584 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 5585 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 5586 } 5587} 5588 5589class sve_int_wrffr<string asm, SDPatternOperator op> 5590: I<(outs), (ins PPR8:$Pn), 5591 asm, "\t$Pn", 5592 "", 5593 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 5594 bits<4> Pn; 5595 let Inst{31-9} = 0b00100101001010001001000; 5596 let Inst{8-5} = Pn; 5597 let Inst{4-0} = 0b00000; 5598 5599 let hasSideEffects = 1; 5600 let Defs = [FFR]; 5601} 5602 5603class sve_int_setffr<string asm, SDPatternOperator op> 5604: I<(outs), (ins), 5605 asm, "", 5606 "", 5607 [(op)]>, Sched<[]> { 5608 let Inst{31-0} = 0b00100101001011001001000000000000; 5609 5610 let hasSideEffects = 1; 5611 let Defs = [FFR]; 5612} 5613 5614//===----------------------------------------------------------------------===// 5615// SVE Permute Vector - Predicated Group 5616//===----------------------------------------------------------------------===// 5617 5618class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 5619 ZPRRegOp zprty, RegisterClass rt> 5620: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 5621 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 5622 "", 5623 []>, Sched<[]> { 5624 bits<3> Pg; 5625 bits<5> Rdn; 5626 bits<5> Zm; 5627 let Inst{31-24} = 0b00000101; 5628 let Inst{23-22} = sz8_64; 5629 let Inst{21-17} = 0b11000; 5630 let Inst{16} = ab; 5631 let Inst{15-13} = 0b101; 5632 let Inst{12-10} = Pg; 5633 let Inst{9-5} = Zm; 5634 let Inst{4-0} = Rdn; 5635 5636 let Constraints = "$Rdn = $_Rdn"; 5637} 5638 5639multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 5640 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 5641 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 5642 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 5643 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 5644 5645 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 5646 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 5647 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5648 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5649} 5650 5651class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 5652 ZPRRegOp zprty, RegisterClass rt> 5653: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 5654 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5655 "", 5656 []>, Sched<[]> { 5657 bits<3> Pg; 5658 bits<5> Vdn; 5659 bits<5> Zm; 5660 let Inst{31-24} = 0b00000101; 5661 let Inst{23-22} = sz8_64; 5662 let Inst{21-17} = 0b10101; 5663 let Inst{16} = ab; 5664 let Inst{15-13} = 0b100; 5665 let Inst{12-10} = Pg; 5666 let Inst{9-5} = Zm; 5667 let Inst{4-0} = Vdn; 5668 5669 let Constraints = "$Vdn = $_Vdn"; 5670} 5671 5672multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 5673 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 5674 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 5675 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 5676 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 5677 5678 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5679 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5680 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5681} 5682 5683class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 5684 ZPRRegOp zprty> 5685: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 5686 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 5687 "", 5688 []>, Sched<[]> { 5689 bits<3> Pg; 5690 bits<5> Zdn; 5691 bits<5> Zm; 5692 let Inst{31-24} = 0b00000101; 5693 let Inst{23-22} = sz8_64; 5694 let Inst{21-17} = 0b10100; 5695 let Inst{16} = ab; 5696 let Inst{15-13} = 0b100; 5697 let Inst{12-10} = Pg; 5698 let Inst{9-5} = Zm; 5699 let Inst{4-0} = Zdn; 5700 5701 let Constraints = "$Zdn = $_Zdn"; 5702 let DestructiveInstType = DestructiveOther; 5703 let ElementSize = ElementSizeNone; 5704} 5705 5706multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 5707 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 5708 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 5709 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 5710 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 5711 5712 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5713 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5714 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5715 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5716 5717 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5718 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5719 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5720} 5721 5722class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 5723 ZPRRegOp zprty, RegisterClass resultRegType> 5724: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 5725 asm, "\t$Rd, $Pg, $Zn", 5726 "", 5727 []>, Sched<[]> { 5728 bits<3> Pg; 5729 bits<5> Rd; 5730 bits<5> Zn; 5731 let Inst{31-24} = 0b00000101; 5732 let Inst{23-22} = sz8_64; 5733 let Inst{21-17} = 0b10000; 5734 let Inst{16} = ab; 5735 let Inst{15-13} = 0b101; 5736 let Inst{12-10} = Pg; 5737 let Inst{9-5} = Zn; 5738 let Inst{4-0} = Rd; 5739} 5740 5741multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 5742 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 5743 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 5744 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 5745 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 5746 5747 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 5748 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 5749 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 5750 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5751} 5752 5753class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 5754 ZPRRegOp zprty, RegisterClass dstRegtype> 5755: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5756 asm, "\t$Vd, $Pg, $Zn", 5757 "", 5758 []>, Sched<[]> { 5759 bits<3> Pg; 5760 bits<5> Vd; 5761 bits<5> Zn; 5762 let Inst{31-24} = 0b00000101; 5763 let Inst{23-22} = sz8_64; 5764 let Inst{21-17} = 0b10001; 5765 let Inst{16} = ab; 5766 let Inst{15-13} = 0b100; 5767 let Inst{12-10} = Pg; 5768 let Inst{9-5} = Zn; 5769 let Inst{4-0} = Vd; 5770} 5771 5772multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 5773 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 5774 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 5775 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 5776 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 5777 5778 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5779 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5780 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5781 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5782} 5783 5784class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 5785: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 5786 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 5787 "", 5788 []>, Sched<[]> { 5789 bits<3> Pg; 5790 bits<5> Zdn; 5791 bits<5> Zm; 5792 let Inst{31-24} = 0b00000101; 5793 let Inst{23-22} = sz8_64; 5794 let Inst{21-13} = 0b101100100; 5795 let Inst{12-10} = Pg; 5796 let Inst{9-5} = Zm; 5797 let Inst{4-0} = Zdn; 5798 5799 let Constraints = "$Zdn = $_Zdn"; 5800 let DestructiveInstType = DestructiveOther; 5801 let ElementSize = ElementSizeNone; 5802} 5803 5804multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 5805 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 5806 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 5807 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 5808 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 5809 5810 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5811 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5812 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5813 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5814 5815 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5816 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5817 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5818} 5819 5820class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 5821 ZPRRegOp zprty, RegisterOperand VecList> 5822: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 5823 asm, "\t$Zd, $Pg, $Zn", 5824 "", 5825 []>, Sched<[]> { 5826 bits<3> Pg; 5827 bits<5> Zn; 5828 bits<5> Zd; 5829 let Inst{31-24} = 0b00000101; 5830 let Inst{23-22} = sz8_64; 5831 let Inst{21-13} = 0b101101100; 5832 let Inst{12-10} = Pg; 5833 let Inst{9-5} = Zn; 5834 let Inst{4-0} = Zd; 5835} 5836 5837multiclass sve2_int_perm_splice_cons<string asm> { 5838 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 5839 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 5840 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 5841 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 5842} 5843 5844class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 5845 ZPRRegOp zprty> 5846: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 5847 asm, "\t$Zd, $Pg/m, $Zn", 5848 "", 5849 []>, Sched<[]> { 5850 bits<5> Zd; 5851 bits<3> Pg; 5852 bits<5> Zn; 5853 let Inst{31-24} = 0b00000101; 5854 let Inst{23-22} = sz8_64; 5855 let Inst{21-18} = 0b1001; 5856 let Inst{17-16} = opc; 5857 let Inst{15-13} = 0b100; 5858 let Inst{12-10} = Pg; 5859 let Inst{9-5} = Zn; 5860 let Inst{4-0} = Zd; 5861 5862 let Constraints = "$Zd = $_Zd"; 5863 let DestructiveInstType = DestructiveOther; 5864 let ElementSize = zprty.ElementSize; 5865} 5866 5867multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 5868 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 5869 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 5870 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 5871 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 5872 5873 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 5874 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 5875 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 5876 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5877} 5878 5879multiclass sve_int_perm_rev_revb<string asm, 5880 SDPatternOperator int_op, 5881 SDPatternOperator ir_op> { 5882 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 5883 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 5884 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 5885 5886 def : SVE_3_Op_Pat<nxv8i16, int_op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 5887 def : SVE_3_Op_Pat<nxv4i32, int_op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 5888 def : SVE_3_Op_Pat<nxv2i64, int_op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5889 5890 def : SVE_1_Op_AllActive_Pat<nxv8i16, ir_op, nxv8i16, !cast<Instruction>(NAME # _H), PTRUE_H>; 5891 def : SVE_1_Op_AllActive_Pat<nxv4i32, ir_op, nxv4i32, !cast<Instruction>(NAME # _S), PTRUE_S>; 5892 def : SVE_1_Op_AllActive_Pat<nxv2i64, ir_op, nxv2i64, !cast<Instruction>(NAME # _D), PTRUE_D>; 5893} 5894 5895multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 5896 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 5897 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 5898 5899 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 5900 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5901} 5902 5903multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 5904 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 5905 5906 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5907} 5908 5909class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5910 RegisterClass srcRegType> 5911: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 5912 asm, "\t$Zd, $Pg/m, $Rn", 5913 "", 5914 []>, Sched<[]> { 5915 bits<3> Pg; 5916 bits<5> Rn; 5917 bits<5> Zd; 5918 let Inst{31-24} = 0b00000101; 5919 let Inst{23-22} = sz8_64; 5920 let Inst{21-13} = 0b101000101; 5921 let Inst{12-10} = Pg; 5922 let Inst{9-5} = Rn; 5923 let Inst{4-0} = Zd; 5924 5925 let Constraints = "$Zd = $_Zd"; 5926 let DestructiveInstType = DestructiveOther; 5927 let ElementSize = zprty.ElementSize; 5928} 5929 5930multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 5931 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 5932 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 5933 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 5934 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 5935 5936 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 5937 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 5938 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 5939 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 5940 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 5941 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 5942 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 5943 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 5944 5945 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 5946 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 5947 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 5948 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 5949 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 5950 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 5951 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 5952 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 5953} 5954 5955class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5956 RegisterClass srcRegtype> 5957: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 5958 asm, "\t$Zd, $Pg/m, $Vn", 5959 "", 5960 []>, Sched<[]> { 5961 bits<3> Pg; 5962 bits<5> Vn; 5963 bits<5> Zd; 5964 let Inst{31-24} = 0b00000101; 5965 let Inst{23-22} = sz8_64; 5966 let Inst{21-13} = 0b100000100; 5967 let Inst{12-10} = Pg; 5968 let Inst{9-5} = Vn; 5969 let Inst{4-0} = Zd; 5970 5971 let Constraints = "$Zd = $_Zd"; 5972 let DestructiveInstType = DestructiveOther; 5973 let ElementSize = zprty.ElementSize; 5974} 5975 5976multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 5977 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 5978 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 5979 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 5980 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 5981 5982 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 5983 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 5984 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 5985 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 5986 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 5987 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 5988 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 5989 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 5990 5991 5992 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 5993 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 5994 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 5995 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 5996 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 5997 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 5998 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 5999 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6000} 6001 6002class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6003: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6004 asm, "\t$Zd, $Pg, $Zn", 6005 "", 6006 []>, Sched<[]> { 6007 bits<3> Pg; 6008 bits<5> Zd; 6009 bits<5> Zn; 6010 let Inst{31-23} = 0b000001011; 6011 let Inst{22} = sz; 6012 let Inst{21-13} = 0b100001100; 6013 let Inst{12-10} = Pg; 6014 let Inst{9-5} = Zn; 6015 let Inst{4-0} = Zd; 6016} 6017 6018multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6019 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6020 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6021 6022 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6023 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6024 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6025 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6026} 6027 6028 6029//===----------------------------------------------------------------------===// 6030// SVE Memory - Contiguous Load Group 6031//===----------------------------------------------------------------------===// 6032 6033class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6034 RegisterOperand VecList> 6035: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6036 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6037 "", 6038 []>, Sched<[]> { 6039 bits<3> Pg; 6040 bits<5> Rn; 6041 bits<5> Zt; 6042 bits<4> imm4; 6043 let Inst{31-25} = 0b1010010; 6044 let Inst{24-21} = dtype; 6045 let Inst{20} = nf; 6046 let Inst{19-16} = imm4; 6047 let Inst{15-13} = 0b101; 6048 let Inst{12-10} = Pg; 6049 let Inst{9-5} = Rn; 6050 let Inst{4-0} = Zt; 6051 6052 let mayLoad = 1; 6053 let Uses = !if(!eq(nf, 1), [FFR], []); 6054 let Defs = !if(!eq(nf, 1), [FFR], []); 6055} 6056 6057multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6058 RegisterOperand listty, ZPRRegOp zprty> { 6059 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6060 6061 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6062 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6063 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6064 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6065 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6066 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6067 6068 // We need a layer of indirection because early machine code passes balk at 6069 // physical register (i.e. FFR) uses that have no previous definition. 6070 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6071 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6072 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6073 } 6074} 6075 6076multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6077 ZPRRegOp zprty> 6078: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6079 6080class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6081: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6082 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6083 "", 6084 []>, Sched<[]> { 6085 bits<5> Zt; 6086 bits<3> Pg; 6087 bits<5> Rn; 6088 bits<4> imm4; 6089 let Inst{31-25} = 0b1010010; 6090 let Inst{24-23} = msz; 6091 let Inst{22-20} = 0b000; 6092 let Inst{19-16} = imm4; 6093 let Inst{15-13} = 0b111; 6094 let Inst{12-10} = Pg; 6095 let Inst{9-5} = Rn; 6096 let Inst{4-0} = Zt; 6097 6098 let mayLoad = 1; 6099} 6100 6101multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6102 ZPRRegOp zprty> { 6103 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6104 6105 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6106 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6107 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6108 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6109 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6110 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6111} 6112 6113class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6114 RegisterOperand gprty> 6115: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6116 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6117 "", 6118 []>, Sched<[]> { 6119 bits<3> Pg; 6120 bits<5> Rm; 6121 bits<5> Rn; 6122 bits<5> Zt; 6123 let Inst{31-25} = 0b1010010; 6124 let Inst{24-23} = msz; 6125 let Inst{22-21} = 0b00; 6126 let Inst{20-16} = Rm; 6127 let Inst{15-13} = 0b110; 6128 let Inst{12-10} = Pg; 6129 let Inst{9-5} = Rn; 6130 let Inst{4-0} = Zt; 6131 6132 let mayLoad = 1; 6133} 6134 6135multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6136 ZPRRegOp zprty, RegisterOperand gprty> { 6137 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6138 6139 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6140 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6141} 6142 6143class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6144: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6145 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6146 bits<5> Zt; 6147 bits<5> Rn; 6148 bits<3> Pg; 6149 bits<4> imm4; 6150 let Inst{31-25} = 0b1010010; 6151 let Inst{24-23} = sz; 6152 let Inst{22-20} = 0; 6153 let Inst{19-16} = imm4; 6154 let Inst{15-13} = 0b001; 6155 let Inst{12-10} = Pg; 6156 let Inst{9-5} = Rn; 6157 let Inst{4-0} = Zt; 6158 6159 let mayLoad = 1; 6160} 6161 6162multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6163 ZPRRegOp zprty> { 6164 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6165 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6166 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6167 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6168 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6169 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6170 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6171} 6172 6173class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6174 RegisterOperand gprty> 6175: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6176 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6177 bits<5> Zt; 6178 bits<3> Pg; 6179 bits<5> Rn; 6180 bits<5> Rm; 6181 let Inst{31-25} = 0b1010010; 6182 let Inst{24-23} = sz; 6183 let Inst{22-21} = 0; 6184 let Inst{20-16} = Rm; 6185 let Inst{15-13} = 0; 6186 let Inst{12-10} = Pg; 6187 let Inst{9-5} = Rn; 6188 let Inst{4-0} = Zt; 6189 6190 let mayLoad = 1; 6191} 6192 6193multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6194 ZPRRegOp zprty, RegisterOperand gprty> { 6195 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6196 6197 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6198 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6199} 6200 6201class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6202 RegisterOperand VecList, Operand immtype> 6203: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6204 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6205 "", 6206 []>, Sched<[]> { 6207 bits<3> Pg; 6208 bits<5> Rn; 6209 bits<5> Zt; 6210 bits<6> imm6; 6211 let Inst{31-25} = 0b1000010; 6212 let Inst{24-23} = dtypeh; 6213 let Inst{22} = 1; 6214 let Inst{21-16} = imm6; 6215 let Inst{15} = 0b1; 6216 let Inst{14-13} = dtypel; 6217 let Inst{12-10} = Pg; 6218 let Inst{9-5} = Rn; 6219 let Inst{4-0} = Zt; 6220 6221 let mayLoad = 1; 6222} 6223 6224multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6225 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6226 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6227 6228 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6229 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6230 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6231 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6232 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6233 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6234} 6235 6236class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6237 RegisterOperand VecList> 6238: I<(outs VecList:$Zt), iops, 6239 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6240 "", 6241 []>, Sched<[]> { 6242 bits<5> Zt; 6243 bits<3> Pg; 6244 bits<5> Rm; 6245 bits<5> Rn; 6246 let Inst{31-25} = 0b1010010; 6247 let Inst{24-21} = dtype; 6248 let Inst{20-16} = Rm; 6249 let Inst{15-14} = 0b01; 6250 let Inst{13} = ff; 6251 let Inst{12-10} = Pg; 6252 let Inst{9-5} = Rn; 6253 let Inst{4-0} = Zt; 6254 6255 let mayLoad = 1; 6256 let Uses = !if(!eq(ff, 1), [FFR], []); 6257 let Defs = !if(!eq(ff, 1), [FFR], []); 6258} 6259 6260multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6261 ZPRRegOp zprty, RegisterOperand gprty> { 6262 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6263 asm, listty>; 6264 6265 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6266 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6267} 6268 6269multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 6270 ZPRRegOp zprty, RegisterOperand gprty> { 6271 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6272 asm, listty>; 6273 6274 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6275 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6276 6277 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6278 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 6279 6280 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6281 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 6282 6283 // We need a layer of indirection because early machine code passes balk at 6284 // physical register (i.e. FFR) uses that have no previous definition. 6285 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6286 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 6287 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 6288 } 6289} 6290 6291multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 6292 ZPRRegOp zprty> 6293: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 6294 6295class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6296 string asm, Operand immtype> 6297: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 6298 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6299 "", 6300 []>, Sched<[]> { 6301 bits<5> Zt; 6302 bits<3> Pg; 6303 bits<5> Rn; 6304 bits<4> imm4; 6305 let Inst{31-25} = 0b1010010; 6306 let Inst{24-23} = sz; 6307 let Inst{22-21} = nregs; 6308 let Inst{20} = 0; 6309 let Inst{19-16} = imm4; 6310 let Inst{15-13} = 0b111; 6311 let Inst{12-10} = Pg; 6312 let Inst{9-5} = Rn; 6313 let Inst{4-0} = Zt; 6314 6315 let mayLoad = 1; 6316} 6317 6318multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6319 string asm, Operand immtype> { 6320 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 6321 6322 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6323 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6324} 6325 6326class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6327 string asm, RegisterOperand gprty> 6328: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6329 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6330 "", 6331 []>, Sched<[]> { 6332 bits<3> Pg; 6333 bits<5> Rm; 6334 bits<5> Rn; 6335 bits<5> Zt; 6336 let Inst{31-25} = 0b1010010; 6337 let Inst{24-23} = sz; 6338 let Inst{22-21} = nregs; 6339 let Inst{20-16} = Rm; 6340 let Inst{15-13} = 0b110; 6341 let Inst{12-10} = Pg; 6342 let Inst{9-5} = Rn; 6343 let Inst{4-0} = Zt; 6344 6345 let mayLoad = 1; 6346} 6347 6348//===----------------------------------------------------------------------===// 6349// SVE Memory - 32-bit Gather and Unsized Contiguous Group 6350//===----------------------------------------------------------------------===// 6351 6352// bit xs is '1' if offsets are signed 6353// bit scaled is '1' if the offsets are scaled 6354class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 6355 RegisterOperand zprext> 6356: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6357 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6358 "", 6359 []>, Sched<[]> { 6360 bits<3> Pg; 6361 bits<5> Rn; 6362 bits<5> Zm; 6363 bits<5> Zt; 6364 let Inst{31-25} = 0b1000010; 6365 let Inst{24-23} = opc{3-2}; 6366 let Inst{22} = xs; 6367 let Inst{21} = scaled; 6368 let Inst{20-16} = Zm; 6369 let Inst{15} = 0b0; 6370 let Inst{14-13} = opc{1-0}; 6371 let Inst{12-10} = Pg; 6372 let Inst{9-5} = Rn; 6373 let Inst{4-0} = Zt; 6374 6375 let mayLoad = 1; 6376 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6377 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6378} 6379 6380multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 6381 SDPatternOperator sxtw_op, 6382 SDPatternOperator uxtw_op, 6383 RegisterOperand sxtw_opnd, 6384 RegisterOperand uxtw_opnd, 6385 ValueType vt> { 6386 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 6387 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 6388 6389 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6390 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6391 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6392 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6393 6394 // We need a layer of indirection because early machine code passes balk at 6395 // physical register (i.e. FFR) uses that have no previous definition. 6396 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6397 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6398 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6399 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6400 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6401 } 6402 6403 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6404 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6405 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6406 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6407} 6408 6409multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 6410 SDPatternOperator sxtw_op, 6411 SDPatternOperator uxtw_op, 6412 RegisterOperand sxtw_opnd, 6413 RegisterOperand uxtw_opnd, 6414 ValueType vt> { 6415 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 6416 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 6417 6418 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6419 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6420 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6421 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6422 6423 // We need a layer of indirection because early machine code passes balk at 6424 // physical register (i.e. FFR) uses that have no previous definition. 6425 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6426 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6427 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6428 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6429 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6430 } 6431 6432 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6433 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6434 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6435 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6436} 6437 6438 6439class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 6440: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 6441 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 6442 "", 6443 []>, Sched<[]> { 6444 bits<3> Pg; 6445 bits<5> Zn; 6446 bits<5> Zt; 6447 bits<5> imm5; 6448 let Inst{31-25} = 0b1000010; 6449 let Inst{24-23} = opc{3-2}; 6450 let Inst{22-21} = 0b01; 6451 let Inst{20-16} = imm5; 6452 let Inst{15} = 0b1; 6453 let Inst{14-13} = opc{1-0}; 6454 let Inst{12-10} = Pg; 6455 let Inst{9-5} = Zn; 6456 let Inst{4-0} = Zt; 6457 6458 let mayLoad = 1; 6459 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6460 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6461} 6462 6463multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 6464 SDPatternOperator op, ValueType vt> { 6465 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 6466 6467 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6468 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6469 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 6470 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6471 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6472 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6473 6474 // We need a layer of indirection because early machine code passes balk at 6475 // physical register (i.e. FFR) uses that have no previous definition. 6476 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6477 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 6478 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 6479 } 6480 6481 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 6482 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6483} 6484 6485class sve_mem_prfm_si<bits<2> msz, string asm> 6486: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 6487 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 6488 "", 6489 []>, Sched<[]> { 6490 bits<5> Rn; 6491 bits<3> Pg; 6492 bits<6> imm6; 6493 bits<4> prfop; 6494 let Inst{31-22} = 0b1000010111; 6495 let Inst{21-16} = imm6; 6496 let Inst{15} = 0b0; 6497 let Inst{14-13} = msz; 6498 let Inst{12-10} = Pg; 6499 let Inst{9-5} = Rn; 6500 let Inst{4} = 0b0; 6501 let Inst{3-0} = prfop; 6502 6503 let hasSideEffects = 1; 6504} 6505 6506multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 6507 def NAME : sve_mem_prfm_si<msz, asm>; 6508 6509 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 6510 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6511} 6512 6513class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 6514: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6515 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 6516 "", 6517 []>, Sched<[]> { 6518 bits<5> Rm; 6519 bits<5> Rn; 6520 bits<3> Pg; 6521 bits<4> prfop; 6522 let Inst{31-25} = 0b1000010; 6523 let Inst{24-23} = opc{2-1}; 6524 let Inst{22-21} = 0b00; 6525 let Inst{20-16} = Rm; 6526 let Inst{15} = 0b1; 6527 let Inst{14} = opc{0}; 6528 let Inst{13} = 0b0; 6529 let Inst{12-10} = Pg; 6530 let Inst{9-5} = Rn; 6531 let Inst{4} = 0b0; 6532 let Inst{3-0} = prfop; 6533 6534 let hasSideEffects = 1; 6535} 6536 6537class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 6538 RegisterOperand zprext> 6539: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6540 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 6541 "", 6542 []>, Sched<[]> { 6543 bits<3> Pg; 6544 bits<5> Rn; 6545 bits<5> Zm; 6546 bits<4> prfop; 6547 let Inst{31-23} = 0b100001000; 6548 let Inst{22} = xs; 6549 let Inst{21} = 0b1; 6550 let Inst{20-16} = Zm; 6551 let Inst{15} = 0b0; 6552 let Inst{14-13} = msz; 6553 let Inst{12-10} = Pg; 6554 let Inst{9-5} = Rn; 6555 let Inst{4} = 0b0; 6556 let Inst{3-0} = prfop; 6557 6558 let hasSideEffects = 1; 6559} 6560 6561multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 6562 RegisterOperand sxtw_opnd, 6563 RegisterOperand uxtw_opnd, 6564 PatFrag op_sxtw, 6565 PatFrag op_uxtw> { 6566 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 6567 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 6568 6569 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6570 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6571 6572 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6573 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6574} 6575 6576class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 6577: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 6578 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 6579 "", 6580 []>, Sched<[]> { 6581 bits<3> Pg; 6582 bits<5> Zn; 6583 bits<5> imm5; 6584 bits<4> prfop; 6585 let Inst{31-25} = 0b1000010; 6586 let Inst{24-23} = msz; 6587 let Inst{22-21} = 0b00; 6588 let Inst{20-16} = imm5; 6589 let Inst{15-13} = 0b111; 6590 let Inst{12-10} = Pg; 6591 let Inst{9-5} = Zn; 6592 let Inst{4} = 0b0; 6593 let Inst{3-0} = prfop; 6594} 6595 6596multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 6597 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 6598 6599 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 6600 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6601 6602 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 6603 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 6604} 6605 6606class sve_mem_z_fill<string asm> 6607: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 6608 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6609 "", 6610 []>, Sched<[]> { 6611 bits<5> Rn; 6612 bits<5> Zt; 6613 bits<9> imm9; 6614 let Inst{31-22} = 0b1000010110; 6615 let Inst{21-16} = imm9{8-3}; 6616 let Inst{15-13} = 0b010; 6617 let Inst{12-10} = imm9{2-0}; 6618 let Inst{9-5} = Rn; 6619 let Inst{4-0} = Zt; 6620 6621 let mayLoad = 1; 6622} 6623 6624multiclass sve_mem_z_fill<string asm> { 6625 def NAME : sve_mem_z_fill<asm>; 6626 6627 def : InstAlias<asm # "\t$Zt, [$Rn]", 6628 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6629} 6630 6631class sve_mem_p_fill<string asm> 6632: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 6633 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6634 "", 6635 []>, Sched<[]> { 6636 bits<4> Pt; 6637 bits<5> Rn; 6638 bits<9> imm9; 6639 let Inst{31-22} = 0b1000010110; 6640 let Inst{21-16} = imm9{8-3}; 6641 let Inst{15-13} = 0b000; 6642 let Inst{12-10} = imm9{2-0}; 6643 let Inst{9-5} = Rn; 6644 let Inst{4} = 0b0; 6645 let Inst{3-0} = Pt; 6646 6647 let mayLoad = 1; 6648} 6649 6650multiclass sve_mem_p_fill<string asm> { 6651 def NAME : sve_mem_p_fill<asm>; 6652 6653 def : InstAlias<asm # "\t$Pt, [$Rn]", 6654 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6655} 6656 6657class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 6658 RegisterOperand VecList> 6659: I<(outs VecList:$Zt), iops, 6660 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 6661 "", 6662 []>, Sched<[]> { 6663 bits<3> Pg; 6664 bits<5> Rm; 6665 bits<5> Zn; 6666 bits<5> Zt; 6667 let Inst{31} = 0b1; 6668 let Inst{30} = opc{4}; 6669 let Inst{29-25} = 0b00010; 6670 let Inst{24-23} = opc{3-2}; 6671 let Inst{22-21} = 0b00; 6672 let Inst{20-16} = Rm; 6673 let Inst{15} = 0b1; 6674 let Inst{14-13} = opc{1-0}; 6675 let Inst{12-10} = Pg; 6676 let Inst{9-5} = Zn; 6677 let Inst{4-0} = Zt; 6678 6679 let mayLoad = 1; 6680} 6681 6682multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 6683 SDPatternOperator op, 6684 ValueType vt> { 6685 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 6686 asm, Z_s>; 6687 6688 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 6689 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 6690 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6691 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 6692 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6693 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 6694 6695 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 6696 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 6697} 6698 6699multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 6700 SDPatternOperator op, 6701 ValueType vt> { 6702 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 6703 asm, Z_d>; 6704 6705 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 6706 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 6707 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6708 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 6709 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6710 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 6711 6712 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 6713 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 6714} 6715 6716//===----------------------------------------------------------------------===// 6717// SVE Memory - 64-bit Gather Group 6718//===----------------------------------------------------------------------===// 6719 6720// bit xs is '1' if offsets are signed 6721// bit scaled is '1' if the offsets are scaled 6722// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 6723class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 6724 RegisterOperand zprext> 6725: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6726 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6727 "", 6728 []>, Sched<[]> { 6729 bits<3> Pg; 6730 bits<5> Rn; 6731 bits<5> Zm; 6732 bits<5> Zt; 6733 let Inst{31-25} = 0b1100010; 6734 let Inst{24-23} = opc{3-2}; 6735 let Inst{22} = xs; 6736 let Inst{21} = scaled; 6737 let Inst{20-16} = Zm; 6738 let Inst{15} = lsl; 6739 let Inst{14-13} = opc{1-0}; 6740 let Inst{12-10} = Pg; 6741 let Inst{9-5} = Rn; 6742 let Inst{4-0} = Zt; 6743 6744 let mayLoad = 1; 6745 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6746 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6747} 6748 6749multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 6750 SDPatternOperator sxtw_op, 6751 SDPatternOperator uxtw_op, 6752 RegisterOperand sxtw_opnd, 6753 RegisterOperand uxtw_opnd, 6754 ValueType vt> { 6755 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 6756 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 6757 6758 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6759 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6760 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6761 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6762 6763 // We need a layer of indirection because early machine code passes balk at 6764 // physical register (i.e. FFR) uses that have no previous definition. 6765 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6766 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6767 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6768 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6769 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6770 } 6771 6772 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6773 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6774 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6775 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6776} 6777 6778multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 6779 SDPatternOperator sxtw_op, 6780 SDPatternOperator uxtw_op, 6781 RegisterOperand sxtw_opnd, 6782 RegisterOperand uxtw_opnd, 6783 ValueType vt> { 6784 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 6785 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 6786 6787 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6788 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6789 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6790 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6791 6792 // We need a layer of indirection because early machine code passes balk at 6793 // physical register (i.e. FFR) uses that have no previous definition. 6794 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6795 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6796 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6797 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6798 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6799 } 6800 6801 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6802 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6803 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6804 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6805} 6806 6807multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 6808 SDPatternOperator op, 6809 RegisterOperand zprext, ValueType vt> { 6810 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 6811 6812 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6813 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6814 6815 // We need a layer of indirection because early machine code passes balk at 6816 // physical register (i.e. FFR) uses that have no previous definition. 6817 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6818 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 6819 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 6820 } 6821 6822 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6823 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6824} 6825 6826multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 6827 SDPatternOperator op, ValueType vt> { 6828 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 6829 6830 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6831 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6832 6833 // We need a layer of indirection because early machine code passes balk at 6834 // physical register (i.e. FFR) uses that have no previous definition. 6835 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6836 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 6837 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 6838 } 6839 6840 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6841 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6842} 6843 6844class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 6845: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 6846 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 6847 "", 6848 []>, Sched<[]> { 6849 bits<3> Pg; 6850 bits<5> Zn; 6851 bits<5> Zt; 6852 bits<5> imm5; 6853 let Inst{31-25} = 0b1100010; 6854 let Inst{24-23} = opc{3-2}; 6855 let Inst{22-21} = 0b01; 6856 let Inst{20-16} = imm5; 6857 let Inst{15} = 0b1; 6858 let Inst{14-13} = opc{1-0}; 6859 let Inst{12-10} = Pg; 6860 let Inst{9-5} = Zn; 6861 let Inst{4-0} = Zt; 6862 6863 let mayLoad = 1; 6864 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6865 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6866} 6867 6868multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 6869 SDPatternOperator op, ValueType vt> { 6870 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 6871 6872 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6873 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 6874 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 6875 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 6876 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6877 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6878 6879 // We need a layer of indirection because early machine code passes balk at 6880 // physical register (i.e. FFR) uses that have no previous definition. 6881 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6882 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 6883 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 6884 } 6885 6886 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 6887 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6888} 6889 6890// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 6891class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 6892 RegisterOperand zprext> 6893: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6894 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 6895 "", 6896 []>, Sched<[]> { 6897 bits<3> Pg; 6898 bits<5> Rn; 6899 bits<5> Zm; 6900 bits<4> prfop; 6901 let Inst{31-23} = 0b110001000; 6902 let Inst{22} = xs; 6903 let Inst{21} = 0b1; 6904 let Inst{20-16} = Zm; 6905 let Inst{15} = lsl; 6906 let Inst{14-13} = msz; 6907 let Inst{12-10} = Pg; 6908 let Inst{9-5} = Rn; 6909 let Inst{4} = 0b0; 6910 let Inst{3-0} = prfop; 6911 6912 let hasSideEffects = 1; 6913} 6914 6915multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 6916 RegisterOperand sxtw_opnd, 6917 RegisterOperand uxtw_opnd, 6918 PatFrag op_sxtw, 6919 PatFrag op_uxtw> { 6920 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 6921 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 6922 6923 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6924 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6925 6926 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6927 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6928 6929} 6930 6931multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 6932 RegisterOperand zprext, PatFrag frag> { 6933 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 6934 6935 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 6936 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 6937 6938} 6939 6940 6941class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 6942: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 6943 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 6944 "", 6945 []>, Sched<[]> { 6946 bits<3> Pg; 6947 bits<5> Zn; 6948 bits<5> imm5; 6949 bits<4> prfop; 6950 let Inst{31-25} = 0b1100010; 6951 let Inst{24-23} = msz; 6952 let Inst{22-21} = 0b00; 6953 let Inst{20-16} = imm5; 6954 let Inst{15-13} = 0b111; 6955 let Inst{12-10} = Pg; 6956 let Inst{9-5} = Zn; 6957 let Inst{4} = 0b0; 6958 let Inst{3-0} = prfop; 6959 6960 let hasSideEffects = 1; 6961} 6962 6963multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 6964 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 6965 6966 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 6967 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6968 6969 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 6970 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 6971} 6972 6973//===----------------------------------------------------------------------===// 6974// SVE Compute Vector Address Group 6975//===----------------------------------------------------------------------===// 6976 6977class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 6978 ZPRRegOp zprty, RegisterOperand zprext> 6979: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 6980 asm, "\t$Zd, [$Zn, $Zm]", 6981 "", 6982 []>, Sched<[]> { 6983 bits<5> Zd; 6984 bits<5> Zn; 6985 bits<5> Zm; 6986 let Inst{31-24} = 0b00000100; 6987 let Inst{23-22} = opc; 6988 let Inst{21} = 0b1; 6989 let Inst{20-16} = Zm; 6990 let Inst{15-12} = 0b1010; 6991 let Inst{11-10} = msz; 6992 let Inst{9-5} = Zn; 6993 let Inst{4-0} = Zd; 6994} 6995 6996multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 6997 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 6998 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 6999 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7000 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7001} 7002 7003multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7004 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7005 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7006 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7007 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7008} 7009 7010multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7011 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7012 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7013 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7014 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7015} 7016 7017multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7018 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7019 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7020 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7021 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7022} 7023 7024 7025//===----------------------------------------------------------------------===// 7026// SVE Integer Misc - Unpredicated Group 7027//===----------------------------------------------------------------------===// 7028 7029class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7030: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7031 asm, "\t$Zd, $Zn, $Zm", 7032 "", 7033 []>, Sched<[]> { 7034 bits<5> Zd; 7035 bits<5> Zm; 7036 bits<5> Zn; 7037 let Inst{31-24} = 0b00000100; 7038 let Inst{23-22} = sz; 7039 let Inst{21} = 0b1; 7040 let Inst{20-16} = Zm; 7041 let Inst{15-10} = 0b101100; 7042 let Inst{9-5} = Zn; 7043 let Inst{4-0} = Zd; 7044} 7045 7046multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7047 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7048 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7049 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7050 7051 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7052 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7053 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7054} 7055 7056class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7057: I<(outs zprty:$Zd), (ins zprty:$Zn), 7058 asm, "\t$Zd, $Zn", 7059 "", 7060 []>, Sched<[]> { 7061 bits<5> Zd; 7062 bits<5> Zn; 7063 let Inst{31-24} = 0b00000100; 7064 let Inst{23-22} = opc{7-6}; 7065 let Inst{21} = 0b1; 7066 let Inst{20-16} = opc{5-1}; 7067 let Inst{15-11} = 0b10111; 7068 let Inst{10} = opc{0}; 7069 let Inst{9-5} = Zn; 7070 let Inst{4-0} = Zd; 7071} 7072 7073multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7074 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7075 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7076 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7077 7078 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7079 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7080 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7081} 7082 7083//===----------------------------------------------------------------------===// 7084// SVE Integer Reduction Group 7085//===----------------------------------------------------------------------===// 7086 7087class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7088 ZPRRegOp zprty, RegisterClass regtype> 7089: I<(outs regtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7090 asm, "\t$Vd, $Pg, $Zn", 7091 "", 7092 []>, Sched<[]> { 7093 bits<3> Pg; 7094 bits<5> Vd; 7095 bits<5> Zn; 7096 let Inst{31-24} = 0b00000100; 7097 let Inst{23-22} = sz8_32; 7098 let Inst{21} = 0b0; 7099 let Inst{20-19} = fmt; 7100 let Inst{18-16} = opc; 7101 let Inst{15-13} = 0b001; 7102 let Inst{12-10} = Pg; 7103 let Inst{9-5} = Zn; 7104 let Inst{4-0} = Vd; 7105} 7106 7107multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, SDPatternOperator op> { 7108 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64>; 7109 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64>; 7110 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64>; 7111 7112 def : SVE_2_Op_Pat<i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7113 def : SVE_2_Op_Pat<i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7114 def : SVE_2_Op_Pat<i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7115} 7116 7117multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, SDPatternOperator op, SDPatternOperator opSaddv> { 7118 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64>; 7119 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64>; 7120 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64>; 7121 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64>; 7122 7123 def : SVE_2_Op_Pat<i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7124 def : SVE_2_Op_Pat<i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7125 def : SVE_2_Op_Pat<i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7126 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7127 def : SVE_2_Op_Pat<i64, opSaddv, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7128} 7129 7130multiclass sve_int_reduce_1<bits<3> opc, string asm, SDPatternOperator op> { 7131 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8>; 7132 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16>; 7133 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32>; 7134 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64>; 7135 7136 def : SVE_2_Op_Pat_Reduce_To_Neon<v16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B), bsub>; 7137 def : SVE_2_Op_Pat_Reduce_To_Neon<v8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H), hsub>; 7138 def : SVE_2_Op_Pat_Reduce_To_Neon<v4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S), ssub>; 7139 def : SVE_2_Op_Pat_Reduce_To_Neon<v2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D), dsub>; 7140} 7141 7142multiclass sve_int_reduce_2<bits<3> opc, string asm, SDPatternOperator op> { 7143 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8>; 7144 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16>; 7145 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32>; 7146 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64>; 7147 7148 def : SVE_2_Op_Pat_Reduce_To_Neon<v16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B), bsub>; 7149 def : SVE_2_Op_Pat_Reduce_To_Neon<v8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H), hsub>; 7150 def : SVE_2_Op_Pat_Reduce_To_Neon<v4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S), ssub>; 7151 def : SVE_2_Op_Pat_Reduce_To_Neon<v2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D), dsub>; 7152} 7153 7154class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7155 ZPRRegOp zprty, string pg_suffix, dag iops> 7156: I<(outs zprty:$Zd), iops, 7157 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7158 "", 7159 []>, Sched<[]> { 7160 bits<3> Pg; 7161 bits<5> Zd; 7162 bits<5> Zn; 7163 let Inst{31-24} = 0b00000100; 7164 let Inst{23-22} = sz8_32; 7165 let Inst{21-19} = 0b010; 7166 let Inst{18-16} = opc; 7167 let Inst{15-13} = 0b001; 7168 let Inst{12-10} = Pg; 7169 let Inst{9-5} = Zn; 7170 let Inst{4-0} = Zd; 7171 7172 let ElementSize = zprty.ElementSize; 7173} 7174 7175multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7176let Constraints = "$Zd = $_Zd" in { 7177 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7178 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7179 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7180 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7181 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7182 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7183 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7184 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7185} 7186} 7187 7188multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7189 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7190 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7191 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7192 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7193 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7194 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7195 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7196 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7197} 7198 7199//===----------------------------------------------------------------------===// 7200// SVE Propagate Break Group 7201//===----------------------------------------------------------------------===// 7202 7203class sve_int_brkp<bits<2> opc, string asm> 7204: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7205 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7206 "", 7207 []>, Sched<[]> { 7208 bits<4> Pd; 7209 bits<4> Pg; 7210 bits<4> Pm; 7211 bits<4> Pn; 7212 let Inst{31-24} = 0b00100101; 7213 let Inst{23} = 0b0; 7214 let Inst{22} = opc{1}; 7215 let Inst{21-20} = 0b00; 7216 let Inst{19-16} = Pm; 7217 let Inst{15-14} = 0b11; 7218 let Inst{13-10} = Pg; 7219 let Inst{9} = 0b0; 7220 let Inst{8-5} = Pn; 7221 let Inst{4} = opc{0}; 7222 let Inst{3-0} = Pd; 7223 7224 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7225} 7226 7227multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7228 def NAME : sve_int_brkp<opc, asm>; 7229 7230 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7231} 7232 7233 7234//===----------------------------------------------------------------------===// 7235// SVE Partition Break Group 7236//===----------------------------------------------------------------------===// 7237 7238class sve_int_brkn<bit S, string asm> 7239: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7240 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7241 "", 7242 []>, Sched<[]> { 7243 bits<4> Pdm; 7244 bits<4> Pg; 7245 bits<4> Pn; 7246 let Inst{31-23} = 0b001001010; 7247 let Inst{22} = S; 7248 let Inst{21-14} = 0b01100001; 7249 let Inst{13-10} = Pg; 7250 let Inst{9} = 0b0; 7251 let Inst{8-5} = Pn; 7252 let Inst{4} = 0b0; 7253 let Inst{3-0} = Pdm; 7254 7255 let Constraints = "$Pdm = $_Pdm"; 7256 let Defs = !if(!eq (S, 0b1), [NZCV], []); 7257} 7258 7259multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7260 def NAME : sve_int_brkn<opc, asm>; 7261 7262 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7263} 7264 7265class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7266: I<(outs PPR8:$Pd), iops, 7267 asm, "\t$Pd, $Pg"#suffix#", $Pn", 7268 "", 7269 []>, Sched<[]> { 7270 bits<4> Pd; 7271 bits<4> Pg; 7272 bits<4> Pn; 7273 let Inst{31-24} = 0b00100101; 7274 let Inst{23-22} = opc{2-1}; 7275 let Inst{21-14} = 0b01000001; 7276 let Inst{13-10} = Pg; 7277 let Inst{9} = 0b0; 7278 let Inst{8-5} = Pn; 7279 let Inst{4} = opc{0}; 7280 let Inst{3-0} = Pd; 7281 7282 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 7283 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7284 7285} 7286 7287multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 7288 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 7289 7290 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7291} 7292 7293multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 7294 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 7295 7296 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7297} 7298 7299//===----------------------------------------------------------------------===// 7300// SVE2 String Processing Group 7301//===----------------------------------------------------------------------===// 7302 7303class sve2_char_match<bit sz, bit opc, string asm, 7304 PPRRegOp pprty, ZPRRegOp zprty> 7305: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7306 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 7307 "", 7308 []>, Sched<[]> { 7309 bits<4> Pd; 7310 bits<3> Pg; 7311 bits<5> Zm; 7312 bits<5> Zn; 7313 let Inst{31-23} = 0b010001010; 7314 let Inst{22} = sz; 7315 let Inst{21} = 0b1; 7316 let Inst{20-16} = Zm; 7317 let Inst{15-13} = 0b100; 7318 let Inst{12-10} = Pg; 7319 let Inst{9-5} = Zn; 7320 let Inst{4} = opc; 7321 let Inst{3-0} = Pd; 7322 7323 let Defs = [NZCV]; 7324} 7325 7326multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 7327 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 7328 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 7329 7330 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 7331 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7332} 7333 7334//===----------------------------------------------------------------------===// 7335// SVE2 Histogram Computation - Segment Group 7336//===----------------------------------------------------------------------===// 7337 7338class sve2_hist_gen_segment<string asm, SDPatternOperator op> 7339: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 7340 asm, "\t$Zd, $Zn, $Zm", 7341 "", 7342 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 7343 bits<5> Zd; 7344 bits<5> Zn; 7345 bits<5> Zm; 7346 let Inst{31-21} = 0b01000101001; 7347 let Inst{20-16} = Zm; 7348 let Inst{15-10} = 0b101000; 7349 let Inst{9-5} = Zn; 7350 let Inst{4-0} = Zd; 7351} 7352 7353//===----------------------------------------------------------------------===// 7354// SVE2 Histogram Computation - Vector Group 7355//===----------------------------------------------------------------------===// 7356 7357class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 7358: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7359 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 7360 "", 7361 []>, Sched<[]> { 7362 bits<5> Zd; 7363 bits<5> Zn; 7364 bits<3> Pg; 7365 bits<5> Zm; 7366 let Inst{31-23} = 0b010001011; 7367 let Inst{22} = sz; 7368 let Inst{21} = 0b1; 7369 let Inst{20-16} = Zm; 7370 let Inst{15-13} = 0b110; 7371 let Inst{12-10} = Pg; 7372 let Inst{9-5} = Zn; 7373 let Inst{4-0} = Zd; 7374} 7375 7376multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 7377 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 7378 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 7379 7380 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7381 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7382} 7383 7384//===----------------------------------------------------------------------===// 7385// SVE2 Crypto Extensions Group 7386//===----------------------------------------------------------------------===// 7387 7388class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 7389: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7390 asm, "\t$Zd, $Zn, $Zm", 7391 "", 7392 []>, Sched<[]> { 7393 bits<5> Zd; 7394 bits<5> Zn; 7395 bits<5> Zm; 7396 let Inst{31-21} = 0b01000101001; 7397 let Inst{20-16} = Zm; 7398 let Inst{15-11} = 0b11110; 7399 let Inst{10} = opc; 7400 let Inst{9-5} = Zn; 7401 let Inst{4-0} = Zd; 7402} 7403 7404multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 7405 SDPatternOperator op, ValueType vt> { 7406 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 7407 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7408} 7409 7410class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 7411: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 7412 asm, "\t$Zdn, $_Zdn, $Zm", 7413 "", 7414 []>, Sched<[]> { 7415 bits<5> Zdn; 7416 bits<5> Zm; 7417 let Inst{31-17} = 0b010001010010001; 7418 let Inst{16} = opc{1}; 7419 let Inst{15-11} = 0b11100; 7420 let Inst{10} = opc{0}; 7421 let Inst{9-5} = Zm; 7422 let Inst{4-0} = Zdn; 7423 7424 let Constraints = "$Zdn = $_Zdn"; 7425} 7426 7427multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 7428 SDPatternOperator op, ValueType vt> { 7429 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 7430 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7431} 7432 7433class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 7434: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 7435 asm, "\t$Zdn, $_Zdn", 7436 "", 7437 []>, Sched<[]> { 7438 bits<5> Zdn; 7439 let Inst{31-11} = 0b010001010010000011100; 7440 let Inst{10} = opc; 7441 let Inst{9-5} = 0b00000; 7442 let Inst{4-0} = Zdn; 7443 7444 let Constraints = "$Zdn = $_Zdn"; 7445} 7446 7447multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 7448 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 7449 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 7450} 7451 7452//===----------------------------------------------------------------------===// 7453// SVE BFloat16 Group 7454//===----------------------------------------------------------------------===// 7455 7456class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 7457: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 7458 bits<5> Zda; 7459 bits<5> Zn; 7460 let Inst{31-21} = 0b01100100011; 7461 let Inst{15-14} = opc; 7462 let Inst{13-10} = 0b0000; 7463 let Inst{9-5} = Zn; 7464 let Inst{4-0} = Zda; 7465 7466 let Constraints = "$Zda = $_Zda"; 7467 let DestructiveInstType = DestructiveOther; 7468 let ElementSize = ElementSizeH; 7469} 7470 7471class sve_bfloat_dot<string asm> 7472: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 7473 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 7474 bits<5> Zm; 7475 let Inst{20-16} = Zm; 7476} 7477 7478multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 7479 def NAME : sve_bfloat_dot<asm>; 7480 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7481} 7482 7483class sve_bfloat_dot_indexed<string asm> 7484: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 7485 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 7486 bits<2> iop; 7487 bits<3> Zm; 7488 let Inst{20-19} = iop; 7489 let Inst{18-16} = Zm; 7490} 7491 7492multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 7493 def NAME : sve_bfloat_dot_indexed<asm>; 7494 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 7495} 7496 7497class sve_bfloat_matmul<string asm> 7498: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 7499 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7500 bits<5> Zm; 7501 bits<5> Zda; 7502 bits<5> Zn; 7503 let Inst{31-21} = 0b01100100011; 7504 let Inst{20-16} = Zm; 7505 let Inst{15-10} = 0b111001; 7506 let Inst{9-5} = Zn; 7507 let Inst{4-0} = Zda; 7508 7509 let Constraints = "$Zda = $_Zda"; 7510 let DestructiveInstType = DestructiveOther; 7511 let ElementSize = ElementSizeH; 7512} 7513 7514multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 7515 def NAME : sve_bfloat_matmul<asm>; 7516 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7517} 7518 7519class sve_bfloat_matmul_longvecl<bit BT, string asm> 7520: sve_bfloat_matmul<asm> { 7521 let Inst{23} = 0b1; 7522 let Inst{14-13} = 0b00; 7523 let Inst{10} = BT; 7524} 7525 7526multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 7527 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 7528 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7529} 7530 7531class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 7532: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 7533 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 7534 bits<3> iop; 7535 bits<3> Zm; 7536 let Inst{23} = 0b1; 7537 let Inst{20-19} = iop{2-1}; 7538 let Inst{18-16} = Zm; 7539 let Inst{11} = iop{0}; 7540 let Inst{10} = BT; 7541} 7542 7543multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 7544 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 7545 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 7546} 7547 7548class sve_bfloat_convert<bit N, string asm> 7549: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 7550 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 7551 bits<5> Zd; 7552 bits<3> Pg; 7553 bits<5> Zn; 7554 let Inst{31-25} = 0b0110010; 7555 let Inst{24} = N; 7556 let Inst{23-13} = 0b10001010101; 7557 let Inst{12-10} = Pg; 7558 let Inst{9-5} = Zn; 7559 let Inst{4-0} = Zd; 7560 7561 let Constraints = "$Zd = $_Zd"; 7562 let DestructiveInstType = DestructiveOther; 7563 let hasSideEffects = 1; 7564 let ElementSize = ElementSizeS; 7565} 7566 7567multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 7568 def NAME : sve_bfloat_convert<N, asm>; 7569 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 7570} 7571 7572//===----------------------------------------------------------------------===// 7573// SVE Integer Matrix Multiply Group 7574//===----------------------------------------------------------------------===// 7575 7576class sve_int_matmul<bits<2> uns, string asm> 7577: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 7578 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7579 bits<5> Zda; 7580 bits<5> Zn; 7581 bits<5> Zm; 7582 let Inst{31-24} = 0b01000101; 7583 let Inst{23-22} = uns; 7584 let Inst{21} = 0; 7585 let Inst{20-16} = Zm; 7586 let Inst{15-10} = 0b100110; 7587 let Inst{9-5} = Zn; 7588 let Inst{4-0} = Zda; 7589 7590 let Constraints = "$Zda = $_Zda"; 7591 let DestructiveInstType = DestructiveOther; 7592 let ElementSize = ZPR32.ElementSize; 7593} 7594 7595multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 7596 def NAME : sve_int_matmul<uns, asm>; 7597 7598 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7599} 7600 7601//===----------------------------------------------------------------------===// 7602// SVE Integer Dot Product Mixed Sign Group 7603//===----------------------------------------------------------------------===// 7604 7605class sve_int_dot_mixed<string asm> 7606: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 7607 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7608 bits<5> Zda; 7609 bits<5> Zn; 7610 bits<5> Zm; 7611 let Inst{31-21} = 0b01000100100; 7612 let Inst{20-16} = Zm; 7613 let Inst{15-10} = 0b011110; 7614 let Inst{9-5} = Zn; 7615 let Inst{4-0} = Zda; 7616 7617 let Constraints = "$Zda = $_Zda"; 7618 let DestructiveInstType = DestructiveOther; 7619 let ElementSize = ZPR32.ElementSize; 7620} 7621 7622multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 7623 def NAME : sve_int_dot_mixed<asm>; 7624 7625 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7626} 7627 7628//===----------------------------------------------------------------------===// 7629// SVE Integer Dot Product Mixed Sign - Indexed Group 7630//===----------------------------------------------------------------------===// 7631 7632class sve_int_dot_mixed_indexed<bit U, string asm> 7633: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 7634 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 7635 bits<5> Zda; 7636 bits<5> Zn; 7637 bits<3> Zm; 7638 bits<2> idx; 7639 let Inst{31-21} = 0b01000100101; 7640 let Inst{20-19} = idx; 7641 let Inst{18-16} = Zm; 7642 let Inst{15-11} = 0b00011; 7643 let Inst{10} = U; 7644 let Inst{9-5} = Zn; 7645 let Inst{4-0} = Zda; 7646 7647 let Constraints = "$Zda = $_Zda"; 7648 let DestructiveInstType = DestructiveOther; 7649 let ElementSize = ZPR32.ElementSize; 7650} 7651 7652multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 7653 def NAME : sve_int_dot_mixed_indexed<U, asm>; 7654 7655 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 7656} 7657 7658//===----------------------------------------------------------------------===// 7659// SVE Floating Point Matrix Multiply Accumulate Group 7660//===----------------------------------------------------------------------===// 7661 7662class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 7663: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 7664 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7665 bits<5> Zda; 7666 bits<5> Zn; 7667 bits<5> Zm; 7668 let Inst{31-23} = 0b011001001; 7669 let Inst{22} = sz; 7670 let Inst{21} = 1; 7671 let Inst{20-16} = Zm; 7672 let Inst{15-10} = 0b111001; 7673 let Inst{9-5} = Zn; 7674 let Inst{4-0} = Zda; 7675 7676 let Constraints = "$Zda = $_Zda"; 7677 let DestructiveInstType = DestructiveOther; 7678 let ElementSize = zprty.ElementSize; 7679} 7680 7681multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 7682 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 7683 7684 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 7685} 7686 7687//===----------------------------------------------------------------------===// 7688// SVE Memory - Contiguous Load And Replicate 256-bit Group 7689//===----------------------------------------------------------------------===// 7690 7691class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 7692: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 7693 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 7694 bits<5> Zt; 7695 bits<5> Rn; 7696 bits<3> Pg; 7697 bits<4> imm4; 7698 let Inst{31-25} = 0b1010010; 7699 let Inst{24-23} = sz; 7700 let Inst{22-20} = 0b010; 7701 let Inst{19-16} = imm4; 7702 let Inst{15-13} = 0b001; 7703 let Inst{12-10} = Pg; 7704 let Inst{9-5} = Rn; 7705 let Inst{4-0} = Zt; 7706 7707 let mayLoad = 1; 7708} 7709 7710multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 7711 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 7712 def NAME : sve_mem_ldor_si<sz, asm, listty>; 7713 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7714 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7715 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7716 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 7717 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 7718 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 7719 7720 // Base addressing mode 7721 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 7722 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 7723 let AddedComplexity = 2 in { 7724 // Reg + Imm addressing mode 7725 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 7726 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 7727 } 7728} 7729 7730class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 7731 RegisterOperand gprty> 7732: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7733 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 7734 bits<5> Zt; 7735 bits<3> Pg; 7736 bits<5> Rn; 7737 bits<5> Rm; 7738 let Inst{31-25} = 0b1010010; 7739 let Inst{24-23} = sz; 7740 let Inst{22-21} = 0b01; 7741 let Inst{20-16} = Rm; 7742 let Inst{15-13} = 0; 7743 let Inst{12-10} = Pg; 7744 let Inst{9-5} = Rn; 7745 let Inst{4-0} = Zt; 7746 7747 let mayLoad = 1; 7748} 7749 7750multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 7751 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 7752 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 7753 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 7754 7755 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7756 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7757 7758 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 7759 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 7760} 7761 7762//===----------------------------------------------------------------------===// 7763// SVE Interleave 128-bit Elements Group 7764//===----------------------------------------------------------------------===// 7765 7766class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 7767: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 7768 asm, "\t$Zd, $Zn, $Zm", 7769 "", 7770 []>, Sched<[]> { 7771 bits<5> Zd; 7772 bits<5> Zm; 7773 bits<5> Zn; 7774 let Inst{31-21} = 0b00000101101; 7775 let Inst{20-16} = Zm; 7776 let Inst{15-13} = 0b000; 7777 let Inst{12-11} = opc; 7778 let Inst{10} = P; 7779 let Inst{9-5} = Zn; 7780 let Inst{4-0} = Zd; 7781} 7782 7783multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 7784 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 7785 7786 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7787 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 7788 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 7789 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 7790 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 7791 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 7792 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 7793} 7794 7795/// Addressing modes 7796def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 7797def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 7798 7799def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>; 7800def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>; 7801def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>; 7802def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>; 7803 7804// Predicated pseudo floating point two operand instructions. 7805multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 7806 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 7807 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7808 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7809 7810 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7811 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7812 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7813} 7814 7815// Predicated pseudo integer two operand instructions. 7816multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 7817 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 7818 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 7819 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7820 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7821 7822 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 7823 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7824 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7825 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7826} 7827 7828// As sve_int_bin_pred but when only i32 and i64 vector types are required. 7829multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 7830 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7831 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7832 7833 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7834 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7835} 7836