Lines Matching refs:AArch64
1 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
104 cl::desc("Allow AArch64 SLI/SRI formation"),
112 cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
117 cl::desc("Enable AArch64 logical imm instruction "
127 // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
135 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
136 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
139 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
140 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
141 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
142 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
146 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
147 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
168 addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
169 addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
170 addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
171 addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
174 addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
175 addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
176 addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
177 addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
179 addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
180 addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
181 addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
182 addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
183 addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
184 addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
344 // AArch64 lacks both left-rotate and popcount instructions.
352 // AArch64 doesn't have {U|S}MUL_LOHI.
505 // AArch64 has implementations of a lot of rounding-like FP operations.
578 // AArch64 does not have floating-point extending loads, i1 sign-extending
680 setStackPointerRegisterToSaveRestore(AArch64::SP);
739 // AArch64 doesn't have a direct vector ->f32 conversion instructions for
762 // when AArch64 doesn't have fullfp16 support, promote the input
773 // AArch64 doesn't have MUL.2d:
829 // AArch64 has implementations of a lot of rounding-like FP operations.
948 addRegisterClass(VT, &AArch64::FPR64RegClass);
953 addRegisterClass(VT, &AArch64::FPR128RegClass);
1085 NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1088 NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1091 NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1229 return AArch64::createFastISel(funcInfo, libInfo);
1450 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
1451 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
1459 TrueBB->addLiveIn(AArch64::NZCV);
1460 EndBB->addLiveIn(AArch64::NZCV);
1463 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
1496 case AArch64::F128CSEL:
1503 case AArch64::CATCHRET:
1505 case AArch64::CATCHPAD:
1511 // AArch64 Lowering private implementation.
1518 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
1547 /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
1608 /// Convert a DAG fp condition code to an AArch64 CC.
1637 /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
2133 // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2249 // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2313 // Emit the AArch64 operation with overflow check.
2733 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
2962 report_fatal_error("Unexpected type for AArch64 NEON intrinic");
3404 RC = &AArch64::GPR32RegClass;
3406 RC = &AArch64::GPR64RegClass;
3408 RC = &AArch64::FPR16RegClass;
3410 RC = &AArch64::FPR32RegClass;
3412 RC = &AArch64::FPR64RegClass;
3414 RC = &AArch64::FPR128RegClass;
3417 RC = &AArch64::PPRRegClass;
3419 RC = &AArch64::ZPRRegClass;
3534 if (!CCInfo.isAllocated(AArch64::X8)) {
3535 unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
3536 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
3599 static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
3600 AArch64::X3, AArch64::X4, AArch64::X5,
3601 AArch64::X6, AArch64::X7 };
3619 unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
3638 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
3639 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
3651 unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
3790 // tail-called on AArch64 when the OS does not support dynamic
4039 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
4427 // Windows AArch64 ABIs require that for returning structs by value we copy
4435 unsigned RetValReg = AArch64::X0;
4448 if (AArch64::GPR64RegClass.contains(*I))
4450 else if (AArch64::FPR64RegClass.contains(*I))
4650 // normal AArch64 call node: x0 takes the address of the descriptor, and
4652 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
4655 Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
4657 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
4679 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
4694 Addr = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
4698 return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
4714 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
4717 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
4737 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVZXi, DL, PtrVT, HiVar,
4740 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, MiVar,
4743 TPOff = SDValue(DAG.getMachineNode(AArch64::MOVKXi, DL, PtrVT, TPOff, LoVar,
4781 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
4850 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
4853 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
4880 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
4923 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
5064 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
5114 setVecVal(AArch64::ssub);
5123 setVecVal(AArch64::dsub);
5127 setVecVal(AArch64::hsub);
5146 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
5148 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
5150 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
5285 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
5451 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
5563 DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
5629 // The layout of the va_list struct is specified in the AArch64 Procedure Call
5800 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
5830 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
5850 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
5874 unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
5920 // AArch64 shifts larger than the register width are wrapped rather than
5975 // AArch64 shifts of larger than register sizes are wrapped rather than
6031 // AArch64 Optimization Hooks
6072 // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
6111 // AArch64 reciprocal iteration instruction: (2 - M * N)
6126 // AArch64 Inline Assembly Support
6266 return std::make_pair(0U, &AArch64::GPR64commonRegClass);
6267 return std::make_pair(0U, &AArch64::GPR32commonRegClass);
6272 return std::make_pair(0U, &AArch64::ZPRRegClass);
6274 return std::make_pair(0U, &AArch64::FPR16RegClass);
6276 return std::make_pair(0U, &AArch64::FPR32RegClass);
6278 return std::make_pair(0U, &AArch64::FPR64RegClass);
6280 return std::make_pair(0U, &AArch64::FPR128RegClass);
6288 return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
6290 return std::make_pair(0U, &AArch64::FPR128_loRegClass);
6296 return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
6304 return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
6305 : std::make_pair(0U, &AArch64::PPRRegClass);
6309 return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
6328 Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
6329 Res.second = &AArch64::FPR64RegClass;
6331 Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
6332 Res.second = &AArch64::FPR128RegClass;
6339 !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
6340 !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
6370 Result = DAG.getRegister(AArch64::XZR, MVT::i64);
6372 Result = DAG.getRegister(AArch64::WZR, MVT::i32);
6501 // AArch64 Advanced SIMD Support
6533 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
8506 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
8623 Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
8626 Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
8653 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
8659 Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
8668 SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
8674 Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
9467 // AArch64 has five basic addressing modes:
9570 AArch64::X16, AArch64::X17, AArch64::LR, 0
10362 // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
10474 // AArch64 high-vector "long" operations are formed by performing the non-high
10540 /// Helper structure to keep track of a SET_CC lowered into AArch64 code.
10549 AArch64SetCCInfo AArch64;
10562 /// AArch64 lowered one.
10583 SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
10585 SetCCInfo.Info.AArch64.CC = static_cast<AArch64CC::CondCode>(
10602 SetCCInfo.Info.AArch64.CC =
10603 AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC);
10637 ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
10647 AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
10649 Cmp = *InfoAndKind.Info.AArch64.Cmp;
10861 AArch64::NeonBitsPerVector / VT.getSizeInBits());
10880 if (VT.getSizeInBits().getKnownMinSize() != AArch64::SVEBitsPerBlock)
11127 // This is effectively a custom type legalization for AArch64.
11141 // For AArch64, the [sz]ext vector instructions can only go up one element
11347 ZeroReg = AArch64::WZR;
11350 ZeroReg = AArch64::XZR;
11545 Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
11720 Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
11857 // when it is safe to remove the AND. Unfortunately it only runs on AArch64 and
12371 if (SrcVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
12436 if (RetVT.getSizeInBits().getKnownMinSize() > AArch64::SVEBitsPerBlock)
12808 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
12847 DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
12848 SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
12849 SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
12877 Opcode = AArch64::CASPX;
12880 Opcode = AArch64::CASPAX;
12883 Opcode = AArch64::CASPLX;
12887 Opcode = AArch64::CASPALX;
12897 unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
12913 AArch64::CMP_SWAP_128, SDLoc(N),
12971 // Non-volatile loads are optimized later in AArch64's load/store
13320 if (AArch64::GPR64RegClass.contains(*I))
13321 RC = &AArch64::GPR64RegClass;
13322 else if (AArch64::FPR64RegClass.contains(*I))
13323 RC = &AArch64::FPR64RegClass;
13349 // Integer division on AArch64 is expensive. However, when aggressively
13352 // The exception to this is vector division. Since AArch64 doesn't have vector