1//===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9/// \file 10/// This file implements the lowering of LLVM calls to machine code calls for 11/// GlobalISel. 12// 13//===----------------------------------------------------------------------===// 14 15#include "X86CallLowering.h" 16#include "X86CallingConv.h" 17#include "X86ISelLowering.h" 18#include "X86InstrInfo.h" 19#include "X86RegisterInfo.h" 20#include "X86Subtarget.h" 21#include "llvm/ADT/ArrayRef.h" 22#include "llvm/ADT/SmallVector.h" 23#include "llvm/CodeGen/Analysis.h" 24#include "llvm/CodeGen/CallingConvLower.h" 25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26#include "llvm/CodeGen/GlobalISel/Utils.h" 27#include "llvm/CodeGen/LowLevelType.h" 28#include "llvm/CodeGen/MachineBasicBlock.h" 29#include "llvm/CodeGen/MachineFrameInfo.h" 30#include "llvm/CodeGen/MachineFunction.h" 31#include "llvm/CodeGen/MachineInstrBuilder.h" 32#include "llvm/CodeGen/MachineMemOperand.h" 33#include "llvm/CodeGen/MachineOperand.h" 34#include "llvm/CodeGen/MachineRegisterInfo.h" 35#include "llvm/CodeGen/TargetInstrInfo.h" 36#include "llvm/CodeGen/TargetSubtargetInfo.h" 37#include "llvm/CodeGen/ValueTypes.h" 38#include "llvm/IR/Attributes.h" 39#include "llvm/IR/DataLayout.h" 40#include "llvm/IR/Function.h" 41#include "llvm/IR/Value.h" 42#include "llvm/MC/MCRegisterInfo.h" 43#include "llvm/Support/LowLevelTypeImpl.h" 44#include "llvm/Support/MachineValueType.h" 45#include <cassert> 46#include <cstdint> 47 48using namespace llvm; 49 50X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 51 : CallLowering(&TLI) {} 52 53bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 54 SmallVectorImpl<ArgInfo> &SplitArgs, 55 const DataLayout &DL, 56 MachineRegisterInfo &MRI, 57 SplitArgTy PerformArgSplit) const { 58 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 59 LLVMContext &Context = OrigArg.Ty->getContext(); 60 61 SmallVector<EVT, 4> SplitVTs; 62 SmallVector<uint64_t, 4> Offsets; 63 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 64 assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); 65 66 if (OrigArg.Ty->isVoidTy()) 67 return true; 68 69 EVT VT = SplitVTs[0]; 70 unsigned NumParts = TLI.getNumRegisters(Context, VT); 71 72 if (NumParts == 1) { 73 // replace the original type ( pointer -> GPR ). 74 SplitArgs.emplace_back(OrigArg.Regs[0], VT.getTypeForEVT(Context), 75 OrigArg.Flags, OrigArg.IsFixed); 76 return true; 77 } 78 79 SmallVector<Register, 8> SplitRegs; 80 81 EVT PartVT = TLI.getRegisterType(Context, VT); 82 Type *PartTy = PartVT.getTypeForEVT(Context); 83 84 for (unsigned i = 0; i < NumParts; ++i) { 85 ArgInfo Info = 86 ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), 87 PartTy, OrigArg.Flags}; 88 SplitArgs.push_back(Info); 89 SplitRegs.push_back(Info.Regs[0]); 90 } 91 92 PerformArgSplit(SplitRegs); 93 return true; 94} 95 96namespace { 97 98struct OutgoingValueHandler : public CallLowering::ValueHandler { 99 OutgoingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 100 MachineInstrBuilder &MIB, CCAssignFn *AssignFn) 101 : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), 102 DL(MIRBuilder.getMF().getDataLayout()), 103 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 104 105 bool isIncomingArgumentHandler() const override { return false; } 106 107 Register getStackAddress(uint64_t Size, int64_t Offset, 108 MachinePointerInfo &MPO) override { 109 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 110 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 111 Register SPReg = MRI.createGenericVirtualRegister(p0); 112 MIRBuilder.buildCopy(SPReg, STI.getRegisterInfo()->getStackRegister()); 113 114 Register OffsetReg = MRI.createGenericVirtualRegister(SType); 115 MIRBuilder.buildConstant(OffsetReg, Offset); 116 117 Register AddrReg = MRI.createGenericVirtualRegister(p0); 118 MIRBuilder.buildPtrAdd(AddrReg, SPReg, OffsetReg); 119 120 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 121 return AddrReg; 122 } 123 124 void assignValueToReg(Register ValVReg, Register PhysReg, 125 CCValAssign &VA) override { 126 MIB.addUse(PhysReg, RegState::Implicit); 127 128 Register ExtReg; 129 // If we are copying the value to a physical register with the 130 // size larger than the size of the value itself - build AnyExt 131 // to the size of the register first and only then do the copy. 132 // The example of that would be copying from s32 to xmm0, for which 133 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 134 // we expect normal extendRegister mechanism to work. 135 unsigned PhysRegSize = 136 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 137 unsigned ValSize = VA.getValVT().getSizeInBits(); 138 unsigned LocSize = VA.getLocVT().getSizeInBits(); 139 if (PhysRegSize > ValSize && LocSize == ValSize) { 140 assert((PhysRegSize == 128 || PhysRegSize == 80) && "We expect that to be 128 bit"); 141 auto MIB = MIRBuilder.buildAnyExt(LLT::scalar(PhysRegSize), ValVReg); 142 ExtReg = MIB->getOperand(0).getReg(); 143 } else 144 ExtReg = extendRegister(ValVReg, VA); 145 146 MIRBuilder.buildCopy(PhysReg, ExtReg); 147 } 148 149 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 150 MachinePointerInfo &MPO, CCValAssign &VA) override { 151 Register ExtReg = extendRegister(ValVReg, VA); 152 auto MMO = MIRBuilder.getMF().getMachineMemOperand( 153 MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(), 154 /* Alignment */ 1); 155 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 156 } 157 158 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, 159 CCValAssign::LocInfo LocInfo, 160 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 161 CCState &State) override { 162 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 163 StackSize = State.getNextStackOffset(); 164 165 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 166 X86::XMM3, X86::XMM4, X86::XMM5, 167 X86::XMM6, X86::XMM7}; 168 if (!Info.IsFixed) 169 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 170 171 return Res; 172 } 173 174 uint64_t getStackSize() { return StackSize; } 175 uint64_t getNumXmmRegs() { return NumXMMRegs; } 176 177protected: 178 MachineInstrBuilder &MIB; 179 uint64_t StackSize = 0; 180 const DataLayout &DL; 181 const X86Subtarget &STI; 182 unsigned NumXMMRegs = 0; 183}; 184 185} // end anonymous namespace 186 187bool X86CallLowering::lowerReturn( 188 MachineIRBuilder &MIRBuilder, const Value *Val, 189 ArrayRef<Register> VRegs) const { 190 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 191 "Return value without a vreg"); 192 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 193 194 if (!VRegs.empty()) { 195 MachineFunction &MF = MIRBuilder.getMF(); 196 const Function &F = MF.getFunction(); 197 MachineRegisterInfo &MRI = MF.getRegInfo(); 198 auto &DL = MF.getDataLayout(); 199 LLVMContext &Ctx = Val->getType()->getContext(); 200 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 201 202 SmallVector<EVT, 4> SplitEVTs; 203 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs); 204 assert(VRegs.size() == SplitEVTs.size() && 205 "For each split Type there should be exactly one VReg."); 206 207 SmallVector<ArgInfo, 8> SplitArgs; 208 for (unsigned i = 0; i < SplitEVTs.size(); ++i) { 209 ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)}; 210 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); 211 if (!splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, 212 [&](ArrayRef<Register> Regs) { 213 MIRBuilder.buildUnmerge(Regs, VRegs[i]); 214 })) 215 return false; 216 } 217 218 OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); 219 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 220 return false; 221 } 222 223 MIRBuilder.insertInstr(MIB); 224 return true; 225} 226 227namespace { 228 229struct IncomingValueHandler : public CallLowering::ValueHandler { 230 IncomingValueHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 231 CCAssignFn *AssignFn) 232 : ValueHandler(MIRBuilder, MRI, AssignFn), 233 DL(MIRBuilder.getMF().getDataLayout()) {} 234 235 bool isIncomingArgumentHandler() const override { return true; } 236 237 Register getStackAddress(uint64_t Size, int64_t Offset, 238 MachinePointerInfo &MPO) override { 239 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 240 int FI = MFI.CreateFixedObject(Size, Offset, true); 241 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 242 243 Register AddrReg = MRI.createGenericVirtualRegister( 244 LLT::pointer(0, DL.getPointerSizeInBits(0))); 245 MIRBuilder.buildFrameIndex(AddrReg, FI); 246 return AddrReg; 247 } 248 249 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 250 MachinePointerInfo &MPO, CCValAssign &VA) override { 251 auto MMO = MIRBuilder.getMF().getMachineMemOperand( 252 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 253 1); 254 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 255 } 256 257 void assignValueToReg(Register ValVReg, Register PhysReg, 258 CCValAssign &VA) override { 259 markPhysRegUsed(PhysReg); 260 261 switch (VA.getLocInfo()) { 262 default: { 263 // If we are copying the value from a physical register with the 264 // size larger than the size of the value itself - build the copy 265 // of the phys reg first and then build the truncation of that copy. 266 // The example of that would be copying from xmm0 to s32, for which 267 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 268 // we expect this to be handled in SExt/ZExt/AExt case. 269 unsigned PhysRegSize = 270 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 271 unsigned ValSize = VA.getValVT().getSizeInBits(); 272 unsigned LocSize = VA.getLocVT().getSizeInBits(); 273 if (PhysRegSize > ValSize && LocSize == ValSize) { 274 auto Copy = MIRBuilder.buildCopy(LLT::scalar(PhysRegSize), PhysReg); 275 MIRBuilder.buildTrunc(ValVReg, Copy); 276 return; 277 } 278 279 MIRBuilder.buildCopy(ValVReg, PhysReg); 280 break; 281 } 282 case CCValAssign::LocInfo::SExt: 283 case CCValAssign::LocInfo::ZExt: 284 case CCValAssign::LocInfo::AExt: { 285 auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg); 286 MIRBuilder.buildTrunc(ValVReg, Copy); 287 break; 288 } 289 } 290 } 291 292 /// How the physical register gets marked varies between formal 293 /// parameters (it's a basic-block live-in), and a call instruction 294 /// (it's an implicit-def of the BL). 295 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 296 297protected: 298 const DataLayout &DL; 299}; 300 301struct FormalArgHandler : public IncomingValueHandler { 302 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 303 CCAssignFn *AssignFn) 304 : IncomingValueHandler(MIRBuilder, MRI, AssignFn) {} 305 306 void markPhysRegUsed(unsigned PhysReg) override { 307 MIRBuilder.getMRI()->addLiveIn(PhysReg); 308 MIRBuilder.getMBB().addLiveIn(PhysReg); 309 } 310}; 311 312struct CallReturnHandler : public IncomingValueHandler { 313 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 314 CCAssignFn *AssignFn, MachineInstrBuilder &MIB) 315 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {} 316 317 void markPhysRegUsed(unsigned PhysReg) override { 318 MIB.addDef(PhysReg, RegState::Implicit); 319 } 320 321protected: 322 MachineInstrBuilder &MIB; 323}; 324 325} // end anonymous namespace 326 327bool X86CallLowering::lowerFormalArguments( 328 MachineIRBuilder &MIRBuilder, const Function &F, 329 ArrayRef<ArrayRef<Register>> VRegs) const { 330 if (F.arg_empty()) 331 return true; 332 333 // TODO: handle variadic function 334 if (F.isVarArg()) 335 return false; 336 337 MachineFunction &MF = MIRBuilder.getMF(); 338 MachineRegisterInfo &MRI = MF.getRegInfo(); 339 auto DL = MF.getDataLayout(); 340 341 SmallVector<ArgInfo, 8> SplitArgs; 342 unsigned Idx = 0; 343 for (auto &Arg : F.args()) { 344 345 // TODO: handle not simple cases. 346 if (Arg.hasAttribute(Attribute::ByVal) || 347 Arg.hasAttribute(Attribute::InReg) || 348 Arg.hasAttribute(Attribute::StructRet) || 349 Arg.hasAttribute(Attribute::SwiftSelf) || 350 Arg.hasAttribute(Attribute::SwiftError) || 351 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 352 return false; 353 354 ArgInfo OrigArg(VRegs[Idx], Arg.getType()); 355 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 356 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 357 [&](ArrayRef<Register> Regs) { 358 MIRBuilder.buildMerge(VRegs[Idx][0], Regs); 359 })) 360 return false; 361 Idx++; 362 } 363 364 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 365 if (!MBB.empty()) 366 MIRBuilder.setInstr(*MBB.begin()); 367 368 FormalArgHandler Handler(MIRBuilder, MRI, CC_X86); 369 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 370 return false; 371 372 // Move back to the end of the basic block. 373 MIRBuilder.setMBB(MBB); 374 375 return true; 376} 377 378bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 379 CallLoweringInfo &Info) const { 380 MachineFunction &MF = MIRBuilder.getMF(); 381 const Function &F = MF.getFunction(); 382 MachineRegisterInfo &MRI = MF.getRegInfo(); 383 auto &DL = F.getParent()->getDataLayout(); 384 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 385 const TargetInstrInfo &TII = *STI.getInstrInfo(); 386 auto TRI = STI.getRegisterInfo(); 387 388 // Handle only Linux C, X86_64_SysV calling conventions for now. 389 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 390 Info.CallConv == CallingConv::X86_64_SysV)) 391 return false; 392 393 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 394 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 395 396 // Create a temporarily-floating call instruction so we can add the implicit 397 // uses of arg registers. 398 bool Is64Bit = STI.is64Bit(); 399 unsigned CallOpc = Info.Callee.isReg() 400 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 401 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 402 403 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 404 .add(Info.Callee) 405 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 406 407 SmallVector<ArgInfo, 8> SplitArgs; 408 for (const auto &OrigArg : Info.OrigArgs) { 409 410 // TODO: handle not simple cases. 411 if (OrigArg.Flags[0].isByVal()) 412 return false; 413 414 if (OrigArg.Regs.size() > 1) 415 return false; 416 417 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 418 [&](ArrayRef<Register> Regs) { 419 MIRBuilder.buildUnmerge(Regs, OrigArg.Regs[0]); 420 })) 421 return false; 422 } 423 // Do the actual argument marshalling. 424 OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86); 425 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 426 return false; 427 428 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 429 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 430 // From AMD64 ABI document: 431 // For calls that may call functions that use varargs or stdargs 432 // (prototype-less calls or calls to functions containing ellipsis (...) in 433 // the declaration) %al is used as hidden argument to specify the number 434 // of SSE registers used. The contents of %al do not need to match exactly 435 // the number of registers, but must be an ubound on the number of SSE 436 // registers used and is in the range 0 - 8 inclusive. 437 438 MIRBuilder.buildInstr(X86::MOV8ri) 439 .addDef(X86::AL) 440 .addImm(Handler.getNumXmmRegs()); 441 MIB.addUse(X86::AL, RegState::Implicit); 442 } 443 444 // Now we can add the actual call instruction to the correct basic block. 445 MIRBuilder.insertInstr(MIB); 446 447 // If Callee is a reg, since it is used by a target specific 448 // instruction, it must have a register class matching the 449 // constraint of that instruction. 450 if (Info.Callee.isReg()) 451 MIB->getOperand(0).setReg(constrainOperandRegClass( 452 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 453 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 454 0)); 455 456 // Finally we can copy the returned value back into its virtual-register. In 457 // symmetry with the arguments, the physical register must be an 458 // implicit-define of the call instruction. 459 460 if (!Info.OrigRet.Ty->isVoidTy()) { 461 if (Info.OrigRet.Regs.size() > 1) 462 return false; 463 464 SplitArgs.clear(); 465 SmallVector<Register, 8> NewRegs; 466 467 if (!splitToValueTypes(Info.OrigRet, SplitArgs, DL, MRI, 468 [&](ArrayRef<Register> Regs) { 469 NewRegs.assign(Regs.begin(), Regs.end()); 470 })) 471 return false; 472 473 CallReturnHandler Handler(MIRBuilder, MRI, RetCC_X86, MIB); 474 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 475 return false; 476 477 if (!NewRegs.empty()) 478 MIRBuilder.buildMerge(Info.OrigRet.Regs[0], NewRegs); 479 } 480 481 CallSeqStart.addImm(Handler.getStackSize()) 482 .addImm(0 /* see getFrameTotalSize */) 483 .addImm(0 /* see getFrameAdjustment */); 484 485 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 486 MIRBuilder.buildInstr(AdjStackUp) 487 .addImm(Handler.getStackSize()) 488 .addImm(0 /* NumBytesForCalleeToPop */); 489 490 return true; 491} 492