SystemZISelLowering.cpp revision 251607
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "systemz-lower" 15 16#include "SystemZISelLowering.h" 17#include "SystemZCallingConv.h" 18#include "SystemZConstantPoolValue.h" 19#include "SystemZMachineFunctionInfo.h" 20#include "SystemZTargetMachine.h" 21#include "llvm/CodeGen/CallingConvLower.h" 22#include "llvm/CodeGen/MachineInstrBuilder.h" 23#include "llvm/CodeGen/MachineRegisterInfo.h" 24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 25 26using namespace llvm; 27 28// Classify VT as either 32 or 64 bit. 29static bool is32Bit(EVT VT) { 30 switch (VT.getSimpleVT().SimpleTy) { 31 case MVT::i32: 32 return true; 33 case MVT::i64: 34 return false; 35 default: 36 llvm_unreachable("Unsupported type"); 37 } 38} 39 40// Return a version of MachineOperand that can be safely used before the 41// final use. 42static MachineOperand earlyUseOperand(MachineOperand Op) { 43 if (Op.isReg()) 44 Op.setIsKill(false); 45 return Op; 46} 47 48SystemZTargetLowering::SystemZTargetLowering(SystemZTargetMachine &tm) 49 : TargetLowering(tm, new TargetLoweringObjectFileELF()), 50 Subtarget(*tm.getSubtargetImpl()), TM(tm) { 51 MVT PtrVT = getPointerTy(); 52 53 // Set up the register classes. 54 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 55 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 56 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 57 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 58 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 59 60 // Compute derived properties from the register classes 61 computeRegisterProperties(); 62 63 // Set up special registers. 64 setExceptionPointerRegister(SystemZ::R6D); 65 setExceptionSelectorRegister(SystemZ::R7D); 66 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 67 68 // TODO: It may be better to default to latency-oriented scheduling, however 69 // LLVM's current latency-oriented scheduler can't handle physreg definitions 70 // such as SystemZ has with PSW, so set this to the register-pressure 71 // scheduler, because it can. 72 setSchedulingPreference(Sched::RegPressure); 73 74 setBooleanContents(ZeroOrOneBooleanContent); 75 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 76 77 // Instructions are strings of 2-byte aligned 2-byte values. 78 setMinFunctionAlignment(2); 79 80 // Handle operations that are handled in a similar way for all types. 81 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 82 I <= MVT::LAST_FP_VALUETYPE; 83 ++I) { 84 MVT VT = MVT::SimpleValueType(I); 85 if (isTypeLegal(VT)) { 86 // Expand SETCC(X, Y, COND) into SELECT_CC(X, Y, 1, 0, COND). 87 setOperationAction(ISD::SETCC, VT, Expand); 88 89 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 90 setOperationAction(ISD::SELECT, VT, Expand); 91 92 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 93 setOperationAction(ISD::SELECT_CC, VT, Custom); 94 setOperationAction(ISD::BR_CC, VT, Custom); 95 } 96 } 97 98 // Expand jump table branches as address arithmetic followed by an 99 // indirect jump. 100 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 101 102 // Expand BRCOND into a BR_CC (see above). 103 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 104 105 // Handle integer types. 106 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 107 I <= MVT::LAST_INTEGER_VALUETYPE; 108 ++I) { 109 MVT VT = MVT::SimpleValueType(I); 110 if (isTypeLegal(VT)) { 111 // Expand individual DIV and REMs into DIVREMs. 112 setOperationAction(ISD::SDIV, VT, Expand); 113 setOperationAction(ISD::UDIV, VT, Expand); 114 setOperationAction(ISD::SREM, VT, Expand); 115 setOperationAction(ISD::UREM, VT, Expand); 116 setOperationAction(ISD::SDIVREM, VT, Custom); 117 setOperationAction(ISD::UDIVREM, VT, Custom); 118 119 // Expand ATOMIC_LOAD and ATOMIC_STORE using ATOMIC_CMP_SWAP. 120 // FIXME: probably much too conservative. 121 setOperationAction(ISD::ATOMIC_LOAD, VT, Expand); 122 setOperationAction(ISD::ATOMIC_STORE, VT, Expand); 123 124 // No special instructions for these. 125 setOperationAction(ISD::CTPOP, VT, Expand); 126 setOperationAction(ISD::CTTZ, VT, Expand); 127 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 128 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 129 setOperationAction(ISD::ROTR, VT, Expand); 130 131 // Use *MUL_LOHI where possible and a wider multiplication otherwise. 132 setOperationAction(ISD::MULHS, VT, Expand); 133 setOperationAction(ISD::MULHU, VT, Expand); 134 135 // We have instructions for signed but not unsigned FP conversion. 136 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 137 } 138 } 139 140 // Type legalization will convert 8- and 16-bit atomic operations into 141 // forms that operate on i32s (but still keeping the original memory VT). 142 // Lower them into full i32 operations. 143 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 144 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 145 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 146 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 147 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 148 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 149 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 150 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 151 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 152 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 153 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 154 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 155 156 // We have instructions for signed but not unsigned FP conversion. 157 // Handle unsigned 32-bit types as signed 64-bit types. 158 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 159 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 160 161 // We have native support for a 64-bit CTLZ, via FLOGR. 162 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 163 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 164 165 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 166 setOperationAction(ISD::OR, MVT::i64, Custom); 167 168 // The architecture has 32-bit SMUL_LOHI and UMUL_LOHI (MR and MLR), 169 // but they aren't really worth using. There is no 64-bit SMUL_LOHI, 170 // but there is a 64-bit UMUL_LOHI: MLGR. 171 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); 172 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 173 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); 174 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Custom); 175 176 // FIXME: Can we support these natively? 177 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 178 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 179 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 180 181 // We have native instructions for i8, i16 and i32 extensions, but not i1. 182 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote); 183 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote); 184 setLoadExtAction(ISD::EXTLOAD, MVT::i1, Promote); 185 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 186 187 // Handle the various types of symbolic address. 188 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 189 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 190 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 191 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 192 setOperationAction(ISD::JumpTable, PtrVT, Custom); 193 194 // We need to handle dynamic allocations specially because of the 195 // 160-byte area at the bottom of the stack. 196 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 197 198 // Use custom expanders so that we can force the function to use 199 // a frame pointer. 200 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 201 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 202 203 // Expand these using getExceptionSelectorRegister() and 204 // getExceptionPointerRegister(). 205 setOperationAction(ISD::EXCEPTIONADDR, PtrVT, Expand); 206 setOperationAction(ISD::EHSELECTION, PtrVT, Expand); 207 208 // Handle floating-point types. 209 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 210 I <= MVT::LAST_FP_VALUETYPE; 211 ++I) { 212 MVT VT = MVT::SimpleValueType(I); 213 if (isTypeLegal(VT)) { 214 // We can use FI for FRINT. 215 setOperationAction(ISD::FRINT, VT, Legal); 216 217 // No special instructions for these. 218 setOperationAction(ISD::FSIN, VT, Expand); 219 setOperationAction(ISD::FCOS, VT, Expand); 220 setOperationAction(ISD::FREM, VT, Expand); 221 } 222 } 223 224 // We have fused multiply-addition for f32 and f64 but not f128. 225 setOperationAction(ISD::FMA, MVT::f32, Legal); 226 setOperationAction(ISD::FMA, MVT::f64, Legal); 227 setOperationAction(ISD::FMA, MVT::f128, Expand); 228 229 // Needed so that we don't try to implement f128 constant loads using 230 // a load-and-extend of a f80 constant (in cases where the constant 231 // would fit in an f80). 232 setLoadExtAction(ISD::EXTLOAD, MVT::f80, Expand); 233 234 // Floating-point truncation and stores need to be done separately. 235 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 236 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 237 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 238 239 // We have 64-bit FPR<->GPR moves, but need special handling for 240 // 32-bit forms. 241 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 242 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 243 244 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 245 // structure, but VAEND is a no-op. 246 setOperationAction(ISD::VASTART, MVT::Other, Custom); 247 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 248 setOperationAction(ISD::VAEND, MVT::Other, Expand); 249} 250 251bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 252 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 253 return Imm.isZero() || Imm.isNegZero(); 254} 255 256//===----------------------------------------------------------------------===// 257// Inline asm support 258//===----------------------------------------------------------------------===// 259 260TargetLowering::ConstraintType 261SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 262 if (Constraint.size() == 1) { 263 switch (Constraint[0]) { 264 case 'a': // Address register 265 case 'd': // Data register (equivalent to 'r') 266 case 'f': // Floating-point register 267 case 'r': // General-purpose register 268 return C_RegisterClass; 269 270 case 'Q': // Memory with base and unsigned 12-bit displacement 271 case 'R': // Likewise, plus an index 272 case 'S': // Memory with base and signed 20-bit displacement 273 case 'T': // Likewise, plus an index 274 case 'm': // Equivalent to 'T'. 275 return C_Memory; 276 277 case 'I': // Unsigned 8-bit constant 278 case 'J': // Unsigned 12-bit constant 279 case 'K': // Signed 16-bit constant 280 case 'L': // Signed 20-bit displacement (on all targets we support) 281 case 'M': // 0x7fffffff 282 return C_Other; 283 284 default: 285 break; 286 } 287 } 288 return TargetLowering::getConstraintType(Constraint); 289} 290 291TargetLowering::ConstraintWeight SystemZTargetLowering:: 292getSingleConstraintMatchWeight(AsmOperandInfo &info, 293 const char *constraint) const { 294 ConstraintWeight weight = CW_Invalid; 295 Value *CallOperandVal = info.CallOperandVal; 296 // If we don't have a value, we can't do a match, 297 // but allow it at the lowest weight. 298 if (CallOperandVal == NULL) 299 return CW_Default; 300 Type *type = CallOperandVal->getType(); 301 // Look at the constraint type. 302 switch (*constraint) { 303 default: 304 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 305 break; 306 307 case 'a': // Address register 308 case 'd': // Data register (equivalent to 'r') 309 case 'r': // General-purpose register 310 if (CallOperandVal->getType()->isIntegerTy()) 311 weight = CW_Register; 312 break; 313 314 case 'f': // Floating-point register 315 if (type->isFloatingPointTy()) 316 weight = CW_Register; 317 break; 318 319 case 'I': // Unsigned 8-bit constant 320 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 321 if (isUInt<8>(C->getZExtValue())) 322 weight = CW_Constant; 323 break; 324 325 case 'J': // Unsigned 12-bit constant 326 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 327 if (isUInt<12>(C->getZExtValue())) 328 weight = CW_Constant; 329 break; 330 331 case 'K': // Signed 16-bit constant 332 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 333 if (isInt<16>(C->getSExtValue())) 334 weight = CW_Constant; 335 break; 336 337 case 'L': // Signed 20-bit displacement (on all targets we support) 338 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 339 if (isInt<20>(C->getSExtValue())) 340 weight = CW_Constant; 341 break; 342 343 case 'M': // 0x7fffffff 344 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) 345 if (C->getZExtValue() == 0x7fffffff) 346 weight = CW_Constant; 347 break; 348 } 349 return weight; 350} 351 352std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 353getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const { 354 if (Constraint.size() == 1) { 355 // GCC Constraint Letters 356 switch (Constraint[0]) { 357 default: break; 358 case 'd': // Data register (equivalent to 'r') 359 case 'r': // General-purpose register 360 if (VT == MVT::i64) 361 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 362 else if (VT == MVT::i128) 363 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 364 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 365 366 case 'a': // Address register 367 if (VT == MVT::i64) 368 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 369 else if (VT == MVT::i128) 370 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 371 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 372 373 case 'f': // Floating-point register 374 if (VT == MVT::f64) 375 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 376 else if (VT == MVT::f128) 377 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 378 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 379 } 380 } 381 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 382} 383 384void SystemZTargetLowering:: 385LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 386 std::vector<SDValue> &Ops, 387 SelectionDAG &DAG) const { 388 // Only support length 1 constraints for now. 389 if (Constraint.length() == 1) { 390 switch (Constraint[0]) { 391 case 'I': // Unsigned 8-bit constant 392 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 393 if (isUInt<8>(C->getZExtValue())) 394 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 395 Op.getValueType())); 396 return; 397 398 case 'J': // Unsigned 12-bit constant 399 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 400 if (isUInt<12>(C->getZExtValue())) 401 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 402 Op.getValueType())); 403 return; 404 405 case 'K': // Signed 16-bit constant 406 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 407 if (isInt<16>(C->getSExtValue())) 408 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 409 Op.getValueType())); 410 return; 411 412 case 'L': // Signed 20-bit displacement (on all targets we support) 413 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 414 if (isInt<20>(C->getSExtValue())) 415 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 416 Op.getValueType())); 417 return; 418 419 case 'M': // 0x7fffffff 420 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) 421 if (C->getZExtValue() == 0x7fffffff) 422 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 423 Op.getValueType())); 424 return; 425 } 426 } 427 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 428} 429 430//===----------------------------------------------------------------------===// 431// Calling conventions 432//===----------------------------------------------------------------------===// 433 434#include "SystemZGenCallingConv.inc" 435 436// Value is a value that has been passed to us in the location described by VA 437// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 438// any loads onto Chain. 439static SDValue convertLocVTToValVT(SelectionDAG &DAG, DebugLoc DL, 440 CCValAssign &VA, SDValue Chain, 441 SDValue Value) { 442 // If the argument has been promoted from a smaller type, insert an 443 // assertion to capture this. 444 if (VA.getLocInfo() == CCValAssign::SExt) 445 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 446 DAG.getValueType(VA.getValVT())); 447 else if (VA.getLocInfo() == CCValAssign::ZExt) 448 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 449 DAG.getValueType(VA.getValVT())); 450 451 if (VA.isExtInLoc()) 452 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 453 else if (VA.getLocInfo() == CCValAssign::Indirect) 454 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 455 MachinePointerInfo(), false, false, false, 0); 456 else 457 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 458 return Value; 459} 460 461// Value is a value of type VA.getValVT() that we need to copy into 462// the location described by VA. Return a copy of Value converted to 463// VA.getValVT(). The caller is responsible for handling indirect values. 464static SDValue convertValVTToLocVT(SelectionDAG &DAG, DebugLoc DL, 465 CCValAssign &VA, SDValue Value) { 466 switch (VA.getLocInfo()) { 467 case CCValAssign::SExt: 468 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 469 case CCValAssign::ZExt: 470 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 471 case CCValAssign::AExt: 472 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 473 case CCValAssign::Full: 474 return Value; 475 default: 476 llvm_unreachable("Unhandled getLocInfo()"); 477 } 478} 479 480SDValue SystemZTargetLowering:: 481LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 482 const SmallVectorImpl<ISD::InputArg> &Ins, 483 DebugLoc DL, SelectionDAG &DAG, 484 SmallVectorImpl<SDValue> &InVals) const { 485 MachineFunction &MF = DAG.getMachineFunction(); 486 MachineFrameInfo *MFI = MF.getFrameInfo(); 487 MachineRegisterInfo &MRI = MF.getRegInfo(); 488 SystemZMachineFunctionInfo *FuncInfo = 489 MF.getInfo<SystemZMachineFunctionInfo>(); 490 const SystemZFrameLowering *TFL = 491 static_cast<const SystemZFrameLowering *>(TM.getFrameLowering()); 492 493 // Assign locations to all of the incoming arguments. 494 SmallVector<CCValAssign, 16> ArgLocs; 495 CCState CCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 496 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 497 498 unsigned NumFixedGPRs = 0; 499 unsigned NumFixedFPRs = 0; 500 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 501 SDValue ArgValue; 502 CCValAssign &VA = ArgLocs[I]; 503 EVT LocVT = VA.getLocVT(); 504 if (VA.isRegLoc()) { 505 // Arguments passed in registers 506 const TargetRegisterClass *RC; 507 switch (LocVT.getSimpleVT().SimpleTy) { 508 default: 509 // Integers smaller than i64 should be promoted to i64. 510 llvm_unreachable("Unexpected argument type"); 511 case MVT::i32: 512 NumFixedGPRs += 1; 513 RC = &SystemZ::GR32BitRegClass; 514 break; 515 case MVT::i64: 516 NumFixedGPRs += 1; 517 RC = &SystemZ::GR64BitRegClass; 518 break; 519 case MVT::f32: 520 NumFixedFPRs += 1; 521 RC = &SystemZ::FP32BitRegClass; 522 break; 523 case MVT::f64: 524 NumFixedFPRs += 1; 525 RC = &SystemZ::FP64BitRegClass; 526 break; 527 } 528 529 unsigned VReg = MRI.createVirtualRegister(RC); 530 MRI.addLiveIn(VA.getLocReg(), VReg); 531 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 532 } else { 533 assert(VA.isMemLoc() && "Argument not register or memory"); 534 535 // Create the frame index object for this incoming parameter. 536 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 537 VA.getLocMemOffset(), true); 538 539 // Create the SelectionDAG nodes corresponding to a load 540 // from this parameter. Unpromoted ints and floats are 541 // passed as right-justified 8-byte values. 542 EVT PtrVT = getPointerTy(); 543 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 544 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 545 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 546 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 547 MachinePointerInfo::getFixedStack(FI), 548 false, false, false, 0); 549 } 550 551 // Convert the value of the argument register into the value that's 552 // being passed. 553 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 554 } 555 556 if (IsVarArg) { 557 // Save the number of non-varargs registers for later use by va_start, etc. 558 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 559 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 560 561 // Likewise the address (in the form of a frame index) of where the 562 // first stack vararg would be. The 1-byte size here is arbitrary. 563 int64_t StackSize = CCInfo.getNextStackOffset(); 564 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 565 566 // ...and a similar frame index for the caller-allocated save area 567 // that will be used to store the incoming registers. 568 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 569 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 570 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 571 572 // Store the FPR varargs in the reserved frame slots. (We store the 573 // GPRs as part of the prologue.) 574 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 575 SDValue MemOps[SystemZ::NumArgFPRs]; 576 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 577 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 578 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 579 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 580 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 581 &SystemZ::FP64BitRegClass); 582 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 583 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 584 MachinePointerInfo::getFixedStack(FI), 585 false, false, 0); 586 587 } 588 // Join the stores, which are independent of one another. 589 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 590 &MemOps[NumFixedFPRs], 591 SystemZ::NumArgFPRs - NumFixedFPRs); 592 } 593 } 594 595 return Chain; 596} 597 598SDValue 599SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 600 SmallVectorImpl<SDValue> &InVals) const { 601 SelectionDAG &DAG = CLI.DAG; 602 DebugLoc &DL = CLI.DL; 603 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; 604 SmallVector<SDValue, 32> &OutVals = CLI.OutVals; 605 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; 606 SDValue Chain = CLI.Chain; 607 SDValue Callee = CLI.Callee; 608 bool &isTailCall = CLI.IsTailCall; 609 CallingConv::ID CallConv = CLI.CallConv; 610 bool IsVarArg = CLI.IsVarArg; 611 MachineFunction &MF = DAG.getMachineFunction(); 612 EVT PtrVT = getPointerTy(); 613 614 // SystemZ target does not yet support tail call optimization. 615 isTailCall = false; 616 617 // Analyze the operands of the call, assigning locations to each operand. 618 SmallVector<CCValAssign, 16> ArgLocs; 619 CCState ArgCCInfo(CallConv, IsVarArg, MF, TM, ArgLocs, *DAG.getContext()); 620 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 621 622 // Get a count of how many bytes are to be pushed on the stack. 623 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 624 625 // Mark the start of the call. 626 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true)); 627 628 // Copy argument values to their designated locations. 629 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 630 SmallVector<SDValue, 8> MemOpChains; 631 SDValue StackPtr; 632 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 633 CCValAssign &VA = ArgLocs[I]; 634 SDValue ArgValue = OutVals[I]; 635 636 if (VA.getLocInfo() == CCValAssign::Indirect) { 637 // Store the argument in a stack slot and pass its address. 638 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 639 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 640 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 641 MachinePointerInfo::getFixedStack(FI), 642 false, false, 0)); 643 ArgValue = SpillSlot; 644 } else 645 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 646 647 if (VA.isRegLoc()) 648 // Queue up the argument copies and emit them at the end. 649 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 650 else { 651 assert(VA.isMemLoc() && "Argument not register or memory"); 652 653 // Work out the address of the stack slot. Unpromoted ints and 654 // floats are passed as right-justified 8-byte values. 655 if (!StackPtr.getNode()) 656 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 657 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 658 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 659 Offset += 4; 660 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 661 DAG.getIntPtrConstant(Offset)); 662 663 // Emit the store. 664 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 665 MachinePointerInfo(), 666 false, false, 0)); 667 } 668 } 669 670 // Join the stores, which are independent of one another. 671 if (!MemOpChains.empty()) 672 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 673 &MemOpChains[0], MemOpChains.size()); 674 675 // Build a sequence of copy-to-reg nodes, chained and glued together. 676 SDValue Glue; 677 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 678 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 679 RegsToPass[I].second, Glue); 680 Glue = Chain.getValue(1); 681 } 682 683 // Accept direct calls by converting symbolic call addresses to the 684 // associated Target* opcodes. 685 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 686 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 687 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 688 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 689 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 690 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 691 } 692 693 // The first call operand is the chain and the second is the target address. 694 SmallVector<SDValue, 8> Ops; 695 Ops.push_back(Chain); 696 Ops.push_back(Callee); 697 698 // Add argument registers to the end of the list so that they are 699 // known live into the call. 700 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 701 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 702 RegsToPass[I].second.getValueType())); 703 704 // Glue the call to the argument copies, if any. 705 if (Glue.getNode()) 706 Ops.push_back(Glue); 707 708 // Emit the call. 709 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 710 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, &Ops[0], Ops.size()); 711 Glue = Chain.getValue(1); 712 713 // Mark the end of the call, which is glued to the call itself. 714 Chain = DAG.getCALLSEQ_END(Chain, 715 DAG.getConstant(NumBytes, PtrVT, true), 716 DAG.getConstant(0, PtrVT, true), 717 Glue); 718 Glue = Chain.getValue(1); 719 720 // Assign locations to each value returned by this call. 721 SmallVector<CCValAssign, 16> RetLocs; 722 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 723 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 724 725 // Copy all of the result registers out of their specified physreg. 726 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 727 CCValAssign &VA = RetLocs[I]; 728 729 // Copy the value out, gluing the copy to the end of the call sequence. 730 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 731 VA.getLocVT(), Glue); 732 Chain = RetValue.getValue(1); 733 Glue = RetValue.getValue(2); 734 735 // Convert the value of the return register into the value that's 736 // being returned. 737 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 738 } 739 740 return Chain; 741} 742 743SDValue 744SystemZTargetLowering::LowerReturn(SDValue Chain, 745 CallingConv::ID CallConv, bool IsVarArg, 746 const SmallVectorImpl<ISD::OutputArg> &Outs, 747 const SmallVectorImpl<SDValue> &OutVals, 748 DebugLoc DL, SelectionDAG &DAG) const { 749 MachineFunction &MF = DAG.getMachineFunction(); 750 751 // Assign locations to each returned value. 752 SmallVector<CCValAssign, 16> RetLocs; 753 CCState RetCCInfo(CallConv, IsVarArg, MF, TM, RetLocs, *DAG.getContext()); 754 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 755 756 // Quick exit for void returns 757 if (RetLocs.empty()) 758 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 759 760 // Copy the result values into the output registers. 761 SDValue Glue; 762 SmallVector<SDValue, 4> RetOps; 763 RetOps.push_back(Chain); 764 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 765 CCValAssign &VA = RetLocs[I]; 766 SDValue RetValue = OutVals[I]; 767 768 // Make the return register live on exit. 769 assert(VA.isRegLoc() && "Can only return in registers!"); 770 771 // Promote the value as required. 772 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 773 774 // Chain and glue the copies together. 775 unsigned Reg = VA.getLocReg(); 776 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 777 Glue = Chain.getValue(1); 778 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 779 } 780 781 // Update chain and glue. 782 RetOps[0] = Chain; 783 if (Glue.getNode()) 784 RetOps.push_back(Glue); 785 786 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, 787 RetOps.data(), RetOps.size()); 788} 789 790// CC is a comparison that will be implemented using an integer or 791// floating-point comparison. Return the condition code mask for 792// a branch on true. In the integer case, CCMASK_CMP_UO is set for 793// unsigned comparisons and clear for signed ones. In the floating-point 794// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 795static unsigned CCMaskForCondCode(ISD::CondCode CC) { 796#define CONV(X) \ 797 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 798 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 799 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 800 801 switch (CC) { 802 default: 803 llvm_unreachable("Invalid integer condition!"); 804 805 CONV(EQ); 806 CONV(NE); 807 CONV(GT); 808 CONV(GE); 809 CONV(LT); 810 CONV(LE); 811 812 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 813 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 814 } 815#undef CONV 816} 817 818// If a comparison described by IsUnsigned, CCMask, CmpOp0 and CmpOp1 819// is suitable for CLI(Y), CHHSI or CLHHSI, adjust the operands as necessary. 820static void adjustSubwordCmp(SelectionDAG &DAG, bool &IsUnsigned, 821 SDValue &CmpOp0, SDValue &CmpOp1, 822 unsigned &CCMask) { 823 // For us to make any changes, it must a comparison between a single-use 824 // load and a constant. 825 if (!CmpOp0.hasOneUse() || 826 CmpOp0.getOpcode() != ISD::LOAD || 827 CmpOp1.getOpcode() != ISD::Constant) 828 return; 829 830 // We must have an 8- or 16-bit load. 831 LoadSDNode *Load = cast<LoadSDNode>(CmpOp0); 832 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 833 if (NumBits != 8 && NumBits != 16) 834 return; 835 836 // The load must be an extending one and the constant must be within the 837 // range of the unextended value. 838 ConstantSDNode *Constant = cast<ConstantSDNode>(CmpOp1); 839 uint64_t Value = Constant->getZExtValue(); 840 uint64_t Mask = (1 << NumBits) - 1; 841 if (Load->getExtensionType() == ISD::SEXTLOAD) { 842 int64_t SignedValue = Constant->getSExtValue(); 843 if (uint64_t(SignedValue) + (1 << (NumBits - 1)) > Mask) 844 return; 845 // Unsigned comparison between two sign-extended values is equivalent 846 // to unsigned comparison between two zero-extended values. 847 if (IsUnsigned) 848 Value &= Mask; 849 else if (CCMask == SystemZ::CCMASK_CMP_EQ || 850 CCMask == SystemZ::CCMASK_CMP_NE) 851 // Any choice of IsUnsigned is OK for equality comparisons. 852 // We could use either CHHSI or CLHHSI for 16-bit comparisons, 853 // but since we use CLHHSI for zero extensions, it seems better 854 // to be consistent and do the same here. 855 Value &= Mask, IsUnsigned = true; 856 else if (NumBits == 8) { 857 // Try to treat the comparison as unsigned, so that we can use CLI. 858 // Adjust CCMask and Value as necessary. 859 if (Value == 0 && CCMask == SystemZ::CCMASK_CMP_LT) 860 // Test whether the high bit of the byte is set. 861 Value = 127, CCMask = SystemZ::CCMASK_CMP_GT, IsUnsigned = true; 862 else if (SignedValue == -1 && CCMask == SystemZ::CCMASK_CMP_GT) 863 // Test whether the high bit of the byte is clear. 864 Value = 128, CCMask = SystemZ::CCMASK_CMP_LT, IsUnsigned = true; 865 else 866 // No instruction exists for this combination. 867 return; 868 } 869 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 870 if (Value > Mask) 871 return; 872 // Signed comparison between two zero-extended values is equivalent 873 // to unsigned comparison. 874 IsUnsigned = true; 875 } else 876 return; 877 878 // Make sure that the first operand is an i32 of the right extension type. 879 ISD::LoadExtType ExtType = IsUnsigned ? ISD::ZEXTLOAD : ISD::SEXTLOAD; 880 if (CmpOp0.getValueType() != MVT::i32 || 881 Load->getExtensionType() != ExtType) 882 CmpOp0 = DAG.getExtLoad(ExtType, Load->getDebugLoc(), MVT::i32, 883 Load->getChain(), Load->getBasePtr(), 884 Load->getPointerInfo(), Load->getMemoryVT(), 885 Load->isVolatile(), Load->isNonTemporal(), 886 Load->getAlignment()); 887 888 // Make sure that the second operand is an i32 with the right value. 889 if (CmpOp1.getValueType() != MVT::i32 || 890 Value != Constant->getZExtValue()) 891 CmpOp1 = DAG.getConstant(Value, MVT::i32); 892} 893 894// Return true if a comparison described by CCMask, CmpOp0 and CmpOp1 895// is an equality comparison that is better implemented using unsigned 896// rather than signed comparison instructions. 897static bool preferUnsignedComparison(SelectionDAG &DAG, SDValue CmpOp0, 898 SDValue CmpOp1, unsigned CCMask) { 899 // The test must be for equality or inequality. 900 if (CCMask != SystemZ::CCMASK_CMP_EQ && CCMask != SystemZ::CCMASK_CMP_NE) 901 return false; 902 903 if (CmpOp1.getOpcode() == ISD::Constant) { 904 uint64_t Value = cast<ConstantSDNode>(CmpOp1)->getSExtValue(); 905 906 // If we're comparing with memory, prefer unsigned comparisons for 907 // values that are in the unsigned 16-bit range but not the signed 908 // 16-bit range. We want to use CLFHSI and CLGHSI. 909 if (CmpOp0.hasOneUse() && 910 ISD::isNormalLoad(CmpOp0.getNode()) && 911 (Value >= 32768 && Value < 65536)) 912 return true; 913 914 // Use unsigned comparisons for values that are in the CLGFI range 915 // but not in the CGFI range. 916 if (CmpOp0.getValueType() == MVT::i64 && (Value >> 31) == 1) 917 return true; 918 919 return false; 920 } 921 922 // Prefer CL for zero-extended loads. 923 if (CmpOp1.getOpcode() == ISD::ZERO_EXTEND || 924 ISD::isZEXTLoad(CmpOp1.getNode())) 925 return true; 926 927 // ...and for "in-register" zero extensions. 928 if (CmpOp1.getOpcode() == ISD::AND && CmpOp1.getValueType() == MVT::i64) { 929 SDValue Mask = CmpOp1.getOperand(1); 930 if (Mask.getOpcode() == ISD::Constant && 931 cast<ConstantSDNode>(Mask)->getZExtValue() == 0xffffffff) 932 return true; 933 } 934 935 return false; 936} 937 938// Return a target node that compares CmpOp0 and CmpOp1. Set CCMask to the 939// 4-bit condition-code mask for CC. 940static SDValue emitCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 941 ISD::CondCode CC, unsigned &CCMask) { 942 bool IsUnsigned = false; 943 CCMask = CCMaskForCondCode(CC); 944 if (!CmpOp0.getValueType().isFloatingPoint()) { 945 IsUnsigned = CCMask & SystemZ::CCMASK_CMP_UO; 946 CCMask &= ~SystemZ::CCMASK_CMP_UO; 947 adjustSubwordCmp(DAG, IsUnsigned, CmpOp0, CmpOp1, CCMask); 948 if (preferUnsignedComparison(DAG, CmpOp0, CmpOp1, CCMask)) 949 IsUnsigned = true; 950 } 951 952 DebugLoc DL = CmpOp0.getDebugLoc(); 953 return DAG.getNode((IsUnsigned ? SystemZISD::UCMP : SystemZISD::CMP), 954 DL, MVT::Glue, CmpOp0, CmpOp1); 955} 956 957// Lower a binary operation that produces two VT results, one in each 958// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 959// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 960// on the extended Op0 and (unextended) Op1. Store the even register result 961// in Even and the odd register result in Odd. 962static void lowerGR128Binary(SelectionDAG &DAG, DebugLoc DL, EVT VT, 963 unsigned Extend, unsigned Opcode, 964 SDValue Op0, SDValue Op1, 965 SDValue &Even, SDValue &Odd) { 966 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 967 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 968 SDValue(In128, 0), Op1); 969 bool Is32Bit = is32Bit(VT); 970 SDValue SubReg0 = DAG.getTargetConstant(SystemZ::even128(Is32Bit), VT); 971 SDValue SubReg1 = DAG.getTargetConstant(SystemZ::odd128(Is32Bit), VT); 972 SDNode *Reg0 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 973 VT, Result, SubReg0); 974 SDNode *Reg1 = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 975 VT, Result, SubReg1); 976 Even = SDValue(Reg0, 0); 977 Odd = SDValue(Reg1, 0); 978} 979 980SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 981 SDValue Chain = Op.getOperand(0); 982 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 983 SDValue CmpOp0 = Op.getOperand(2); 984 SDValue CmpOp1 = Op.getOperand(3); 985 SDValue Dest = Op.getOperand(4); 986 DebugLoc DL = Op.getDebugLoc(); 987 988 unsigned CCMask; 989 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); 990 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 991 Chain, DAG.getConstant(CCMask, MVT::i32), Dest, Flags); 992} 993 994SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 995 SelectionDAG &DAG) const { 996 SDValue CmpOp0 = Op.getOperand(0); 997 SDValue CmpOp1 = Op.getOperand(1); 998 SDValue TrueOp = Op.getOperand(2); 999 SDValue FalseOp = Op.getOperand(3); 1000 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1001 DebugLoc DL = Op.getDebugLoc(); 1002 1003 unsigned CCMask; 1004 SDValue Flags = emitCmp(DAG, CmpOp0, CmpOp1, CC, CCMask); 1005 1006 SmallVector<SDValue, 4> Ops; 1007 Ops.push_back(TrueOp); 1008 Ops.push_back(FalseOp); 1009 Ops.push_back(DAG.getConstant(CCMask, MVT::i32)); 1010 Ops.push_back(Flags); 1011 1012 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1013 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, &Ops[0], Ops.size()); 1014} 1015 1016SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1017 SelectionDAG &DAG) const { 1018 DebugLoc DL = Node->getDebugLoc(); 1019 const GlobalValue *GV = Node->getGlobal(); 1020 int64_t Offset = Node->getOffset(); 1021 EVT PtrVT = getPointerTy(); 1022 Reloc::Model RM = TM.getRelocationModel(); 1023 CodeModel::Model CM = TM.getCodeModel(); 1024 1025 SDValue Result; 1026 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1027 // Make sure that the offset is aligned to a halfword. If it isn't, 1028 // create an "anchor" at the previous 12-bit boundary. 1029 // FIXME check whether there is a better way of handling this. 1030 if (Offset & 1) { 1031 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 1032 Offset & ~uint64_t(0xfff)); 1033 Offset &= 0xfff; 1034 } else { 1035 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Offset); 1036 Offset = 0; 1037 } 1038 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1039 } else { 1040 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1041 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1042 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1043 MachinePointerInfo::getGOT(), false, false, false, 0); 1044 } 1045 1046 // If there was a non-zero offset that we didn't fold, create an explicit 1047 // addition for it. 1048 if (Offset != 0) 1049 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1050 DAG.getConstant(Offset, PtrVT)); 1051 1052 return Result; 1053} 1054 1055SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1056 SelectionDAG &DAG) const { 1057 DebugLoc DL = Node->getDebugLoc(); 1058 const GlobalValue *GV = Node->getGlobal(); 1059 EVT PtrVT = getPointerTy(); 1060 TLSModel::Model model = TM.getTLSModel(GV); 1061 1062 if (model != TLSModel::LocalExec) 1063 llvm_unreachable("only local-exec TLS mode supported"); 1064 1065 // The high part of the thread pointer is in access register 0. 1066 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1067 DAG.getConstant(0, MVT::i32)); 1068 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1069 1070 // The low part of the thread pointer is in access register 1. 1071 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1072 DAG.getConstant(1, MVT::i32)); 1073 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1074 1075 // Merge them into a single 64-bit address. 1076 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1077 DAG.getConstant(32, PtrVT)); 1078 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1079 1080 // Get the offset of GA from the thread pointer. 1081 SystemZConstantPoolValue *CPV = 1082 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1083 1084 // Force the offset into the constant pool and load it from there. 1085 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1086 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1087 CPAddr, MachinePointerInfo::getConstantPool(), 1088 false, false, false, 0); 1089 1090 // Add the base and offset together. 1091 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1092} 1093 1094SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1095 SelectionDAG &DAG) const { 1096 DebugLoc DL = Node->getDebugLoc(); 1097 const BlockAddress *BA = Node->getBlockAddress(); 1098 int64_t Offset = Node->getOffset(); 1099 EVT PtrVT = getPointerTy(); 1100 1101 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1102 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1103 return Result; 1104} 1105 1106SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1107 SelectionDAG &DAG) const { 1108 DebugLoc DL = JT->getDebugLoc(); 1109 EVT PtrVT = getPointerTy(); 1110 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1111 1112 // Use LARL to load the address of the table. 1113 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1114} 1115 1116SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1117 SelectionDAG &DAG) const { 1118 DebugLoc DL = CP->getDebugLoc(); 1119 EVT PtrVT = getPointerTy(); 1120 1121 SDValue Result; 1122 if (CP->isMachineConstantPoolEntry()) 1123 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1124 CP->getAlignment()); 1125 else 1126 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1127 CP->getAlignment(), CP->getOffset()); 1128 1129 // Use LARL to load the address of the constant pool entry. 1130 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1131} 1132 1133SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1134 SelectionDAG &DAG) const { 1135 DebugLoc DL = Op.getDebugLoc(); 1136 SDValue In = Op.getOperand(0); 1137 EVT InVT = In.getValueType(); 1138 EVT ResVT = Op.getValueType(); 1139 1140 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1141 SDValue Shift32 = DAG.getConstant(32, MVT::i64); 1142 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1143 SDValue In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1144 SDValue Shift = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, Shift32); 1145 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Shift); 1146 SDNode *Out = DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 1147 MVT::f32, Out64, SubReg32); 1148 return SDValue(Out, 0); 1149 } 1150 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1151 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1152 SDNode *In64 = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1153 MVT::f64, SDValue(U64, 0), In, SubReg32); 1154 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, SDValue(In64, 0)); 1155 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, Shift32); 1156 SDValue Out = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1157 return Out; 1158 } 1159 llvm_unreachable("Unexpected bitcast combination"); 1160} 1161 1162SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1163 SelectionDAG &DAG) const { 1164 MachineFunction &MF = DAG.getMachineFunction(); 1165 SystemZMachineFunctionInfo *FuncInfo = 1166 MF.getInfo<SystemZMachineFunctionInfo>(); 1167 EVT PtrVT = getPointerTy(); 1168 1169 SDValue Chain = Op.getOperand(0); 1170 SDValue Addr = Op.getOperand(1); 1171 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1172 DebugLoc DL = Op.getDebugLoc(); 1173 1174 // The initial values of each field. 1175 const unsigned NumFields = 4; 1176 SDValue Fields[NumFields] = { 1177 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1178 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1179 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1180 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1181 }; 1182 1183 // Store each field into its respective slot. 1184 SDValue MemOps[NumFields]; 1185 unsigned Offset = 0; 1186 for (unsigned I = 0; I < NumFields; ++I) { 1187 SDValue FieldAddr = Addr; 1188 if (Offset != 0) 1189 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1190 DAG.getIntPtrConstant(Offset)); 1191 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1192 MachinePointerInfo(SV, Offset), 1193 false, false, 0); 1194 Offset += 8; 1195 } 1196 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps, NumFields); 1197} 1198 1199SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1200 SelectionDAG &DAG) const { 1201 SDValue Chain = Op.getOperand(0); 1202 SDValue DstPtr = Op.getOperand(1); 1203 SDValue SrcPtr = Op.getOperand(2); 1204 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1205 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1206 DebugLoc DL = Op.getDebugLoc(); 1207 1208 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1209 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1210 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1211} 1212 1213SDValue SystemZTargetLowering:: 1214lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 1215 SDValue Chain = Op.getOperand(0); 1216 SDValue Size = Op.getOperand(1); 1217 DebugLoc DL = Op.getDebugLoc(); 1218 1219 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 1220 1221 // Get a reference to the stack pointer. 1222 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 1223 1224 // Get the new stack pointer value. 1225 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 1226 1227 // Copy the new stack pointer back. 1228 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 1229 1230 // The allocated data lives above the 160 bytes allocated for the standard 1231 // frame, plus any outgoing stack arguments. We don't know how much that 1232 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 1233 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 1234 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 1235 1236 SDValue Ops[2] = { Result, Chain }; 1237 return DAG.getMergeValues(Ops, 2, DL); 1238} 1239 1240SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 1241 SelectionDAG &DAG) const { 1242 EVT VT = Op.getValueType(); 1243 DebugLoc DL = Op.getDebugLoc(); 1244 assert(!is32Bit(VT) && "Only support 64-bit UMUL_LOHI"); 1245 1246 // UMUL_LOHI64 returns the low result in the odd register and the high 1247 // result in the even register. UMUL_LOHI is defined to return the 1248 // low half first, so the results are in reverse order. 1249 SDValue Ops[2]; 1250 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 1251 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1252 return DAG.getMergeValues(Ops, 2, DL); 1253} 1254 1255SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 1256 SelectionDAG &DAG) const { 1257 SDValue Op0 = Op.getOperand(0); 1258 SDValue Op1 = Op.getOperand(1); 1259 EVT VT = Op.getValueType(); 1260 DebugLoc DL = Op.getDebugLoc(); 1261 1262 // We use DSGF for 32-bit division. 1263 if (is32Bit(VT)) { 1264 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 1265 Op1 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op1); 1266 } 1267 1268 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 1269 // input is "don't care". The instruction returns the remainder in 1270 // the even register and the quotient in the odd register. 1271 SDValue Ops[2]; 1272 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::SDIVREM64, 1273 Op0, Op1, Ops[1], Ops[0]); 1274 return DAG.getMergeValues(Ops, 2, DL); 1275} 1276 1277SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 1278 SelectionDAG &DAG) const { 1279 EVT VT = Op.getValueType(); 1280 DebugLoc DL = Op.getDebugLoc(); 1281 1282 // DL(G) uses a double-width dividend, so we need to clear the even 1283 // register in the GR128 input. The instruction returns the remainder 1284 // in the even register and the quotient in the odd register. 1285 SDValue Ops[2]; 1286 if (is32Bit(VT)) 1287 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 1288 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1289 else 1290 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 1291 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 1292 return DAG.getMergeValues(Ops, 2, DL); 1293} 1294 1295SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 1296 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 1297 1298 // Get the known-zero masks for each operand. 1299 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 1300 APInt KnownZero[2], KnownOne[2]; 1301 DAG.ComputeMaskedBits(Ops[0], KnownZero[0], KnownOne[0]); 1302 DAG.ComputeMaskedBits(Ops[1], KnownZero[1], KnownOne[1]); 1303 1304 // See if the upper 32 bits of one operand and the lower 32 bits of the 1305 // other are known zero. They are the low and high operands respectively. 1306 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 1307 KnownZero[1].getZExtValue() }; 1308 unsigned High, Low; 1309 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 1310 High = 1, Low = 0; 1311 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 1312 High = 0, Low = 1; 1313 else 1314 return Op; 1315 1316 SDValue LowOp = Ops[Low]; 1317 SDValue HighOp = Ops[High]; 1318 1319 // If the high part is a constant, we're better off using IILH. 1320 if (HighOp.getOpcode() == ISD::Constant) 1321 return Op; 1322 1323 // If the low part is a constant that is outside the range of LHI, 1324 // then we're better off using IILF. 1325 if (LowOp.getOpcode() == ISD::Constant) { 1326 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 1327 if (!isInt<16>(Value)) 1328 return Op; 1329 } 1330 1331 // Check whether the high part is an AND that doesn't change the 1332 // high 32 bits and just masks out low bits. We can skip it if so. 1333 if (HighOp.getOpcode() == ISD::AND && 1334 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 1335 ConstantSDNode *MaskNode = cast<ConstantSDNode>(HighOp.getOperand(1)); 1336 uint64_t Mask = MaskNode->getZExtValue() | Masks[High]; 1337 if ((Mask >> 32) == 0xffffffff) 1338 HighOp = HighOp.getOperand(0); 1339 } 1340 1341 // Take advantage of the fact that all GR32 operations only change the 1342 // low 32 bits by truncating Low to an i32 and inserting it directly 1343 // using a subreg. The interesting cases are those where the truncation 1344 // can be folded. 1345 DebugLoc DL = Op.getDebugLoc(); 1346 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 1347 SDValue SubReg32 = DAG.getTargetConstant(SystemZ::subreg_32bit, MVT::i64); 1348 SDNode *Result = DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 1349 MVT::i64, HighOp, Low32, SubReg32); 1350 return SDValue(Result, 0); 1351} 1352 1353// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 1354// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 1355SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 1356 SelectionDAG &DAG, 1357 unsigned Opcode) const { 1358 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1359 1360 // 32-bit operations need no code outside the main loop. 1361 EVT NarrowVT = Node->getMemoryVT(); 1362 EVT WideVT = MVT::i32; 1363 if (NarrowVT == WideVT) 1364 return Op; 1365 1366 int64_t BitSize = NarrowVT.getSizeInBits(); 1367 SDValue ChainIn = Node->getChain(); 1368 SDValue Addr = Node->getBasePtr(); 1369 SDValue Src2 = Node->getVal(); 1370 MachineMemOperand *MMO = Node->getMemOperand(); 1371 DebugLoc DL = Node->getDebugLoc(); 1372 EVT PtrVT = Addr.getValueType(); 1373 1374 // Convert atomic subtracts of constants into additions. 1375 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 1376 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Src2)) { 1377 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 1378 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 1379 } 1380 1381 // Get the address of the containing word. 1382 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1383 DAG.getConstant(-4, PtrVT)); 1384 1385 // Get the number of bits that the word must be rotated left in order 1386 // to bring the field to the top bits of a GR32. 1387 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1388 DAG.getConstant(3, PtrVT)); 1389 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1390 1391 // Get the complementing shift amount, for rotating a field in the top 1392 // bits back to its proper position. 1393 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1394 DAG.getConstant(0, WideVT), BitShift); 1395 1396 // Extend the source operand to 32 bits and prepare it for the inner loop. 1397 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 1398 // operations require the source to be shifted in advance. (This shift 1399 // can be folded if the source is constant.) For AND and NAND, the lower 1400 // bits must be set, while for other opcodes they should be left clear. 1401 if (Opcode != SystemZISD::ATOMIC_SWAPW) 1402 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 1403 DAG.getConstant(32 - BitSize, WideVT)); 1404 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 1405 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 1406 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 1407 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 1408 1409 // Construct the ATOMIC_LOADW_* node. 1410 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1411 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 1412 DAG.getConstant(BitSize, WideVT) }; 1413 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 1414 array_lengthof(Ops), 1415 NarrowVT, MMO); 1416 1417 // Rotate the result of the final CS so that the field is in the lower 1418 // bits of a GR32, then truncate it. 1419 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 1420 DAG.getConstant(BitSize, WideVT)); 1421 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 1422 1423 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 1424 return DAG.getMergeValues(RetOps, 2, DL); 1425} 1426 1427// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 1428// into a fullword ATOMIC_CMP_SWAPW operation. 1429SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 1430 SelectionDAG &DAG) const { 1431 AtomicSDNode *Node = cast<AtomicSDNode>(Op.getNode()); 1432 1433 // We have native support for 32-bit compare and swap. 1434 EVT NarrowVT = Node->getMemoryVT(); 1435 EVT WideVT = MVT::i32; 1436 if (NarrowVT == WideVT) 1437 return Op; 1438 1439 int64_t BitSize = NarrowVT.getSizeInBits(); 1440 SDValue ChainIn = Node->getOperand(0); 1441 SDValue Addr = Node->getOperand(1); 1442 SDValue CmpVal = Node->getOperand(2); 1443 SDValue SwapVal = Node->getOperand(3); 1444 MachineMemOperand *MMO = Node->getMemOperand(); 1445 DebugLoc DL = Node->getDebugLoc(); 1446 EVT PtrVT = Addr.getValueType(); 1447 1448 // Get the address of the containing word. 1449 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 1450 DAG.getConstant(-4, PtrVT)); 1451 1452 // Get the number of bits that the word must be rotated left in order 1453 // to bring the field to the top bits of a GR32. 1454 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 1455 DAG.getConstant(3, PtrVT)); 1456 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 1457 1458 // Get the complementing shift amount, for rotating a field in the top 1459 // bits back to its proper position. 1460 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 1461 DAG.getConstant(0, WideVT), BitShift); 1462 1463 // Construct the ATOMIC_CMP_SWAPW node. 1464 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 1465 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 1466 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 1467 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 1468 VTList, Ops, array_lengthof(Ops), 1469 NarrowVT, MMO); 1470 return AtomicOp; 1471} 1472 1473SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 1474 SelectionDAG &DAG) const { 1475 MachineFunction &MF = DAG.getMachineFunction(); 1476 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1477 return DAG.getCopyFromReg(Op.getOperand(0), Op.getDebugLoc(), 1478 SystemZ::R15D, Op.getValueType()); 1479} 1480 1481SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 1482 SelectionDAG &DAG) const { 1483 MachineFunction &MF = DAG.getMachineFunction(); 1484 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 1485 return DAG.getCopyToReg(Op.getOperand(0), Op.getDebugLoc(), 1486 SystemZ::R15D, Op.getOperand(1)); 1487} 1488 1489SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 1490 SelectionDAG &DAG) const { 1491 switch (Op.getOpcode()) { 1492 case ISD::BR_CC: 1493 return lowerBR_CC(Op, DAG); 1494 case ISD::SELECT_CC: 1495 return lowerSELECT_CC(Op, DAG); 1496 case ISD::GlobalAddress: 1497 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 1498 case ISD::GlobalTLSAddress: 1499 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 1500 case ISD::BlockAddress: 1501 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 1502 case ISD::JumpTable: 1503 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 1504 case ISD::ConstantPool: 1505 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 1506 case ISD::BITCAST: 1507 return lowerBITCAST(Op, DAG); 1508 case ISD::VASTART: 1509 return lowerVASTART(Op, DAG); 1510 case ISD::VACOPY: 1511 return lowerVACOPY(Op, DAG); 1512 case ISD::DYNAMIC_STACKALLOC: 1513 return lowerDYNAMIC_STACKALLOC(Op, DAG); 1514 case ISD::UMUL_LOHI: 1515 return lowerUMUL_LOHI(Op, DAG); 1516 case ISD::SDIVREM: 1517 return lowerSDIVREM(Op, DAG); 1518 case ISD::UDIVREM: 1519 return lowerUDIVREM(Op, DAG); 1520 case ISD::OR: 1521 return lowerOR(Op, DAG); 1522 case ISD::ATOMIC_SWAP: 1523 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_SWAPW); 1524 case ISD::ATOMIC_LOAD_ADD: 1525 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 1526 case ISD::ATOMIC_LOAD_SUB: 1527 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 1528 case ISD::ATOMIC_LOAD_AND: 1529 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 1530 case ISD::ATOMIC_LOAD_OR: 1531 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 1532 case ISD::ATOMIC_LOAD_XOR: 1533 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 1534 case ISD::ATOMIC_LOAD_NAND: 1535 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 1536 case ISD::ATOMIC_LOAD_MIN: 1537 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 1538 case ISD::ATOMIC_LOAD_MAX: 1539 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 1540 case ISD::ATOMIC_LOAD_UMIN: 1541 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 1542 case ISD::ATOMIC_LOAD_UMAX: 1543 return lowerATOMIC_LOAD(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 1544 case ISD::ATOMIC_CMP_SWAP: 1545 return lowerATOMIC_CMP_SWAP(Op, DAG); 1546 case ISD::STACKSAVE: 1547 return lowerSTACKSAVE(Op, DAG); 1548 case ISD::STACKRESTORE: 1549 return lowerSTACKRESTORE(Op, DAG); 1550 default: 1551 llvm_unreachable("Unexpected node to lower"); 1552 } 1553} 1554 1555const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 1556#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 1557 switch (Opcode) { 1558 OPCODE(RET_FLAG); 1559 OPCODE(CALL); 1560 OPCODE(PCREL_WRAPPER); 1561 OPCODE(CMP); 1562 OPCODE(UCMP); 1563 OPCODE(BR_CCMASK); 1564 OPCODE(SELECT_CCMASK); 1565 OPCODE(ADJDYNALLOC); 1566 OPCODE(EXTRACT_ACCESS); 1567 OPCODE(UMUL_LOHI64); 1568 OPCODE(SDIVREM64); 1569 OPCODE(UDIVREM32); 1570 OPCODE(UDIVREM64); 1571 OPCODE(ATOMIC_SWAPW); 1572 OPCODE(ATOMIC_LOADW_ADD); 1573 OPCODE(ATOMIC_LOADW_SUB); 1574 OPCODE(ATOMIC_LOADW_AND); 1575 OPCODE(ATOMIC_LOADW_OR); 1576 OPCODE(ATOMIC_LOADW_XOR); 1577 OPCODE(ATOMIC_LOADW_NAND); 1578 OPCODE(ATOMIC_LOADW_MIN); 1579 OPCODE(ATOMIC_LOADW_MAX); 1580 OPCODE(ATOMIC_LOADW_UMIN); 1581 OPCODE(ATOMIC_LOADW_UMAX); 1582 OPCODE(ATOMIC_CMP_SWAPW); 1583 } 1584 return NULL; 1585#undef OPCODE 1586} 1587 1588//===----------------------------------------------------------------------===// 1589// Custom insertion 1590//===----------------------------------------------------------------------===// 1591 1592// Create a new basic block after MBB. 1593static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 1594 MachineFunction &MF = *MBB->getParent(); 1595 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 1596 MF.insert(llvm::next(MachineFunction::iterator(MBB)), NewMBB); 1597 return NewMBB; 1598} 1599 1600// Split MBB after MI and return the new block (the one that contains 1601// instructions after MI). 1602static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 1603 MachineBasicBlock *MBB) { 1604 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 1605 NewMBB->splice(NewMBB->begin(), MBB, 1606 llvm::next(MachineBasicBlock::iterator(MI)), 1607 MBB->end()); 1608 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 1609 return NewMBB; 1610} 1611 1612// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 1613MachineBasicBlock * 1614SystemZTargetLowering::emitSelect(MachineInstr *MI, 1615 MachineBasicBlock *MBB) const { 1616 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1617 1618 unsigned DestReg = MI->getOperand(0).getReg(); 1619 unsigned TrueReg = MI->getOperand(1).getReg(); 1620 unsigned FalseReg = MI->getOperand(2).getReg(); 1621 unsigned CCMask = MI->getOperand(3).getImm(); 1622 DebugLoc DL = MI->getDebugLoc(); 1623 1624 MachineBasicBlock *StartMBB = MBB; 1625 MachineBasicBlock *JoinMBB = splitBlockAfter(MI, MBB); 1626 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 1627 1628 // StartMBB: 1629 // ... 1630 // TrueVal = ... 1631 // cmpTY ccX, r1, r2 1632 // jCC JoinMBB 1633 // # fallthrough to FalseMBB 1634 MBB = StartMBB; 1635 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(CCMask).addMBB(JoinMBB); 1636 MBB->addSuccessor(JoinMBB); 1637 MBB->addSuccessor(FalseMBB); 1638 1639 // FalseMBB: 1640 // # fallthrough to JoinMBB 1641 MBB = FalseMBB; 1642 MBB->addSuccessor(JoinMBB); 1643 1644 // JoinMBB: 1645 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 1646 // ... 1647 MBB = JoinMBB; 1648 BuildMI(*MBB, MBB->begin(), DL, TII->get(SystemZ::PHI), DestReg) 1649 .addReg(TrueReg).addMBB(StartMBB) 1650 .addReg(FalseReg).addMBB(FalseMBB); 1651 1652 MI->eraseFromParent(); 1653 return JoinMBB; 1654} 1655 1656// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 1657// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 1658// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 1659// BitSize is the width of the field in bits, or 0 if this is a partword 1660// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 1661// is one of the operands. Invert says whether the field should be 1662// inverted after performing BinOpcode (e.g. for NAND). 1663MachineBasicBlock * 1664SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 1665 MachineBasicBlock *MBB, 1666 unsigned BinOpcode, 1667 unsigned BitSize, 1668 bool Invert) const { 1669 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1670 MachineFunction &MF = *MBB->getParent(); 1671 MachineRegisterInfo &MRI = MF.getRegInfo(); 1672 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1673 bool IsSubWord = (BitSize < 32); 1674 1675 // Extract the operands. Base can be a register or a frame index. 1676 // Src2 can be a register or immediate. 1677 unsigned Dest = MI->getOperand(0).getReg(); 1678 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1679 int64_t Disp = MI->getOperand(2).getImm(); 1680 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 1681 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 1682 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 1683 DebugLoc DL = MI->getDebugLoc(); 1684 if (IsSubWord) 1685 BitSize = MI->getOperand(6).getImm(); 1686 1687 // Subword operations use 32-bit registers. 1688 const TargetRegisterClass *RC = (BitSize <= 32 ? 1689 &SystemZ::GR32BitRegClass : 1690 &SystemZ::GR64BitRegClass); 1691 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 1692 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 1693 1694 // Get the right opcodes for the displacement. 1695 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 1696 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 1697 assert(LOpcode && CSOpcode && "Displacement out of range"); 1698 1699 // Create virtual registers for temporary results. 1700 unsigned OrigVal = MRI.createVirtualRegister(RC); 1701 unsigned OldVal = MRI.createVirtualRegister(RC); 1702 unsigned NewVal = (BinOpcode || IsSubWord ? 1703 MRI.createVirtualRegister(RC) : Src2.getReg()); 1704 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 1705 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 1706 1707 // Insert a basic block for the main loop. 1708 MachineBasicBlock *StartMBB = MBB; 1709 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1710 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1711 1712 // StartMBB: 1713 // ... 1714 // %OrigVal = L Disp(%Base) 1715 // # fall through to LoopMMB 1716 MBB = StartMBB; 1717 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 1718 .addOperand(Base).addImm(Disp).addReg(0); 1719 MBB->addSuccessor(LoopMBB); 1720 1721 // LoopMBB: 1722 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 1723 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 1724 // %RotatedNewVal = OP %RotatedOldVal, %Src2 1725 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 1726 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 1727 // JNE LoopMBB 1728 // # fall through to DoneMMB 1729 MBB = LoopMBB; 1730 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 1731 .addReg(OrigVal).addMBB(StartMBB) 1732 .addReg(Dest).addMBB(LoopMBB); 1733 if (IsSubWord) 1734 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 1735 .addReg(OldVal).addReg(BitShift).addImm(0); 1736 if (Invert) { 1737 // Perform the operation normally and then invert every bit of the field. 1738 unsigned Tmp = MRI.createVirtualRegister(RC); 1739 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 1740 .addReg(RotatedOldVal).addOperand(Src2); 1741 if (BitSize < 32) 1742 // XILF with the upper BitSize bits set. 1743 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 1744 .addReg(Tmp).addImm(uint32_t(~0 << (32 - BitSize))); 1745 else if (BitSize == 32) 1746 // XILF with every bit set. 1747 BuildMI(MBB, DL, TII->get(SystemZ::XILF32), RotatedNewVal) 1748 .addReg(Tmp).addImm(~uint32_t(0)); 1749 else { 1750 // Use LCGR and add -1 to the result, which is more compact than 1751 // an XILF, XILH pair. 1752 unsigned Tmp2 = MRI.createVirtualRegister(RC); 1753 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 1754 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 1755 .addReg(Tmp2).addImm(-1); 1756 } 1757 } else if (BinOpcode) 1758 // A simply binary operation. 1759 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 1760 .addReg(RotatedOldVal).addOperand(Src2); 1761 else if (IsSubWord) 1762 // Use RISBG to rotate Src2 into position and use it to replace the 1763 // field in RotatedOldVal. 1764 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 1765 .addReg(RotatedOldVal).addReg(Src2.getReg()) 1766 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 1767 if (IsSubWord) 1768 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 1769 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 1770 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 1771 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 1772 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); 1773 MBB->addSuccessor(LoopMBB); 1774 MBB->addSuccessor(DoneMBB); 1775 1776 MI->eraseFromParent(); 1777 return DoneMBB; 1778} 1779 1780// Implement EmitInstrWithCustomInserter for pseudo 1781// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 1782// instruction that should be used to compare the current field with the 1783// minimum or maximum value. KeepOldMask is the BRC condition-code mask 1784// for when the current field should be kept. BitSize is the width of 1785// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 1786MachineBasicBlock * 1787SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 1788 MachineBasicBlock *MBB, 1789 unsigned CompareOpcode, 1790 unsigned KeepOldMask, 1791 unsigned BitSize) const { 1792 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1793 MachineFunction &MF = *MBB->getParent(); 1794 MachineRegisterInfo &MRI = MF.getRegInfo(); 1795 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1796 bool IsSubWord = (BitSize < 32); 1797 1798 // Extract the operands. Base can be a register or a frame index. 1799 unsigned Dest = MI->getOperand(0).getReg(); 1800 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1801 int64_t Disp = MI->getOperand(2).getImm(); 1802 unsigned Src2 = MI->getOperand(3).getReg(); 1803 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 1804 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 1805 DebugLoc DL = MI->getDebugLoc(); 1806 if (IsSubWord) 1807 BitSize = MI->getOperand(6).getImm(); 1808 1809 // Subword operations use 32-bit registers. 1810 const TargetRegisterClass *RC = (BitSize <= 32 ? 1811 &SystemZ::GR32BitRegClass : 1812 &SystemZ::GR64BitRegClass); 1813 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 1814 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 1815 1816 // Get the right opcodes for the displacement. 1817 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 1818 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 1819 assert(LOpcode && CSOpcode && "Displacement out of range"); 1820 1821 // Create virtual registers for temporary results. 1822 unsigned OrigVal = MRI.createVirtualRegister(RC); 1823 unsigned OldVal = MRI.createVirtualRegister(RC); 1824 unsigned NewVal = MRI.createVirtualRegister(RC); 1825 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 1826 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 1827 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 1828 1829 // Insert 3 basic blocks for the loop. 1830 MachineBasicBlock *StartMBB = MBB; 1831 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1832 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1833 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 1834 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 1835 1836 // StartMBB: 1837 // ... 1838 // %OrigVal = L Disp(%Base) 1839 // # fall through to LoopMMB 1840 MBB = StartMBB; 1841 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 1842 .addOperand(Base).addImm(Disp).addReg(0); 1843 MBB->addSuccessor(LoopMBB); 1844 1845 // LoopMBB: 1846 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 1847 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 1848 // CompareOpcode %RotatedOldVal, %Src2 1849 // BRCL KeepOldMask, UpdateMBB 1850 MBB = LoopMBB; 1851 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 1852 .addReg(OrigVal).addMBB(StartMBB) 1853 .addReg(Dest).addMBB(UpdateMBB); 1854 if (IsSubWord) 1855 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 1856 .addReg(OldVal).addReg(BitShift).addImm(0); 1857 BuildMI(MBB, DL, TII->get(CompareOpcode)) 1858 .addReg(RotatedOldVal).addReg(Src2); 1859 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)) 1860 .addImm(KeepOldMask).addMBB(UpdateMBB); 1861 MBB->addSuccessor(UpdateMBB); 1862 MBB->addSuccessor(UseAltMBB); 1863 1864 // UseAltMBB: 1865 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 1866 // # fall through to UpdateMMB 1867 MBB = UseAltMBB; 1868 if (IsSubWord) 1869 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 1870 .addReg(RotatedOldVal).addReg(Src2) 1871 .addImm(32).addImm(31 + BitSize).addImm(0); 1872 MBB->addSuccessor(UpdateMBB); 1873 1874 // UpdateMBB: 1875 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 1876 // [ %RotatedAltVal, UseAltMBB ] 1877 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 1878 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 1879 // JNE LoopMBB 1880 // # fall through to DoneMMB 1881 MBB = UpdateMBB; 1882 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 1883 .addReg(RotatedOldVal).addMBB(LoopMBB) 1884 .addReg(RotatedAltVal).addMBB(UseAltMBB); 1885 if (IsSubWord) 1886 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 1887 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 1888 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 1889 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 1890 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); 1891 MBB->addSuccessor(LoopMBB); 1892 MBB->addSuccessor(DoneMBB); 1893 1894 MI->eraseFromParent(); 1895 return DoneMBB; 1896} 1897 1898// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 1899// instruction MI. 1900MachineBasicBlock * 1901SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 1902 MachineBasicBlock *MBB) const { 1903 const SystemZInstrInfo *TII = TM.getInstrInfo(); 1904 MachineFunction &MF = *MBB->getParent(); 1905 MachineRegisterInfo &MRI = MF.getRegInfo(); 1906 unsigned MaskNE = CCMaskForCondCode(ISD::SETNE); 1907 1908 // Extract the operands. Base can be a register or a frame index. 1909 unsigned Dest = MI->getOperand(0).getReg(); 1910 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 1911 int64_t Disp = MI->getOperand(2).getImm(); 1912 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 1913 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 1914 unsigned BitShift = MI->getOperand(5).getReg(); 1915 unsigned NegBitShift = MI->getOperand(6).getReg(); 1916 int64_t BitSize = MI->getOperand(7).getImm(); 1917 DebugLoc DL = MI->getDebugLoc(); 1918 1919 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 1920 1921 // Get the right opcodes for the displacement. 1922 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 1923 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 1924 assert(LOpcode && CSOpcode && "Displacement out of range"); 1925 1926 // Create virtual registers for temporary results. 1927 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 1928 unsigned OldVal = MRI.createVirtualRegister(RC); 1929 unsigned CmpVal = MRI.createVirtualRegister(RC); 1930 unsigned SwapVal = MRI.createVirtualRegister(RC); 1931 unsigned StoreVal = MRI.createVirtualRegister(RC); 1932 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 1933 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 1934 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 1935 1936 // Insert 2 basic blocks for the loop. 1937 MachineBasicBlock *StartMBB = MBB; 1938 MachineBasicBlock *DoneMBB = splitBlockAfter(MI, MBB); 1939 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 1940 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 1941 1942 // StartMBB: 1943 // ... 1944 // %OrigOldVal = L Disp(%Base) 1945 // # fall through to LoopMMB 1946 MBB = StartMBB; 1947 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 1948 .addOperand(Base).addImm(Disp).addReg(0); 1949 MBB->addSuccessor(LoopMBB); 1950 1951 // LoopMBB: 1952 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 1953 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 1954 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 1955 // %Dest = RLL %OldVal, BitSize(%BitShift) 1956 // ^^ The low BitSize bits contain the field 1957 // of interest. 1958 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 1959 // ^^ Replace the upper 32-BitSize bits of the 1960 // comparison value with those that we loaded, 1961 // so that we can use a full word comparison. 1962 // CR %Dest, %RetryCmpVal 1963 // JNE DoneMBB 1964 // # Fall through to SetMBB 1965 MBB = LoopMBB; 1966 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 1967 .addReg(OrigOldVal).addMBB(StartMBB) 1968 .addReg(RetryOldVal).addMBB(SetMBB); 1969 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 1970 .addReg(OrigCmpVal).addMBB(StartMBB) 1971 .addReg(RetryCmpVal).addMBB(SetMBB); 1972 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 1973 .addReg(OrigSwapVal).addMBB(StartMBB) 1974 .addReg(RetrySwapVal).addMBB(SetMBB); 1975 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 1976 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 1977 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 1978 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 1979 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 1980 .addReg(Dest).addReg(RetryCmpVal); 1981 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(DoneMBB); 1982 MBB->addSuccessor(DoneMBB); 1983 MBB->addSuccessor(SetMBB); 1984 1985 // SetMBB: 1986 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 1987 // ^^ Replace the upper 32-BitSize bits of the new 1988 // value with those that we loaded. 1989 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 1990 // ^^ Rotate the new field to its proper position. 1991 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 1992 // JNE LoopMBB 1993 // # fall through to ExitMMB 1994 MBB = SetMBB; 1995 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 1996 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 1997 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 1998 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 1999 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 2000 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 2001 BuildMI(MBB, DL, TII->get(SystemZ::BRCL)).addImm(MaskNE).addMBB(LoopMBB); 2002 MBB->addSuccessor(LoopMBB); 2003 MBB->addSuccessor(DoneMBB); 2004 2005 MI->eraseFromParent(); 2006 return DoneMBB; 2007} 2008 2009// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 2010// if the high register of the GR128 value must be cleared or false if 2011// it's "don't care". SubReg is subreg_odd32 when extending a GR32 2012// and subreg_odd when extending a GR64. 2013MachineBasicBlock * 2014SystemZTargetLowering::emitExt128(MachineInstr *MI, 2015 MachineBasicBlock *MBB, 2016 bool ClearEven, unsigned SubReg) const { 2017 const SystemZInstrInfo *TII = TM.getInstrInfo(); 2018 MachineFunction &MF = *MBB->getParent(); 2019 MachineRegisterInfo &MRI = MF.getRegInfo(); 2020 DebugLoc DL = MI->getDebugLoc(); 2021 2022 unsigned Dest = MI->getOperand(0).getReg(); 2023 unsigned Src = MI->getOperand(1).getReg(); 2024 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2025 2026 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 2027 if (ClearEven) { 2028 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 2029 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 2030 2031 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 2032 .addImm(0); 2033 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 2034 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_high); 2035 In128 = NewIn128; 2036 } 2037 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 2038 .addReg(In128).addReg(Src).addImm(SubReg); 2039 2040 MI->eraseFromParent(); 2041 return MBB; 2042} 2043 2044MachineBasicBlock *SystemZTargetLowering:: 2045EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 2046 switch (MI->getOpcode()) { 2047 case SystemZ::Select32: 2048 case SystemZ::SelectF32: 2049 case SystemZ::Select64: 2050 case SystemZ::SelectF64: 2051 case SystemZ::SelectF128: 2052 return emitSelect(MI, MBB); 2053 2054 case SystemZ::AEXT128_64: 2055 return emitExt128(MI, MBB, false, SystemZ::subreg_low); 2056 case SystemZ::ZEXT128_32: 2057 return emitExt128(MI, MBB, true, SystemZ::subreg_low32); 2058 case SystemZ::ZEXT128_64: 2059 return emitExt128(MI, MBB, true, SystemZ::subreg_low); 2060 2061 case SystemZ::ATOMIC_SWAPW: 2062 return emitAtomicLoadBinary(MI, MBB, 0, 0); 2063 case SystemZ::ATOMIC_SWAP_32: 2064 return emitAtomicLoadBinary(MI, MBB, 0, 32); 2065 case SystemZ::ATOMIC_SWAP_64: 2066 return emitAtomicLoadBinary(MI, MBB, 0, 64); 2067 2068 case SystemZ::ATOMIC_LOADW_AR: 2069 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 2070 case SystemZ::ATOMIC_LOADW_AFI: 2071 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 2072 case SystemZ::ATOMIC_LOAD_AR: 2073 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 2074 case SystemZ::ATOMIC_LOAD_AHI: 2075 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 2076 case SystemZ::ATOMIC_LOAD_AFI: 2077 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 2078 case SystemZ::ATOMIC_LOAD_AGR: 2079 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 2080 case SystemZ::ATOMIC_LOAD_AGHI: 2081 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 2082 case SystemZ::ATOMIC_LOAD_AGFI: 2083 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 2084 2085 case SystemZ::ATOMIC_LOADW_SR: 2086 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 2087 case SystemZ::ATOMIC_LOAD_SR: 2088 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 2089 case SystemZ::ATOMIC_LOAD_SGR: 2090 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 2091 2092 case SystemZ::ATOMIC_LOADW_NR: 2093 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 2094 case SystemZ::ATOMIC_LOADW_NILH: 2095 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0); 2096 case SystemZ::ATOMIC_LOAD_NR: 2097 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 2098 case SystemZ::ATOMIC_LOAD_NILL32: 2099 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32); 2100 case SystemZ::ATOMIC_LOAD_NILH32: 2101 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32); 2102 case SystemZ::ATOMIC_LOAD_NILF32: 2103 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32); 2104 case SystemZ::ATOMIC_LOAD_NGR: 2105 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 2106 case SystemZ::ATOMIC_LOAD_NILL: 2107 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64); 2108 case SystemZ::ATOMIC_LOAD_NILH: 2109 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64); 2110 case SystemZ::ATOMIC_LOAD_NIHL: 2111 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64); 2112 case SystemZ::ATOMIC_LOAD_NIHH: 2113 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64); 2114 case SystemZ::ATOMIC_LOAD_NILF: 2115 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64); 2116 case SystemZ::ATOMIC_LOAD_NIHF: 2117 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64); 2118 2119 case SystemZ::ATOMIC_LOADW_OR: 2120 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 2121 case SystemZ::ATOMIC_LOADW_OILH: 2122 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 0); 2123 case SystemZ::ATOMIC_LOAD_OR: 2124 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 2125 case SystemZ::ATOMIC_LOAD_OILL32: 2126 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL32, 32); 2127 case SystemZ::ATOMIC_LOAD_OILH32: 2128 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH32, 32); 2129 case SystemZ::ATOMIC_LOAD_OILF32: 2130 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF32, 32); 2131 case SystemZ::ATOMIC_LOAD_OGR: 2132 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 2133 case SystemZ::ATOMIC_LOAD_OILL: 2134 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 64); 2135 case SystemZ::ATOMIC_LOAD_OILH: 2136 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 64); 2137 case SystemZ::ATOMIC_LOAD_OIHL: 2138 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL, 64); 2139 case SystemZ::ATOMIC_LOAD_OIHH: 2140 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH, 64); 2141 case SystemZ::ATOMIC_LOAD_OILF: 2142 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 64); 2143 case SystemZ::ATOMIC_LOAD_OIHF: 2144 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF, 64); 2145 2146 case SystemZ::ATOMIC_LOADW_XR: 2147 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 2148 case SystemZ::ATOMIC_LOADW_XILF: 2149 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 0); 2150 case SystemZ::ATOMIC_LOAD_XR: 2151 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 2152 case SystemZ::ATOMIC_LOAD_XILF32: 2153 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF32, 32); 2154 case SystemZ::ATOMIC_LOAD_XGR: 2155 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 2156 case SystemZ::ATOMIC_LOAD_XILF: 2157 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 64); 2158 case SystemZ::ATOMIC_LOAD_XIHF: 2159 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF, 64); 2160 2161 case SystemZ::ATOMIC_LOADW_NRi: 2162 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 2163 case SystemZ::ATOMIC_LOADW_NILHi: 2164 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 0, true); 2165 case SystemZ::ATOMIC_LOAD_NRi: 2166 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 2167 case SystemZ::ATOMIC_LOAD_NILL32i: 2168 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL32, 32, true); 2169 case SystemZ::ATOMIC_LOAD_NILH32i: 2170 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH32, 32, true); 2171 case SystemZ::ATOMIC_LOAD_NILF32i: 2172 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF32, 32, true); 2173 case SystemZ::ATOMIC_LOAD_NGRi: 2174 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 2175 case SystemZ::ATOMIC_LOAD_NILLi: 2176 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 64, true); 2177 case SystemZ::ATOMIC_LOAD_NILHi: 2178 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 64, true); 2179 case SystemZ::ATOMIC_LOAD_NIHLi: 2180 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL, 64, true); 2181 case SystemZ::ATOMIC_LOAD_NIHHi: 2182 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH, 64, true); 2183 case SystemZ::ATOMIC_LOAD_NILFi: 2184 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 64, true); 2185 case SystemZ::ATOMIC_LOAD_NIHFi: 2186 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF, 64, true); 2187 2188 case SystemZ::ATOMIC_LOADW_MIN: 2189 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2190 SystemZ::CCMASK_CMP_LE, 0); 2191 case SystemZ::ATOMIC_LOAD_MIN_32: 2192 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2193 SystemZ::CCMASK_CMP_LE, 32); 2194 case SystemZ::ATOMIC_LOAD_MIN_64: 2195 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2196 SystemZ::CCMASK_CMP_LE, 64); 2197 2198 case SystemZ::ATOMIC_LOADW_MAX: 2199 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2200 SystemZ::CCMASK_CMP_GE, 0); 2201 case SystemZ::ATOMIC_LOAD_MAX_32: 2202 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 2203 SystemZ::CCMASK_CMP_GE, 32); 2204 case SystemZ::ATOMIC_LOAD_MAX_64: 2205 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 2206 SystemZ::CCMASK_CMP_GE, 64); 2207 2208 case SystemZ::ATOMIC_LOADW_UMIN: 2209 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2210 SystemZ::CCMASK_CMP_LE, 0); 2211 case SystemZ::ATOMIC_LOAD_UMIN_32: 2212 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2213 SystemZ::CCMASK_CMP_LE, 32); 2214 case SystemZ::ATOMIC_LOAD_UMIN_64: 2215 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2216 SystemZ::CCMASK_CMP_LE, 64); 2217 2218 case SystemZ::ATOMIC_LOADW_UMAX: 2219 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2220 SystemZ::CCMASK_CMP_GE, 0); 2221 case SystemZ::ATOMIC_LOAD_UMAX_32: 2222 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 2223 SystemZ::CCMASK_CMP_GE, 32); 2224 case SystemZ::ATOMIC_LOAD_UMAX_64: 2225 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 2226 SystemZ::CCMASK_CMP_GE, 64); 2227 2228 case SystemZ::ATOMIC_CMP_SWAPW: 2229 return emitAtomicCmpSwapW(MI, MBB); 2230 default: 2231 llvm_unreachable("Unexpected instr type to insert"); 2232 } 2233} 2234