SystemZISelLowering.cpp revision 280031
1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SystemZTargetLowering class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "SystemZISelLowering.h" 15#include "SystemZCallingConv.h" 16#include "SystemZConstantPoolValue.h" 17#include "SystemZMachineFunctionInfo.h" 18#include "SystemZTargetMachine.h" 19#include "llvm/CodeGen/CallingConvLower.h" 20#include "llvm/CodeGen/MachineInstrBuilder.h" 21#include "llvm/CodeGen/MachineRegisterInfo.h" 22#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 23#include <cctype> 24 25using namespace llvm; 26 27#define DEBUG_TYPE "systemz-lower" 28 29namespace { 30// Represents a sequence for extracting a 0/1 value from an IPM result: 31// (((X ^ XORValue) + AddValue) >> Bit) 32struct IPMConversion { 33 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 34 : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 35 36 int64_t XORValue; 37 int64_t AddValue; 38 unsigned Bit; 39}; 40 41// Represents information about a comparison. 42struct Comparison { 43 Comparison(SDValue Op0In, SDValue Op1In) 44 : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {} 45 46 // The operands to the comparison. 47 SDValue Op0, Op1; 48 49 // The opcode that should be used to compare Op0 and Op1. 50 unsigned Opcode; 51 52 // A SystemZICMP value. Only used for integer comparisons. 53 unsigned ICmpType; 54 55 // The mask of CC values that Opcode can produce. 56 unsigned CCValid; 57 58 // The mask of CC values for which the original condition is true. 59 unsigned CCMask; 60}; 61} // end anonymous namespace 62 63// Classify VT as either 32 or 64 bit. 64static bool is32Bit(EVT VT) { 65 switch (VT.getSimpleVT().SimpleTy) { 66 case MVT::i32: 67 return true; 68 case MVT::i64: 69 return false; 70 default: 71 llvm_unreachable("Unsupported type"); 72 } 73} 74 75// Return a version of MachineOperand that can be safely used before the 76// final use. 77static MachineOperand earlyUseOperand(MachineOperand Op) { 78 if (Op.isReg()) 79 Op.setIsKill(false); 80 return Op; 81} 82 83SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &tm) 84 : TargetLowering(tm), 85 Subtarget(tm.getSubtarget<SystemZSubtarget>()) { 86 MVT PtrVT = getPointerTy(); 87 88 // Set up the register classes. 89 if (Subtarget.hasHighWord()) 90 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 91 else 92 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 93 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 94 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 95 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 96 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 97 98 // Compute derived properties from the register classes 99 computeRegisterProperties(); 100 101 // Set up special registers. 102 setExceptionPointerRegister(SystemZ::R6D); 103 setExceptionSelectorRegister(SystemZ::R7D); 104 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 105 106 // TODO: It may be better to default to latency-oriented scheduling, however 107 // LLVM's current latency-oriented scheduler can't handle physreg definitions 108 // such as SystemZ has with CC, so set this to the register-pressure 109 // scheduler, because it can. 110 setSchedulingPreference(Sched::RegPressure); 111 112 setBooleanContents(ZeroOrOneBooleanContent); 113 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct? 114 115 // Instructions are strings of 2-byte aligned 2-byte values. 116 setMinFunctionAlignment(2); 117 118 // Handle operations that are handled in a similar way for all types. 119 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 120 I <= MVT::LAST_FP_VALUETYPE; 121 ++I) { 122 MVT VT = MVT::SimpleValueType(I); 123 if (isTypeLegal(VT)) { 124 // Lower SET_CC into an IPM-based sequence. 125 setOperationAction(ISD::SETCC, VT, Custom); 126 127 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 128 setOperationAction(ISD::SELECT, VT, Expand); 129 130 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 131 setOperationAction(ISD::SELECT_CC, VT, Custom); 132 setOperationAction(ISD::BR_CC, VT, Custom); 133 } 134 } 135 136 // Expand jump table branches as address arithmetic followed by an 137 // indirect jump. 138 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 139 140 // Expand BRCOND into a BR_CC (see above). 141 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 142 143 // Handle integer types. 144 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 145 I <= MVT::LAST_INTEGER_VALUETYPE; 146 ++I) { 147 MVT VT = MVT::SimpleValueType(I); 148 if (isTypeLegal(VT)) { 149 // Expand individual DIV and REMs into DIVREMs. 150 setOperationAction(ISD::SDIV, VT, Expand); 151 setOperationAction(ISD::UDIV, VT, Expand); 152 setOperationAction(ISD::SREM, VT, Expand); 153 setOperationAction(ISD::UREM, VT, Expand); 154 setOperationAction(ISD::SDIVREM, VT, Custom); 155 setOperationAction(ISD::UDIVREM, VT, Custom); 156 157 // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and 158 // stores, putting a serialization instruction after the stores. 159 setOperationAction(ISD::ATOMIC_LOAD, VT, Custom); 160 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 161 162 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are 163 // available, or if the operand is constant. 164 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 165 166 // No special instructions for these. 167 setOperationAction(ISD::CTPOP, VT, Expand); 168 setOperationAction(ISD::CTTZ, VT, Expand); 169 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); 170 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); 171 setOperationAction(ISD::ROTR, VT, Expand); 172 173 // Use *MUL_LOHI where possible instead of MULH*. 174 setOperationAction(ISD::MULHS, VT, Expand); 175 setOperationAction(ISD::MULHU, VT, Expand); 176 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 177 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 178 179 // Only z196 and above have native support for conversions to unsigned. 180 if (!Subtarget.hasFPExtension()) 181 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 182 } 183 } 184 185 // Type legalization will convert 8- and 16-bit atomic operations into 186 // forms that operate on i32s (but still keeping the original memory VT). 187 // Lower them into full i32 operations. 188 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 189 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 190 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 191 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 192 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 193 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 194 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 195 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 196 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 197 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 198 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 199 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); 200 201 // z10 has instructions for signed but not unsigned FP conversion. 202 // Handle unsigned 32-bit types as signed 64-bit types. 203 if (!Subtarget.hasFPExtension()) { 204 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 205 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 206 } 207 208 // We have native support for a 64-bit CTLZ, via FLOGR. 209 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 210 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 211 212 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 213 setOperationAction(ISD::OR, MVT::i64, Custom); 214 215 // FIXME: Can we support these natively? 216 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 217 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 218 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 219 220 // We have native instructions for i8, i16 and i32 extensions, but not i1. 221 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 222 for (MVT VT : MVT::integer_valuetypes()) { 223 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 224 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 225 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 226 } 227 228 // Handle the various types of symbolic address. 229 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 230 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 231 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 232 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 233 setOperationAction(ISD::JumpTable, PtrVT, Custom); 234 235 // We need to handle dynamic allocations specially because of the 236 // 160-byte area at the bottom of the stack. 237 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 238 239 // Use custom expanders so that we can force the function to use 240 // a frame pointer. 241 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 242 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 243 244 // Handle prefetches with PFD or PFDRL. 245 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 246 247 // Handle floating-point types. 248 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 249 I <= MVT::LAST_FP_VALUETYPE; 250 ++I) { 251 MVT VT = MVT::SimpleValueType(I); 252 if (isTypeLegal(VT)) { 253 // We can use FI for FRINT. 254 setOperationAction(ISD::FRINT, VT, Legal); 255 256 // We can use the extended form of FI for other rounding operations. 257 if (Subtarget.hasFPExtension()) { 258 setOperationAction(ISD::FNEARBYINT, VT, Legal); 259 setOperationAction(ISD::FFLOOR, VT, Legal); 260 setOperationAction(ISD::FCEIL, VT, Legal); 261 setOperationAction(ISD::FTRUNC, VT, Legal); 262 setOperationAction(ISD::FROUND, VT, Legal); 263 } 264 265 // No special instructions for these. 266 setOperationAction(ISD::FSIN, VT, Expand); 267 setOperationAction(ISD::FCOS, VT, Expand); 268 setOperationAction(ISD::FREM, VT, Expand); 269 } 270 } 271 272 // We have fused multiply-addition for f32 and f64 but not f128. 273 setOperationAction(ISD::FMA, MVT::f32, Legal); 274 setOperationAction(ISD::FMA, MVT::f64, Legal); 275 setOperationAction(ISD::FMA, MVT::f128, Expand); 276 277 // Needed so that we don't try to implement f128 constant loads using 278 // a load-and-extend of a f80 constant (in cases where the constant 279 // would fit in an f80). 280 for (MVT VT : MVT::fp_valuetypes()) 281 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); 282 283 // Floating-point truncation and stores need to be done separately. 284 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 285 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 286 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 287 288 // We have 64-bit FPR<->GPR moves, but need special handling for 289 // 32-bit forms. 290 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 291 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 292 293 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 294 // structure, but VAEND is a no-op. 295 setOperationAction(ISD::VASTART, MVT::Other, Custom); 296 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 297 setOperationAction(ISD::VAEND, MVT::Other, Expand); 298 299 // Codes for which we want to perform some z-specific combinations. 300 setTargetDAGCombine(ISD::SIGN_EXTEND); 301 302 // We want to use MVC in preference to even a single load/store pair. 303 MaxStoresPerMemcpy = 0; 304 MaxStoresPerMemcpyOptSize = 0; 305 306 // The main memset sequence is a byte store followed by an MVC. 307 // Two STC or MV..I stores win over that, but the kind of fused stores 308 // generated by target-independent code don't when the byte value is 309 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 310 // than "STC;MVC". Handle the choice in target-specific code instead. 311 MaxStoresPerMemset = 0; 312 MaxStoresPerMemsetOptSize = 0; 313} 314 315EVT SystemZTargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const { 316 if (!VT.isVector()) 317 return MVT::i32; 318 return VT.changeVectorElementTypeToInteger(); 319} 320 321bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 322 VT = VT.getScalarType(); 323 324 if (!VT.isSimple()) 325 return false; 326 327 switch (VT.getSimpleVT().SimpleTy) { 328 case MVT::f32: 329 case MVT::f64: 330 return true; 331 case MVT::f128: 332 return false; 333 default: 334 break; 335 } 336 337 return false; 338} 339 340bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 341 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 342 return Imm.isZero() || Imm.isNegZero(); 343} 344 345bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 346 unsigned, 347 unsigned, 348 bool *Fast) const { 349 // Unaligned accesses should never be slower than the expanded version. 350 // We check specifically for aligned accesses in the few cases where 351 // they are required. 352 if (Fast) 353 *Fast = true; 354 return true; 355} 356 357bool SystemZTargetLowering::isLegalAddressingMode(const AddrMode &AM, 358 Type *Ty) const { 359 // Punt on globals for now, although they can be used in limited 360 // RELATIVE LONG cases. 361 if (AM.BaseGV) 362 return false; 363 364 // Require a 20-bit signed offset. 365 if (!isInt<20>(AM.BaseOffs)) 366 return false; 367 368 // Indexing is OK but no scale factor can be applied. 369 return AM.Scale == 0 || AM.Scale == 1; 370} 371 372bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 373 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 374 return false; 375 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 376 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 377 return FromBits > ToBits; 378} 379 380bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 381 if (!FromVT.isInteger() || !ToVT.isInteger()) 382 return false; 383 unsigned FromBits = FromVT.getSizeInBits(); 384 unsigned ToBits = ToVT.getSizeInBits(); 385 return FromBits > ToBits; 386} 387 388//===----------------------------------------------------------------------===// 389// Inline asm support 390//===----------------------------------------------------------------------===// 391 392TargetLowering::ConstraintType 393SystemZTargetLowering::getConstraintType(const std::string &Constraint) const { 394 if (Constraint.size() == 1) { 395 switch (Constraint[0]) { 396 case 'a': // Address register 397 case 'd': // Data register (equivalent to 'r') 398 case 'f': // Floating-point register 399 case 'h': // High-part register 400 case 'r': // General-purpose register 401 return C_RegisterClass; 402 403 case 'Q': // Memory with base and unsigned 12-bit displacement 404 case 'R': // Likewise, plus an index 405 case 'S': // Memory with base and signed 20-bit displacement 406 case 'T': // Likewise, plus an index 407 case 'm': // Equivalent to 'T'. 408 return C_Memory; 409 410 case 'I': // Unsigned 8-bit constant 411 case 'J': // Unsigned 12-bit constant 412 case 'K': // Signed 16-bit constant 413 case 'L': // Signed 20-bit displacement (on all targets we support) 414 case 'M': // 0x7fffffff 415 return C_Other; 416 417 default: 418 break; 419 } 420 } 421 return TargetLowering::getConstraintType(Constraint); 422} 423 424TargetLowering::ConstraintWeight SystemZTargetLowering:: 425getSingleConstraintMatchWeight(AsmOperandInfo &info, 426 const char *constraint) const { 427 ConstraintWeight weight = CW_Invalid; 428 Value *CallOperandVal = info.CallOperandVal; 429 // If we don't have a value, we can't do a match, 430 // but allow it at the lowest weight. 431 if (!CallOperandVal) 432 return CW_Default; 433 Type *type = CallOperandVal->getType(); 434 // Look at the constraint type. 435 switch (*constraint) { 436 default: 437 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 438 break; 439 440 case 'a': // Address register 441 case 'd': // Data register (equivalent to 'r') 442 case 'h': // High-part register 443 case 'r': // General-purpose register 444 if (CallOperandVal->getType()->isIntegerTy()) 445 weight = CW_Register; 446 break; 447 448 case 'f': // Floating-point register 449 if (type->isFloatingPointTy()) 450 weight = CW_Register; 451 break; 452 453 case 'I': // Unsigned 8-bit constant 454 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 455 if (isUInt<8>(C->getZExtValue())) 456 weight = CW_Constant; 457 break; 458 459 case 'J': // Unsigned 12-bit constant 460 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 461 if (isUInt<12>(C->getZExtValue())) 462 weight = CW_Constant; 463 break; 464 465 case 'K': // Signed 16-bit constant 466 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 467 if (isInt<16>(C->getSExtValue())) 468 weight = CW_Constant; 469 break; 470 471 case 'L': // Signed 20-bit displacement (on all targets we support) 472 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 473 if (isInt<20>(C->getSExtValue())) 474 weight = CW_Constant; 475 break; 476 477 case 'M': // 0x7fffffff 478 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 479 if (C->getZExtValue() == 0x7fffffff) 480 weight = CW_Constant; 481 break; 482 } 483 return weight; 484} 485 486// Parse a "{tNNN}" register constraint for which the register type "t" 487// has already been verified. MC is the class associated with "t" and 488// Map maps 0-based register numbers to LLVM register numbers. 489static std::pair<unsigned, const TargetRegisterClass *> 490parseRegisterNumber(const std::string &Constraint, 491 const TargetRegisterClass *RC, const unsigned *Map) { 492 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 493 if (isdigit(Constraint[2])) { 494 std::string Suffix(Constraint.data() + 2, Constraint.size() - 2); 495 unsigned Index = atoi(Suffix.c_str()); 496 if (Index < 16 && Map[Index]) 497 return std::make_pair(Map[Index], RC); 498 } 499 return std::make_pair(0U, nullptr); 500} 501 502std::pair<unsigned, const TargetRegisterClass *> SystemZTargetLowering:: 503getRegForInlineAsmConstraint(const std::string &Constraint, MVT VT) const { 504 if (Constraint.size() == 1) { 505 // GCC Constraint Letters 506 switch (Constraint[0]) { 507 default: break; 508 case 'd': // Data register (equivalent to 'r') 509 case 'r': // General-purpose register 510 if (VT == MVT::i64) 511 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 512 else if (VT == MVT::i128) 513 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 514 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 515 516 case 'a': // Address register 517 if (VT == MVT::i64) 518 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 519 else if (VT == MVT::i128) 520 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 521 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 522 523 case 'h': // High-part register (an LLVM extension) 524 return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 525 526 case 'f': // Floating-point register 527 if (VT == MVT::f64) 528 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 529 else if (VT == MVT::f128) 530 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 531 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 532 } 533 } 534 if (Constraint[0] == '{') { 535 // We need to override the default register parsing for GPRs and FPRs 536 // because the interpretation depends on VT. The internal names of 537 // the registers are also different from the external names 538 // (F0D and F0S instead of F0, etc.). 539 if (Constraint[1] == 'r') { 540 if (VT == MVT::i32) 541 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 542 SystemZMC::GR32Regs); 543 if (VT == MVT::i128) 544 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 545 SystemZMC::GR128Regs); 546 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 547 SystemZMC::GR64Regs); 548 } 549 if (Constraint[1] == 'f') { 550 if (VT == MVT::f32) 551 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 552 SystemZMC::FP32Regs); 553 if (VT == MVT::f128) 554 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 555 SystemZMC::FP128Regs); 556 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 557 SystemZMC::FP64Regs); 558 } 559 } 560 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT); 561} 562 563void SystemZTargetLowering:: 564LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 565 std::vector<SDValue> &Ops, 566 SelectionDAG &DAG) const { 567 // Only support length 1 constraints for now. 568 if (Constraint.length() == 1) { 569 switch (Constraint[0]) { 570 case 'I': // Unsigned 8-bit constant 571 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 572 if (isUInt<8>(C->getZExtValue())) 573 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 574 Op.getValueType())); 575 return; 576 577 case 'J': // Unsigned 12-bit constant 578 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 579 if (isUInt<12>(C->getZExtValue())) 580 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 581 Op.getValueType())); 582 return; 583 584 case 'K': // Signed 16-bit constant 585 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 586 if (isInt<16>(C->getSExtValue())) 587 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 588 Op.getValueType())); 589 return; 590 591 case 'L': // Signed 20-bit displacement (on all targets we support) 592 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 593 if (isInt<20>(C->getSExtValue())) 594 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), 595 Op.getValueType())); 596 return; 597 598 case 'M': // 0x7fffffff 599 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 600 if (C->getZExtValue() == 0x7fffffff) 601 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), 602 Op.getValueType())); 603 return; 604 } 605 } 606 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 607} 608 609//===----------------------------------------------------------------------===// 610// Calling conventions 611//===----------------------------------------------------------------------===// 612 613#include "SystemZGenCallingConv.inc" 614 615bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 616 Type *ToType) const { 617 return isTruncateFree(FromType, ToType); 618} 619 620bool SystemZTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const { 621 if (!CI->isTailCall()) 622 return false; 623 return true; 624} 625 626// Value is a value that has been passed to us in the location described by VA 627// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 628// any loads onto Chain. 629static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDLoc DL, 630 CCValAssign &VA, SDValue Chain, 631 SDValue Value) { 632 // If the argument has been promoted from a smaller type, insert an 633 // assertion to capture this. 634 if (VA.getLocInfo() == CCValAssign::SExt) 635 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 636 DAG.getValueType(VA.getValVT())); 637 else if (VA.getLocInfo() == CCValAssign::ZExt) 638 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 639 DAG.getValueType(VA.getValVT())); 640 641 if (VA.isExtInLoc()) 642 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 643 else if (VA.getLocInfo() == CCValAssign::Indirect) 644 Value = DAG.getLoad(VA.getValVT(), DL, Chain, Value, 645 MachinePointerInfo(), false, false, false, 0); 646 else 647 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 648 return Value; 649} 650 651// Value is a value of type VA.getValVT() that we need to copy into 652// the location described by VA. Return a copy of Value converted to 653// VA.getValVT(). The caller is responsible for handling indirect values. 654static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDLoc DL, 655 CCValAssign &VA, SDValue Value) { 656 switch (VA.getLocInfo()) { 657 case CCValAssign::SExt: 658 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 659 case CCValAssign::ZExt: 660 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 661 case CCValAssign::AExt: 662 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 663 case CCValAssign::Full: 664 return Value; 665 default: 666 llvm_unreachable("Unhandled getLocInfo()"); 667 } 668} 669 670SDValue SystemZTargetLowering:: 671LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 672 const SmallVectorImpl<ISD::InputArg> &Ins, 673 SDLoc DL, SelectionDAG &DAG, 674 SmallVectorImpl<SDValue> &InVals) const { 675 MachineFunction &MF = DAG.getMachineFunction(); 676 MachineFrameInfo *MFI = MF.getFrameInfo(); 677 MachineRegisterInfo &MRI = MF.getRegInfo(); 678 SystemZMachineFunctionInfo *FuncInfo = 679 MF.getInfo<SystemZMachineFunctionInfo>(); 680 auto *TFL = static_cast<const SystemZFrameLowering *>( 681 DAG.getSubtarget().getFrameLowering()); 682 683 // Assign locations to all of the incoming arguments. 684 SmallVector<CCValAssign, 16> ArgLocs; 685 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 686 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 687 688 unsigned NumFixedGPRs = 0; 689 unsigned NumFixedFPRs = 0; 690 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 691 SDValue ArgValue; 692 CCValAssign &VA = ArgLocs[I]; 693 EVT LocVT = VA.getLocVT(); 694 if (VA.isRegLoc()) { 695 // Arguments passed in registers 696 const TargetRegisterClass *RC; 697 switch (LocVT.getSimpleVT().SimpleTy) { 698 default: 699 // Integers smaller than i64 should be promoted to i64. 700 llvm_unreachable("Unexpected argument type"); 701 case MVT::i32: 702 NumFixedGPRs += 1; 703 RC = &SystemZ::GR32BitRegClass; 704 break; 705 case MVT::i64: 706 NumFixedGPRs += 1; 707 RC = &SystemZ::GR64BitRegClass; 708 break; 709 case MVT::f32: 710 NumFixedFPRs += 1; 711 RC = &SystemZ::FP32BitRegClass; 712 break; 713 case MVT::f64: 714 NumFixedFPRs += 1; 715 RC = &SystemZ::FP64BitRegClass; 716 break; 717 } 718 719 unsigned VReg = MRI.createVirtualRegister(RC); 720 MRI.addLiveIn(VA.getLocReg(), VReg); 721 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 722 } else { 723 assert(VA.isMemLoc() && "Argument not register or memory"); 724 725 // Create the frame index object for this incoming parameter. 726 int FI = MFI->CreateFixedObject(LocVT.getSizeInBits() / 8, 727 VA.getLocMemOffset(), true); 728 729 // Create the SelectionDAG nodes corresponding to a load 730 // from this parameter. Unpromoted ints and floats are 731 // passed as right-justified 8-byte values. 732 EVT PtrVT = getPointerTy(); 733 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 734 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 735 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4)); 736 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 737 MachinePointerInfo::getFixedStack(FI), 738 false, false, false, 0); 739 } 740 741 // Convert the value of the argument register into the value that's 742 // being passed. 743 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 744 } 745 746 if (IsVarArg) { 747 // Save the number of non-varargs registers for later use by va_start, etc. 748 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 749 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 750 751 // Likewise the address (in the form of a frame index) of where the 752 // first stack vararg would be. The 1-byte size here is arbitrary. 753 int64_t StackSize = CCInfo.getNextStackOffset(); 754 FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize, true)); 755 756 // ...and a similar frame index for the caller-allocated save area 757 // that will be used to store the incoming registers. 758 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 759 unsigned RegSaveIndex = MFI->CreateFixedObject(1, RegSaveOffset, true); 760 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 761 762 // Store the FPR varargs in the reserved frame slots. (We store the 763 // GPRs as part of the prologue.) 764 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 765 SDValue MemOps[SystemZ::NumArgFPRs]; 766 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 767 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 768 int FI = MFI->CreateFixedObject(8, RegSaveOffset + Offset, true); 769 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy()); 770 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 771 &SystemZ::FP64BitRegClass); 772 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 773 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 774 MachinePointerInfo::getFixedStack(FI), 775 false, false, 0); 776 777 } 778 // Join the stores, which are independent of one another. 779 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 780 makeArrayRef(&MemOps[NumFixedFPRs], 781 SystemZ::NumArgFPRs-NumFixedFPRs)); 782 } 783 } 784 785 return Chain; 786} 787 788static bool canUseSiblingCall(const CCState &ArgCCInfo, 789 SmallVectorImpl<CCValAssign> &ArgLocs) { 790 // Punt if there are any indirect or stack arguments, or if the call 791 // needs the call-saved argument register R6. 792 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 793 CCValAssign &VA = ArgLocs[I]; 794 if (VA.getLocInfo() == CCValAssign::Indirect) 795 return false; 796 if (!VA.isRegLoc()) 797 return false; 798 unsigned Reg = VA.getLocReg(); 799 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 800 return false; 801 } 802 return true; 803} 804 805SDValue 806SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 807 SmallVectorImpl<SDValue> &InVals) const { 808 SelectionDAG &DAG = CLI.DAG; 809 SDLoc &DL = CLI.DL; 810 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 811 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 812 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 813 SDValue Chain = CLI.Chain; 814 SDValue Callee = CLI.Callee; 815 bool &IsTailCall = CLI.IsTailCall; 816 CallingConv::ID CallConv = CLI.CallConv; 817 bool IsVarArg = CLI.IsVarArg; 818 MachineFunction &MF = DAG.getMachineFunction(); 819 EVT PtrVT = getPointerTy(); 820 821 // Analyze the operands of the call, assigning locations to each operand. 822 SmallVector<CCValAssign, 16> ArgLocs; 823 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 824 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 825 826 // We don't support GuaranteedTailCallOpt, only automatically-detected 827 // sibling calls. 828 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs)) 829 IsTailCall = false; 830 831 // Get a count of how many bytes are to be pushed on the stack. 832 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 833 834 // Mark the start of the call. 835 if (!IsTailCall) 836 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumBytes, PtrVT, true), 837 DL); 838 839 // Copy argument values to their designated locations. 840 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 841 SmallVector<SDValue, 8> MemOpChains; 842 SDValue StackPtr; 843 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 844 CCValAssign &VA = ArgLocs[I]; 845 SDValue ArgValue = OutVals[I]; 846 847 if (VA.getLocInfo() == CCValAssign::Indirect) { 848 // Store the argument in a stack slot and pass its address. 849 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT()); 850 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 851 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, SpillSlot, 852 MachinePointerInfo::getFixedStack(FI), 853 false, false, 0)); 854 ArgValue = SpillSlot; 855 } else 856 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 857 858 if (VA.isRegLoc()) 859 // Queue up the argument copies and emit them at the end. 860 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 861 else { 862 assert(VA.isMemLoc() && "Argument not register or memory"); 863 864 // Work out the address of the stack slot. Unpromoted ints and 865 // floats are passed as right-justified 8-byte values. 866 if (!StackPtr.getNode()) 867 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 868 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 869 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 870 Offset += 4; 871 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 872 DAG.getIntPtrConstant(Offset)); 873 874 // Emit the store. 875 MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, Address, 876 MachinePointerInfo(), 877 false, false, 0)); 878 } 879 } 880 881 // Join the stores, which are independent of one another. 882 if (!MemOpChains.empty()) 883 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 884 885 // Accept direct calls by converting symbolic call addresses to the 886 // associated Target* opcodes. Force %r1 to be used for indirect 887 // tail calls. 888 SDValue Glue; 889 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 890 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 891 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 892 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 893 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 894 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 895 } else if (IsTailCall) { 896 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 897 Glue = Chain.getValue(1); 898 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 899 } 900 901 // Build a sequence of copy-to-reg nodes, chained and glued together. 902 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 903 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 904 RegsToPass[I].second, Glue); 905 Glue = Chain.getValue(1); 906 } 907 908 // The first call operand is the chain and the second is the target address. 909 SmallVector<SDValue, 8> Ops; 910 Ops.push_back(Chain); 911 Ops.push_back(Callee); 912 913 // Add argument registers to the end of the list so that they are 914 // known live into the call. 915 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 916 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 917 RegsToPass[I].second.getValueType())); 918 919 // Add a register mask operand representing the call-preserved registers. 920 const TargetRegisterInfo *TRI = 921 getTargetMachine().getSubtargetImpl()->getRegisterInfo(); 922 const uint32_t *Mask = TRI->getCallPreservedMask(CallConv); 923 assert(Mask && "Missing call preserved mask for calling convention"); 924 Ops.push_back(DAG.getRegisterMask(Mask)); 925 926 // Glue the call to the argument copies, if any. 927 if (Glue.getNode()) 928 Ops.push_back(Glue); 929 930 // Emit the call. 931 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 932 if (IsTailCall) 933 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops); 934 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops); 935 Glue = Chain.getValue(1); 936 937 // Mark the end of the call, which is glued to the call itself. 938 Chain = DAG.getCALLSEQ_END(Chain, 939 DAG.getConstant(NumBytes, PtrVT, true), 940 DAG.getConstant(0, PtrVT, true), 941 Glue, DL); 942 Glue = Chain.getValue(1); 943 944 // Assign locations to each value returned by this call. 945 SmallVector<CCValAssign, 16> RetLocs; 946 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 947 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 948 949 // Copy all of the result registers out of their specified physreg. 950 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 951 CCValAssign &VA = RetLocs[I]; 952 953 // Copy the value out, gluing the copy to the end of the call sequence. 954 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 955 VA.getLocVT(), Glue); 956 Chain = RetValue.getValue(1); 957 Glue = RetValue.getValue(2); 958 959 // Convert the value of the return register into the value that's 960 // being returned. 961 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 962 } 963 964 return Chain; 965} 966 967SDValue 968SystemZTargetLowering::LowerReturn(SDValue Chain, 969 CallingConv::ID CallConv, bool IsVarArg, 970 const SmallVectorImpl<ISD::OutputArg> &Outs, 971 const SmallVectorImpl<SDValue> &OutVals, 972 SDLoc DL, SelectionDAG &DAG) const { 973 MachineFunction &MF = DAG.getMachineFunction(); 974 975 // Assign locations to each returned value. 976 SmallVector<CCValAssign, 16> RetLocs; 977 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 978 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 979 980 // Quick exit for void returns 981 if (RetLocs.empty()) 982 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 983 984 // Copy the result values into the output registers. 985 SDValue Glue; 986 SmallVector<SDValue, 4> RetOps; 987 RetOps.push_back(Chain); 988 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 989 CCValAssign &VA = RetLocs[I]; 990 SDValue RetValue = OutVals[I]; 991 992 // Make the return register live on exit. 993 assert(VA.isRegLoc() && "Can only return in registers!"); 994 995 // Promote the value as required. 996 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 997 998 // Chain and glue the copies together. 999 unsigned Reg = VA.getLocReg(); 1000 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 1001 Glue = Chain.getValue(1); 1002 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 1003 } 1004 1005 // Update chain and glue. 1006 RetOps[0] = Chain; 1007 if (Glue.getNode()) 1008 RetOps.push_back(Glue); 1009 1010 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps); 1011} 1012 1013SDValue SystemZTargetLowering:: 1014prepareVolatileOrAtomicLoad(SDValue Chain, SDLoc DL, SelectionDAG &DAG) const { 1015 return DAG.getNode(SystemZISD::SERIALIZE, DL, MVT::Other, Chain); 1016} 1017 1018// CC is a comparison that will be implemented using an integer or 1019// floating-point comparison. Return the condition code mask for 1020// a branch on true. In the integer case, CCMASK_CMP_UO is set for 1021// unsigned comparisons and clear for signed ones. In the floating-point 1022// case, CCMASK_CMP_UO has its normal mask meaning (unordered). 1023static unsigned CCMaskForCondCode(ISD::CondCode CC) { 1024#define CONV(X) \ 1025 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 1026 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 1027 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 1028 1029 switch (CC) { 1030 default: 1031 llvm_unreachable("Invalid integer condition!"); 1032 1033 CONV(EQ); 1034 CONV(NE); 1035 CONV(GT); 1036 CONV(GE); 1037 CONV(LT); 1038 CONV(LE); 1039 1040 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 1041 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 1042 } 1043#undef CONV 1044} 1045 1046// Return a sequence for getting a 1 from an IPM result when CC has a 1047// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1048// The handling of CC values outside CCValid doesn't matter. 1049static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1050 // Deal with cases where the result can be taken directly from a bit 1051 // of the IPM result. 1052 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1053 return IPMConversion(0, 0, SystemZ::IPM_CC); 1054 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1055 return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1056 1057 // Deal with cases where we can add a value to force the sign bit 1058 // to contain the right value. Putting the bit in 31 means we can 1059 // use SRL rather than RISBG(L), and also makes it easier to get a 1060 // 0/-1 value, so it has priority over the other tests below. 1061 // 1062 // These sequences rely on the fact that the upper two bits of the 1063 // IPM result are zero. 1064 uint64_t TopBit = uint64_t(1) << 31; 1065 if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1066 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1067 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1068 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1069 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1070 | SystemZ::CCMASK_1 1071 | SystemZ::CCMASK_2))) 1072 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1073 if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1074 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1075 if (CCMask == (CCValid & (SystemZ::CCMASK_1 1076 | SystemZ::CCMASK_2 1077 | SystemZ::CCMASK_3))) 1078 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1079 1080 // Next try inverting the value and testing a bit. 0/1 could be 1081 // handled this way too, but we dealt with that case above. 1082 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1083 return IPMConversion(-1, 0, SystemZ::IPM_CC); 1084 1085 // Handle cases where adding a value forces a non-sign bit to contain 1086 // the right value. 1087 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1088 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1089 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1090 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1091 1092 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are 1093 // can be done by inverting the low CC bit and applying one of the 1094 // sign-based extractions above. 1095 if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1096 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1097 if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1098 return IPMConversion(1 << SystemZ::IPM_CC, 1099 TopBit - (3 << SystemZ::IPM_CC), 31); 1100 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1101 | SystemZ::CCMASK_1 1102 | SystemZ::CCMASK_3))) 1103 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1104 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1105 | SystemZ::CCMASK_2 1106 | SystemZ::CCMASK_3))) 1107 return IPMConversion(1 << SystemZ::IPM_CC, 1108 TopBit - (1 << SystemZ::IPM_CC), 31); 1109 1110 llvm_unreachable("Unexpected CC combination"); 1111} 1112 1113// If C can be converted to a comparison against zero, adjust the operands 1114// as necessary. 1115static void adjustZeroCmp(SelectionDAG &DAG, Comparison &C) { 1116 if (C.ICmpType == SystemZICMP::UnsignedOnly) 1117 return; 1118 1119 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode()); 1120 if (!ConstOp1) 1121 return; 1122 1123 int64_t Value = ConstOp1->getSExtValue(); 1124 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) || 1125 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) || 1126 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) || 1127 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) { 1128 C.CCMask ^= SystemZ::CCMASK_CMP_EQ; 1129 C.Op1 = DAG.getConstant(0, C.Op1.getValueType()); 1130 } 1131} 1132 1133// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI, 1134// adjust the operands as necessary. 1135static void adjustSubwordCmp(SelectionDAG &DAG, Comparison &C) { 1136 // For us to make any changes, it must a comparison between a single-use 1137 // load and a constant. 1138 if (!C.Op0.hasOneUse() || 1139 C.Op0.getOpcode() != ISD::LOAD || 1140 C.Op1.getOpcode() != ISD::Constant) 1141 return; 1142 1143 // We must have an 8- or 16-bit load. 1144 auto *Load = cast<LoadSDNode>(C.Op0); 1145 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1146 if (NumBits != 8 && NumBits != 16) 1147 return; 1148 1149 // The load must be an extending one and the constant must be within the 1150 // range of the unextended value. 1151 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1); 1152 uint64_t Value = ConstOp1->getZExtValue(); 1153 uint64_t Mask = (1 << NumBits) - 1; 1154 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1155 // Make sure that ConstOp1 is in range of C.Op0. 1156 int64_t SignedValue = ConstOp1->getSExtValue(); 1157 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask) 1158 return; 1159 if (C.ICmpType != SystemZICMP::SignedOnly) { 1160 // Unsigned comparison between two sign-extended values is equivalent 1161 // to unsigned comparison between two zero-extended values. 1162 Value &= Mask; 1163 } else if (NumBits == 8) { 1164 // Try to treat the comparison as unsigned, so that we can use CLI. 1165 // Adjust CCMask and Value as necessary. 1166 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT) 1167 // Test whether the high bit of the byte is set. 1168 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT; 1169 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE) 1170 // Test whether the high bit of the byte is clear. 1171 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT; 1172 else 1173 // No instruction exists for this combination. 1174 return; 1175 C.ICmpType = SystemZICMP::UnsignedOnly; 1176 } 1177 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1178 if (Value > Mask) 1179 return; 1180 assert(C.ICmpType == SystemZICMP::Any && 1181 "Signedness shouldn't matter here."); 1182 } else 1183 return; 1184 1185 // Make sure that the first operand is an i32 of the right extension type. 1186 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ? 1187 ISD::SEXTLOAD : 1188 ISD::ZEXTLOAD); 1189 if (C.Op0.getValueType() != MVT::i32 || 1190 Load->getExtensionType() != ExtType) 1191 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, 1192 Load->getChain(), Load->getBasePtr(), 1193 Load->getPointerInfo(), Load->getMemoryVT(), 1194 Load->isVolatile(), Load->isNonTemporal(), 1195 Load->isInvariant(), Load->getAlignment()); 1196 1197 // Make sure that the second operand is an i32 with the right value. 1198 if (C.Op1.getValueType() != MVT::i32 || 1199 Value != ConstOp1->getZExtValue()) 1200 C.Op1 = DAG.getConstant(Value, MVT::i32); 1201} 1202 1203// Return true if Op is either an unextended load, or a load suitable 1204// for integer register-memory comparisons of type ICmpType. 1205static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 1206 auto *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1207 if (Load) { 1208 // There are no instructions to compare a register with a memory byte. 1209 if (Load->getMemoryVT() == MVT::i8) 1210 return false; 1211 // Otherwise decide on extension type. 1212 switch (Load->getExtensionType()) { 1213 case ISD::NON_EXTLOAD: 1214 return true; 1215 case ISD::SEXTLOAD: 1216 return ICmpType != SystemZICMP::UnsignedOnly; 1217 case ISD::ZEXTLOAD: 1218 return ICmpType != SystemZICMP::SignedOnly; 1219 default: 1220 break; 1221 } 1222 } 1223 return false; 1224} 1225 1226// Return true if it is better to swap the operands of C. 1227static bool shouldSwapCmpOperands(const Comparison &C) { 1228 // Leave f128 comparisons alone, since they have no memory forms. 1229 if (C.Op0.getValueType() == MVT::f128) 1230 return false; 1231 1232 // Always keep a floating-point constant second, since comparisons with 1233 // zero can use LOAD TEST and comparisons with other constants make a 1234 // natural memory operand. 1235 if (isa<ConstantFPSDNode>(C.Op1)) 1236 return false; 1237 1238 // Never swap comparisons with zero since there are many ways to optimize 1239 // those later. 1240 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 1241 if (ConstOp1 && ConstOp1->getZExtValue() == 0) 1242 return false; 1243 1244 // Also keep natural memory operands second if the loaded value is 1245 // only used here. Several comparisons have memory forms. 1246 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse()) 1247 return false; 1248 1249 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1250 // In that case we generally prefer the memory to be second. 1251 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) { 1252 // The only exceptions are when the second operand is a constant and 1253 // we can use things like CHHSI. 1254 if (!ConstOp1) 1255 return true; 1256 // The unsigned memory-immediate instructions can handle 16-bit 1257 // unsigned integers. 1258 if (C.ICmpType != SystemZICMP::SignedOnly && 1259 isUInt<16>(ConstOp1->getZExtValue())) 1260 return false; 1261 // The signed memory-immediate instructions can handle 16-bit 1262 // signed integers. 1263 if (C.ICmpType != SystemZICMP::UnsignedOnly && 1264 isInt<16>(ConstOp1->getSExtValue())) 1265 return false; 1266 return true; 1267 } 1268 1269 // Try to promote the use of CGFR and CLGFR. 1270 unsigned Opcode0 = C.Op0.getOpcode(); 1271 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND) 1272 return true; 1273 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND) 1274 return true; 1275 if (C.ICmpType != SystemZICMP::SignedOnly && 1276 Opcode0 == ISD::AND && 1277 C.Op0.getOperand(1).getOpcode() == ISD::Constant && 1278 cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff) 1279 return true; 1280 1281 return false; 1282} 1283 1284// Return a version of comparison CC mask CCMask in which the LT and GT 1285// actions are swapped. 1286static unsigned reverseCCMask(unsigned CCMask) { 1287 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1288 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1289 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1290 (CCMask & SystemZ::CCMASK_CMP_UO)); 1291} 1292 1293// Check whether C tests for equality between X and Y and whether X - Y 1294// or Y - X is also computed. In that case it's better to compare the 1295// result of the subtraction against zero. 1296static void adjustForSubtraction(SelectionDAG &DAG, Comparison &C) { 1297 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 1298 C.CCMask == SystemZ::CCMASK_CMP_NE) { 1299 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1300 SDNode *N = *I; 1301 if (N->getOpcode() == ISD::SUB && 1302 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) || 1303 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) { 1304 C.Op0 = SDValue(N, 0); 1305 C.Op1 = DAG.getConstant(0, N->getValueType(0)); 1306 return; 1307 } 1308 } 1309 } 1310} 1311 1312// Check whether C compares a floating-point value with zero and if that 1313// floating-point value is also negated. In this case we can use the 1314// negation to set CC, so avoiding separate LOAD AND TEST and 1315// LOAD (NEGATIVE/COMPLEMENT) instructions. 1316static void adjustForFNeg(Comparison &C) { 1317 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1); 1318 if (C1 && C1->isZero()) { 1319 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1320 SDNode *N = *I; 1321 if (N->getOpcode() == ISD::FNEG) { 1322 C.Op0 = SDValue(N, 0); 1323 C.CCMask = reverseCCMask(C.CCMask); 1324 return; 1325 } 1326 } 1327 } 1328} 1329 1330// Check whether C compares (shl X, 32) with 0 and whether X is 1331// also sign-extended. In that case it is better to test the result 1332// of the sign extension using LTGFR. 1333// 1334// This case is important because InstCombine transforms a comparison 1335// with (sext (trunc X)) into a comparison with (shl X, 32). 1336static void adjustForLTGFR(Comparison &C) { 1337 // Check for a comparison between (shl X, 32) and 0. 1338 if (C.Op0.getOpcode() == ISD::SHL && 1339 C.Op0.getValueType() == MVT::i64 && 1340 C.Op1.getOpcode() == ISD::Constant && 1341 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1342 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1)); 1343 if (C1 && C1->getZExtValue() == 32) { 1344 SDValue ShlOp0 = C.Op0.getOperand(0); 1345 // See whether X has any SIGN_EXTEND_INREG uses. 1346 for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) { 1347 SDNode *N = *I; 1348 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG && 1349 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) { 1350 C.Op0 = SDValue(N, 0); 1351 return; 1352 } 1353 } 1354 } 1355 } 1356} 1357 1358// If C compares the truncation of an extending load, try to compare 1359// the untruncated value instead. This exposes more opportunities to 1360// reuse CC. 1361static void adjustICmpTruncate(SelectionDAG &DAG, Comparison &C) { 1362 if (C.Op0.getOpcode() == ISD::TRUNCATE && 1363 C.Op0.getOperand(0).getOpcode() == ISD::LOAD && 1364 C.Op1.getOpcode() == ISD::Constant && 1365 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1366 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0)); 1367 if (L->getMemoryVT().getStoreSizeInBits() 1368 <= C.Op0.getValueType().getSizeInBits()) { 1369 unsigned Type = L->getExtensionType(); 1370 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) || 1371 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) { 1372 C.Op0 = C.Op0.getOperand(0); 1373 C.Op1 = DAG.getConstant(0, C.Op0.getValueType()); 1374 } 1375 } 1376 } 1377} 1378 1379// Return true if shift operation N has an in-range constant shift value. 1380// Store it in ShiftVal if so. 1381static bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 1382 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1383 if (!Shift) 1384 return false; 1385 1386 uint64_t Amount = Shift->getZExtValue(); 1387 if (Amount >= N.getValueType().getSizeInBits()) 1388 return false; 1389 1390 ShiftVal = Amount; 1391 return true; 1392} 1393 1394// Check whether an AND with Mask is suitable for a TEST UNDER MASK 1395// instruction and whether the CC value is descriptive enough to handle 1396// a comparison of type Opcode between the AND result and CmpVal. 1397// CCMask says which comparison result is being tested and BitSize is 1398// the number of bits in the operands. If TEST UNDER MASK can be used, 1399// return the corresponding CC mask, otherwise return 0. 1400static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 1401 uint64_t Mask, uint64_t CmpVal, 1402 unsigned ICmpType) { 1403 assert(Mask != 0 && "ANDs with zero should have been removed by now"); 1404 1405 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 1406 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 1407 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 1408 return 0; 1409 1410 // Work out the masks for the lowest and highest bits. 1411 unsigned HighShift = 63 - countLeadingZeros(Mask); 1412 uint64_t High = uint64_t(1) << HighShift; 1413 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 1414 1415 // Signed ordered comparisons are effectively unsigned if the sign 1416 // bit is dropped. 1417 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 1418 1419 // Check for equality comparisons with 0, or the equivalent. 1420 if (CmpVal == 0) { 1421 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1422 return SystemZ::CCMASK_TM_ALL_0; 1423 if (CCMask == SystemZ::CCMASK_CMP_NE) 1424 return SystemZ::CCMASK_TM_SOME_1; 1425 } 1426 if (EffectivelyUnsigned && CmpVal <= Low) { 1427 if (CCMask == SystemZ::CCMASK_CMP_LT) 1428 return SystemZ::CCMASK_TM_ALL_0; 1429 if (CCMask == SystemZ::CCMASK_CMP_GE) 1430 return SystemZ::CCMASK_TM_SOME_1; 1431 } 1432 if (EffectivelyUnsigned && CmpVal < Low) { 1433 if (CCMask == SystemZ::CCMASK_CMP_LE) 1434 return SystemZ::CCMASK_TM_ALL_0; 1435 if (CCMask == SystemZ::CCMASK_CMP_GT) 1436 return SystemZ::CCMASK_TM_SOME_1; 1437 } 1438 1439 // Check for equality comparisons with the mask, or the equivalent. 1440 if (CmpVal == Mask) { 1441 if (CCMask == SystemZ::CCMASK_CMP_EQ) 1442 return SystemZ::CCMASK_TM_ALL_1; 1443 if (CCMask == SystemZ::CCMASK_CMP_NE) 1444 return SystemZ::CCMASK_TM_SOME_0; 1445 } 1446 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 1447 if (CCMask == SystemZ::CCMASK_CMP_GT) 1448 return SystemZ::CCMASK_TM_ALL_1; 1449 if (CCMask == SystemZ::CCMASK_CMP_LE) 1450 return SystemZ::CCMASK_TM_SOME_0; 1451 } 1452 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 1453 if (CCMask == SystemZ::CCMASK_CMP_GE) 1454 return SystemZ::CCMASK_TM_ALL_1; 1455 if (CCMask == SystemZ::CCMASK_CMP_LT) 1456 return SystemZ::CCMASK_TM_SOME_0; 1457 } 1458 1459 // Check for ordered comparisons with the top bit. 1460 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 1461 if (CCMask == SystemZ::CCMASK_CMP_LE) 1462 return SystemZ::CCMASK_TM_MSB_0; 1463 if (CCMask == SystemZ::CCMASK_CMP_GT) 1464 return SystemZ::CCMASK_TM_MSB_1; 1465 } 1466 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 1467 if (CCMask == SystemZ::CCMASK_CMP_LT) 1468 return SystemZ::CCMASK_TM_MSB_0; 1469 if (CCMask == SystemZ::CCMASK_CMP_GE) 1470 return SystemZ::CCMASK_TM_MSB_1; 1471 } 1472 1473 // If there are just two bits, we can do equality checks for Low and High 1474 // as well. 1475 if (Mask == Low + High) { 1476 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 1477 return SystemZ::CCMASK_TM_MIXED_MSB_0; 1478 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 1479 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 1480 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 1481 return SystemZ::CCMASK_TM_MIXED_MSB_1; 1482 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 1483 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 1484 } 1485 1486 // Looks like we've exhausted our options. 1487 return 0; 1488} 1489 1490// See whether C can be implemented as a TEST UNDER MASK instruction. 1491// Update the arguments with the TM version if so. 1492static void adjustForTestUnderMask(SelectionDAG &DAG, Comparison &C) { 1493 // Check that we have a comparison with a constant. 1494 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 1495 if (!ConstOp1) 1496 return; 1497 uint64_t CmpVal = ConstOp1->getZExtValue(); 1498 1499 // Check whether the nonconstant input is an AND with a constant mask. 1500 Comparison NewC(C); 1501 uint64_t MaskVal; 1502 ConstantSDNode *Mask = nullptr; 1503 if (C.Op0.getOpcode() == ISD::AND) { 1504 NewC.Op0 = C.Op0.getOperand(0); 1505 NewC.Op1 = C.Op0.getOperand(1); 1506 Mask = dyn_cast<ConstantSDNode>(NewC.Op1); 1507 if (!Mask) 1508 return; 1509 MaskVal = Mask->getZExtValue(); 1510 } else { 1511 // There is no instruction to compare with a 64-bit immediate 1512 // so use TMHH instead if possible. We need an unsigned ordered 1513 // comparison with an i64 immediate. 1514 if (NewC.Op0.getValueType() != MVT::i64 || 1515 NewC.CCMask == SystemZ::CCMASK_CMP_EQ || 1516 NewC.CCMask == SystemZ::CCMASK_CMP_NE || 1517 NewC.ICmpType == SystemZICMP::SignedOnly) 1518 return; 1519 // Convert LE and GT comparisons into LT and GE. 1520 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE || 1521 NewC.CCMask == SystemZ::CCMASK_CMP_GT) { 1522 if (CmpVal == uint64_t(-1)) 1523 return; 1524 CmpVal += 1; 1525 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ; 1526 } 1527 // If the low N bits of Op1 are zero than the low N bits of Op0 can 1528 // be masked off without changing the result. 1529 MaskVal = -(CmpVal & -CmpVal); 1530 NewC.ICmpType = SystemZICMP::UnsignedOnly; 1531 } 1532 1533 // Check whether the combination of mask, comparison value and comparison 1534 // type are suitable. 1535 unsigned BitSize = NewC.Op0.getValueType().getSizeInBits(); 1536 unsigned NewCCMask, ShiftVal; 1537 if (NewC.ICmpType != SystemZICMP::SignedOnly && 1538 NewC.Op0.getOpcode() == ISD::SHL && 1539 isSimpleShift(NewC.Op0, ShiftVal) && 1540 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 1541 MaskVal >> ShiftVal, 1542 CmpVal >> ShiftVal, 1543 SystemZICMP::Any))) { 1544 NewC.Op0 = NewC.Op0.getOperand(0); 1545 MaskVal >>= ShiftVal; 1546 } else if (NewC.ICmpType != SystemZICMP::SignedOnly && 1547 NewC.Op0.getOpcode() == ISD::SRL && 1548 isSimpleShift(NewC.Op0, ShiftVal) && 1549 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 1550 MaskVal << ShiftVal, 1551 CmpVal << ShiftVal, 1552 SystemZICMP::UnsignedOnly))) { 1553 NewC.Op0 = NewC.Op0.getOperand(0); 1554 MaskVal <<= ShiftVal; 1555 } else { 1556 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal, 1557 NewC.ICmpType); 1558 if (!NewCCMask) 1559 return; 1560 } 1561 1562 // Go ahead and make the change. 1563 C.Opcode = SystemZISD::TM; 1564 C.Op0 = NewC.Op0; 1565 if (Mask && Mask->getZExtValue() == MaskVal) 1566 C.Op1 = SDValue(Mask, 0); 1567 else 1568 C.Op1 = DAG.getConstant(MaskVal, C.Op0.getValueType()); 1569 C.CCValid = SystemZ::CCMASK_TM; 1570 C.CCMask = NewCCMask; 1571} 1572 1573// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1. 1574static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 1575 ISD::CondCode Cond) { 1576 Comparison C(CmpOp0, CmpOp1); 1577 C.CCMask = CCMaskForCondCode(Cond); 1578 if (C.Op0.getValueType().isFloatingPoint()) { 1579 C.CCValid = SystemZ::CCMASK_FCMP; 1580 C.Opcode = SystemZISD::FCMP; 1581 adjustForFNeg(C); 1582 } else { 1583 C.CCValid = SystemZ::CCMASK_ICMP; 1584 C.Opcode = SystemZISD::ICMP; 1585 // Choose the type of comparison. Equality and inequality tests can 1586 // use either signed or unsigned comparisons. The choice also doesn't 1587 // matter if both sign bits are known to be clear. In those cases we 1588 // want to give the main isel code the freedom to choose whichever 1589 // form fits best. 1590 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 1591 C.CCMask == SystemZ::CCMASK_CMP_NE || 1592 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1))) 1593 C.ICmpType = SystemZICMP::Any; 1594 else if (C.CCMask & SystemZ::CCMASK_CMP_UO) 1595 C.ICmpType = SystemZICMP::UnsignedOnly; 1596 else 1597 C.ICmpType = SystemZICMP::SignedOnly; 1598 C.CCMask &= ~SystemZ::CCMASK_CMP_UO; 1599 adjustZeroCmp(DAG, C); 1600 adjustSubwordCmp(DAG, C); 1601 adjustForSubtraction(DAG, C); 1602 adjustForLTGFR(C); 1603 adjustICmpTruncate(DAG, C); 1604 } 1605 1606 if (shouldSwapCmpOperands(C)) { 1607 std::swap(C.Op0, C.Op1); 1608 C.CCMask = reverseCCMask(C.CCMask); 1609 } 1610 1611 adjustForTestUnderMask(DAG, C); 1612 return C; 1613} 1614 1615// Emit the comparison instruction described by C. 1616static SDValue emitCmp(SelectionDAG &DAG, SDLoc DL, Comparison &C) { 1617 if (C.Opcode == SystemZISD::ICMP) 1618 return DAG.getNode(SystemZISD::ICMP, DL, MVT::Glue, C.Op0, C.Op1, 1619 DAG.getConstant(C.ICmpType, MVT::i32)); 1620 if (C.Opcode == SystemZISD::TM) { 1621 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 1622 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 1623 return DAG.getNode(SystemZISD::TM, DL, MVT::Glue, C.Op0, C.Op1, 1624 DAG.getConstant(RegisterOnly, MVT::i32)); 1625 } 1626 return DAG.getNode(C.Opcode, DL, MVT::Glue, C.Op0, C.Op1); 1627} 1628 1629// Implement a 32-bit *MUL_LOHI operation by extending both operands to 1630// 64 bits. Extend is the extension type to use. Store the high part 1631// in Hi and the low part in Lo. 1632static void lowerMUL_LOHI32(SelectionDAG &DAG, SDLoc DL, 1633 unsigned Extend, SDValue Op0, SDValue Op1, 1634 SDValue &Hi, SDValue &Lo) { 1635 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 1636 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 1637 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 1638 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, DAG.getConstant(32, MVT::i64)); 1639 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 1640 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 1641} 1642 1643// Lower a binary operation that produces two VT results, one in each 1644// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 1645// Extend extends Op0 to a GR128, and Opcode performs the GR128 operation 1646// on the extended Op0 and (unextended) Op1. Store the even register result 1647// in Even and the odd register result in Odd. 1648static void lowerGR128Binary(SelectionDAG &DAG, SDLoc DL, EVT VT, 1649 unsigned Extend, unsigned Opcode, 1650 SDValue Op0, SDValue Op1, 1651 SDValue &Even, SDValue &Odd) { 1652 SDNode *In128 = DAG.getMachineNode(Extend, DL, MVT::Untyped, Op0); 1653 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, 1654 SDValue(In128, 0), Op1); 1655 bool Is32Bit = is32Bit(VT); 1656 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 1657 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 1658} 1659 1660// Return an i32 value that is 1 if the CC value produced by Glue is 1661// in the mask CCMask and 0 otherwise. CC is known to have a value 1662// in CCValid, so other values can be ignored. 1663static SDValue emitSETCC(SelectionDAG &DAG, SDLoc DL, SDValue Glue, 1664 unsigned CCValid, unsigned CCMask) { 1665 IPMConversion Conversion = getIPMConversion(CCValid, CCMask); 1666 SDValue Result = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, Glue); 1667 1668 if (Conversion.XORValue) 1669 Result = DAG.getNode(ISD::XOR, DL, MVT::i32, Result, 1670 DAG.getConstant(Conversion.XORValue, MVT::i32)); 1671 1672 if (Conversion.AddValue) 1673 Result = DAG.getNode(ISD::ADD, DL, MVT::i32, Result, 1674 DAG.getConstant(Conversion.AddValue, MVT::i32)); 1675 1676 // The SHR/AND sequence should get optimized to an RISBG. 1677 Result = DAG.getNode(ISD::SRL, DL, MVT::i32, Result, 1678 DAG.getConstant(Conversion.Bit, MVT::i32)); 1679 if (Conversion.Bit != 31) 1680 Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result, 1681 DAG.getConstant(1, MVT::i32)); 1682 return Result; 1683} 1684 1685SDValue SystemZTargetLowering::lowerSETCC(SDValue Op, 1686 SelectionDAG &DAG) const { 1687 SDValue CmpOp0 = Op.getOperand(0); 1688 SDValue CmpOp1 = Op.getOperand(1); 1689 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 1690 SDLoc DL(Op); 1691 1692 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1693 SDValue Glue = emitCmp(DAG, DL, C); 1694 return emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 1695} 1696 1697SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 1698 SDValue Chain = Op.getOperand(0); 1699 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 1700 SDValue CmpOp0 = Op.getOperand(2); 1701 SDValue CmpOp1 = Op.getOperand(3); 1702 SDValue Dest = Op.getOperand(4); 1703 SDLoc DL(Op); 1704 1705 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1706 SDValue Glue = emitCmp(DAG, DL, C); 1707 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 1708 Chain, DAG.getConstant(C.CCValid, MVT::i32), 1709 DAG.getConstant(C.CCMask, MVT::i32), Dest, Glue); 1710} 1711 1712// Return true if Pos is CmpOp and Neg is the negative of CmpOp, 1713// allowing Pos and Neg to be wider than CmpOp. 1714static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) { 1715 return (Neg.getOpcode() == ISD::SUB && 1716 Neg.getOperand(0).getOpcode() == ISD::Constant && 1717 cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 && 1718 Neg.getOperand(1) == Pos && 1719 (Pos == CmpOp || 1720 (Pos.getOpcode() == ISD::SIGN_EXTEND && 1721 Pos.getOperand(0) == CmpOp))); 1722} 1723 1724// Return the absolute or negative absolute of Op; IsNegative decides which. 1725static SDValue getAbsolute(SelectionDAG &DAG, SDLoc DL, SDValue Op, 1726 bool IsNegative) { 1727 Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op); 1728 if (IsNegative) 1729 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(), 1730 DAG.getConstant(0, Op.getValueType()), Op); 1731 return Op; 1732} 1733 1734SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 1735 SelectionDAG &DAG) const { 1736 SDValue CmpOp0 = Op.getOperand(0); 1737 SDValue CmpOp1 = Op.getOperand(1); 1738 SDValue TrueOp = Op.getOperand(2); 1739 SDValue FalseOp = Op.getOperand(3); 1740 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 1741 SDLoc DL(Op); 1742 1743 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC)); 1744 1745 // Check for absolute and negative-absolute selections, including those 1746 // where the comparison value is sign-extended (for LPGFR and LNGFR). 1747 // This check supplements the one in DAGCombiner. 1748 if (C.Opcode == SystemZISD::ICMP && 1749 C.CCMask != SystemZ::CCMASK_CMP_EQ && 1750 C.CCMask != SystemZ::CCMASK_CMP_NE && 1751 C.Op1.getOpcode() == ISD::Constant && 1752 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1753 if (isAbsolute(C.Op0, TrueOp, FalseOp)) 1754 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT); 1755 if (isAbsolute(C.Op0, FalseOp, TrueOp)) 1756 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT); 1757 } 1758 1759 SDValue Glue = emitCmp(DAG, DL, C); 1760 1761 // Special case for handling -1/0 results. The shifts we use here 1762 // should get optimized with the IPM conversion sequence. 1763 auto *TrueC = dyn_cast<ConstantSDNode>(TrueOp); 1764 auto *FalseC = dyn_cast<ConstantSDNode>(FalseOp); 1765 if (TrueC && FalseC) { 1766 int64_t TrueVal = TrueC->getSExtValue(); 1767 int64_t FalseVal = FalseC->getSExtValue(); 1768 if ((TrueVal == -1 && FalseVal == 0) || (TrueVal == 0 && FalseVal == -1)) { 1769 // Invert the condition if we want -1 on false. 1770 if (TrueVal == 0) 1771 C.CCMask ^= C.CCValid; 1772 SDValue Result = emitSETCC(DAG, DL, Glue, C.CCValid, C.CCMask); 1773 EVT VT = Op.getValueType(); 1774 // Extend the result to VT. Upper bits are ignored. 1775 if (!is32Bit(VT)) 1776 Result = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Result); 1777 // Sign-extend from the low bit. 1778 SDValue ShAmt = DAG.getConstant(VT.getSizeInBits() - 1, MVT::i32); 1779 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Result, ShAmt); 1780 return DAG.getNode(ISD::SRA, DL, VT, Shl, ShAmt); 1781 } 1782 } 1783 1784 SmallVector<SDValue, 5> Ops; 1785 Ops.push_back(TrueOp); 1786 Ops.push_back(FalseOp); 1787 Ops.push_back(DAG.getConstant(C.CCValid, MVT::i32)); 1788 Ops.push_back(DAG.getConstant(C.CCMask, MVT::i32)); 1789 Ops.push_back(Glue); 1790 1791 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); 1792 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VTs, Ops); 1793} 1794 1795SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 1796 SelectionDAG &DAG) const { 1797 SDLoc DL(Node); 1798 const GlobalValue *GV = Node->getGlobal(); 1799 int64_t Offset = Node->getOffset(); 1800 EVT PtrVT = getPointerTy(); 1801 Reloc::Model RM = DAG.getTarget().getRelocationModel(); 1802 CodeModel::Model CM = DAG.getTarget().getCodeModel(); 1803 1804 SDValue Result; 1805 if (Subtarget.isPC32DBLSymbol(GV, RM, CM)) { 1806 // Assign anchors at 1<<12 byte boundaries. 1807 uint64_t Anchor = Offset & ~uint64_t(0xfff); 1808 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 1809 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1810 1811 // The offset can be folded into the address if it is aligned to a halfword. 1812 Offset -= Anchor; 1813 if (Offset != 0 && (Offset & 1) == 0) { 1814 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 1815 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 1816 Offset = 0; 1817 } 1818 } else { 1819 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 1820 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1821 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 1822 MachinePointerInfo::getGOT(), false, false, false, 0); 1823 } 1824 1825 // If there was a non-zero offset that we didn't fold, create an explicit 1826 // addition for it. 1827 if (Offset != 0) 1828 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 1829 DAG.getConstant(Offset, PtrVT)); 1830 1831 return Result; 1832} 1833 1834SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 1835 SelectionDAG &DAG) const { 1836 SDLoc DL(Node); 1837 const GlobalValue *GV = Node->getGlobal(); 1838 EVT PtrVT = getPointerTy(); 1839 TLSModel::Model model = DAG.getTarget().getTLSModel(GV); 1840 1841 if (model != TLSModel::LocalExec) 1842 llvm_unreachable("only local-exec TLS mode supported"); 1843 1844 // The high part of the thread pointer is in access register 0. 1845 SDValue TPHi = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1846 DAG.getConstant(0, MVT::i32)); 1847 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 1848 1849 // The low part of the thread pointer is in access register 1. 1850 SDValue TPLo = DAG.getNode(SystemZISD::EXTRACT_ACCESS, DL, MVT::i32, 1851 DAG.getConstant(1, MVT::i32)); 1852 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 1853 1854 // Merge them into a single 64-bit address. 1855 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 1856 DAG.getConstant(32, PtrVT)); 1857 SDValue TP = DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 1858 1859 // Get the offset of GA from the thread pointer. 1860 SystemZConstantPoolValue *CPV = 1861 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 1862 1863 // Force the offset into the constant pool and load it from there. 1864 SDValue CPAddr = DAG.getConstantPool(CPV, PtrVT, 8); 1865 SDValue Offset = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), 1866 CPAddr, MachinePointerInfo::getConstantPool(), 1867 false, false, false, 0); 1868 1869 // Add the base and offset together. 1870 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 1871} 1872 1873SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 1874 SelectionDAG &DAG) const { 1875 SDLoc DL(Node); 1876 const BlockAddress *BA = Node->getBlockAddress(); 1877 int64_t Offset = Node->getOffset(); 1878 EVT PtrVT = getPointerTy(); 1879 1880 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 1881 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1882 return Result; 1883} 1884 1885SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 1886 SelectionDAG &DAG) const { 1887 SDLoc DL(JT); 1888 EVT PtrVT = getPointerTy(); 1889 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 1890 1891 // Use LARL to load the address of the table. 1892 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1893} 1894 1895SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 1896 SelectionDAG &DAG) const { 1897 SDLoc DL(CP); 1898 EVT PtrVT = getPointerTy(); 1899 1900 SDValue Result; 1901 if (CP->isMachineConstantPoolEntry()) 1902 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 1903 CP->getAlignment()); 1904 else 1905 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 1906 CP->getAlignment(), CP->getOffset()); 1907 1908 // Use LARL to load the address of the constant pool entry. 1909 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 1910} 1911 1912SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 1913 SelectionDAG &DAG) const { 1914 SDLoc DL(Op); 1915 SDValue In = Op.getOperand(0); 1916 EVT InVT = In.getValueType(); 1917 EVT ResVT = Op.getValueType(); 1918 1919 if (InVT == MVT::i32 && ResVT == MVT::f32) { 1920 SDValue In64; 1921 if (Subtarget.hasHighWord()) { 1922 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, 1923 MVT::i64); 1924 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1925 MVT::i64, SDValue(U64, 0), In); 1926 } else { 1927 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 1928 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, 1929 DAG.getConstant(32, MVT::i64)); 1930 } 1931 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); 1932 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, 1933 DL, MVT::f32, Out64); 1934 } 1935 if (InVT == MVT::f32 && ResVT == MVT::i32) { 1936 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 1937 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 1938 MVT::f64, SDValue(U64, 0), In); 1939 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 1940 if (Subtarget.hasHighWord()) 1941 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, 1942 MVT::i32, Out64); 1943 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, 1944 DAG.getConstant(32, MVT::i64)); 1945 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 1946 } 1947 llvm_unreachable("Unexpected bitcast combination"); 1948} 1949 1950SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 1951 SelectionDAG &DAG) const { 1952 MachineFunction &MF = DAG.getMachineFunction(); 1953 SystemZMachineFunctionInfo *FuncInfo = 1954 MF.getInfo<SystemZMachineFunctionInfo>(); 1955 EVT PtrVT = getPointerTy(); 1956 1957 SDValue Chain = Op.getOperand(0); 1958 SDValue Addr = Op.getOperand(1); 1959 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1960 SDLoc DL(Op); 1961 1962 // The initial values of each field. 1963 const unsigned NumFields = 4; 1964 SDValue Fields[NumFields] = { 1965 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), PtrVT), 1966 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), PtrVT), 1967 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 1968 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 1969 }; 1970 1971 // Store each field into its respective slot. 1972 SDValue MemOps[NumFields]; 1973 unsigned Offset = 0; 1974 for (unsigned I = 0; I < NumFields; ++I) { 1975 SDValue FieldAddr = Addr; 1976 if (Offset != 0) 1977 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 1978 DAG.getIntPtrConstant(Offset)); 1979 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 1980 MachinePointerInfo(SV, Offset), 1981 false, false, 0); 1982 Offset += 8; 1983 } 1984 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 1985} 1986 1987SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 1988 SelectionDAG &DAG) const { 1989 SDValue Chain = Op.getOperand(0); 1990 SDValue DstPtr = Op.getOperand(1); 1991 SDValue SrcPtr = Op.getOperand(2); 1992 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 1993 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 1994 SDLoc DL(Op); 1995 1996 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32), 1997 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 1998 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 1999} 2000 2001SDValue SystemZTargetLowering:: 2002lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 2003 SDValue Chain = Op.getOperand(0); 2004 SDValue Size = Op.getOperand(1); 2005 SDLoc DL(Op); 2006 2007 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 2008 2009 // Get a reference to the stack pointer. 2010 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 2011 2012 // Get the new stack pointer value. 2013 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, Size); 2014 2015 // Copy the new stack pointer back. 2016 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 2017 2018 // The allocated data lives above the 160 bytes allocated for the standard 2019 // frame, plus any outgoing stack arguments. We don't know how much that 2020 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 2021 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 2022 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 2023 2024 SDValue Ops[2] = { Result, Chain }; 2025 return DAG.getMergeValues(Ops, DL); 2026} 2027 2028SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 2029 SelectionDAG &DAG) const { 2030 EVT VT = Op.getValueType(); 2031 SDLoc DL(Op); 2032 SDValue Ops[2]; 2033 if (is32Bit(VT)) 2034 // Just do a normal 64-bit multiplication and extract the results. 2035 // We define this so that it can be used for constant division. 2036 lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0), 2037 Op.getOperand(1), Ops[1], Ops[0]); 2038 else { 2039 // Do a full 128-bit multiplication based on UMUL_LOHI64: 2040 // 2041 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64) 2042 // 2043 // but using the fact that the upper halves are either all zeros 2044 // or all ones: 2045 // 2046 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64) 2047 // 2048 // and grouping the right terms together since they are quicker than the 2049 // multiplication: 2050 // 2051 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64) 2052 SDValue C63 = DAG.getConstant(63, MVT::i64); 2053 SDValue LL = Op.getOperand(0); 2054 SDValue RL = Op.getOperand(1); 2055 SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63); 2056 SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63); 2057 // UMUL_LOHI64 returns the low result in the odd register and the high 2058 // result in the even register. SMUL_LOHI is defined to return the 2059 // low half first, so the results are in reverse order. 2060 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2061 LL, RL, Ops[1], Ops[0]); 2062 SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH); 2063 SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL); 2064 SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL); 2065 Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum); 2066 } 2067 return DAG.getMergeValues(Ops, DL); 2068} 2069 2070SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op, 2071 SelectionDAG &DAG) const { 2072 EVT VT = Op.getValueType(); 2073 SDLoc DL(Op); 2074 SDValue Ops[2]; 2075 if (is32Bit(VT)) 2076 // Just do a normal 64-bit multiplication and extract the results. 2077 // We define this so that it can be used for constant division. 2078 lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0), 2079 Op.getOperand(1), Ops[1], Ops[0]); 2080 else 2081 // UMUL_LOHI64 returns the low result in the odd register and the high 2082 // result in the even register. UMUL_LOHI is defined to return the 2083 // low half first, so the results are in reverse order. 2084 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64, 2085 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2086 return DAG.getMergeValues(Ops, DL); 2087} 2088 2089SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op, 2090 SelectionDAG &DAG) const { 2091 SDValue Op0 = Op.getOperand(0); 2092 SDValue Op1 = Op.getOperand(1); 2093 EVT VT = Op.getValueType(); 2094 SDLoc DL(Op); 2095 unsigned Opcode; 2096 2097 // We use DSGF for 32-bit division. 2098 if (is32Bit(VT)) { 2099 Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0); 2100 Opcode = SystemZISD::SDIVREM32; 2101 } else if (DAG.ComputeNumSignBits(Op1) > 32) { 2102 Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1); 2103 Opcode = SystemZISD::SDIVREM32; 2104 } else 2105 Opcode = SystemZISD::SDIVREM64; 2106 2107 // DSG(F) takes a 64-bit dividend, so the even register in the GR128 2108 // input is "don't care". The instruction returns the remainder in 2109 // the even register and the quotient in the odd register. 2110 SDValue Ops[2]; 2111 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode, 2112 Op0, Op1, Ops[1], Ops[0]); 2113 return DAG.getMergeValues(Ops, DL); 2114} 2115 2116SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op, 2117 SelectionDAG &DAG) const { 2118 EVT VT = Op.getValueType(); 2119 SDLoc DL(Op); 2120 2121 // DL(G) uses a double-width dividend, so we need to clear the even 2122 // register in the GR128 input. The instruction returns the remainder 2123 // in the even register and the quotient in the odd register. 2124 SDValue Ops[2]; 2125 if (is32Bit(VT)) 2126 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32, 2127 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2128 else 2129 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64, 2130 Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]); 2131 return DAG.getMergeValues(Ops, DL); 2132} 2133 2134SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const { 2135 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation"); 2136 2137 // Get the known-zero masks for each operand. 2138 SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) }; 2139 APInt KnownZero[2], KnownOne[2]; 2140 DAG.computeKnownBits(Ops[0], KnownZero[0], KnownOne[0]); 2141 DAG.computeKnownBits(Ops[1], KnownZero[1], KnownOne[1]); 2142 2143 // See if the upper 32 bits of one operand and the lower 32 bits of the 2144 // other are known zero. They are the low and high operands respectively. 2145 uint64_t Masks[] = { KnownZero[0].getZExtValue(), 2146 KnownZero[1].getZExtValue() }; 2147 unsigned High, Low; 2148 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff) 2149 High = 1, Low = 0; 2150 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff) 2151 High = 0, Low = 1; 2152 else 2153 return Op; 2154 2155 SDValue LowOp = Ops[Low]; 2156 SDValue HighOp = Ops[High]; 2157 2158 // If the high part is a constant, we're better off using IILH. 2159 if (HighOp.getOpcode() == ISD::Constant) 2160 return Op; 2161 2162 // If the low part is a constant that is outside the range of LHI, 2163 // then we're better off using IILF. 2164 if (LowOp.getOpcode() == ISD::Constant) { 2165 int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue()); 2166 if (!isInt<16>(Value)) 2167 return Op; 2168 } 2169 2170 // Check whether the high part is an AND that doesn't change the 2171 // high 32 bits and just masks out low bits. We can skip it if so. 2172 if (HighOp.getOpcode() == ISD::AND && 2173 HighOp.getOperand(1).getOpcode() == ISD::Constant) { 2174 SDValue HighOp0 = HighOp.getOperand(0); 2175 uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue(); 2176 if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff)))) 2177 HighOp = HighOp0; 2178 } 2179 2180 // Take advantage of the fact that all GR32 operations only change the 2181 // low 32 bits by truncating Low to an i32 and inserting it directly 2182 // using a subreg. The interesting cases are those where the truncation 2183 // can be folded. 2184 SDLoc DL(Op); 2185 SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp); 2186 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL, 2187 MVT::i64, HighOp, Low32); 2188} 2189 2190// Op is an atomic load. Lower it into a normal volatile load. 2191SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op, 2192 SelectionDAG &DAG) const { 2193 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2194 return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(), 2195 Node->getChain(), Node->getBasePtr(), 2196 Node->getMemoryVT(), Node->getMemOperand()); 2197} 2198 2199// Op is an atomic store. Lower it into a normal volatile store followed 2200// by a serialization. 2201SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op, 2202 SelectionDAG &DAG) const { 2203 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2204 SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(), 2205 Node->getBasePtr(), Node->getMemoryVT(), 2206 Node->getMemOperand()); 2207 return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other, 2208 Chain), 0); 2209} 2210 2211// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first 2212// two into the fullword ATOMIC_LOADW_* operation given by Opcode. 2213SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, 2214 SelectionDAG &DAG, 2215 unsigned Opcode) const { 2216 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2217 2218 // 32-bit operations need no code outside the main loop. 2219 EVT NarrowVT = Node->getMemoryVT(); 2220 EVT WideVT = MVT::i32; 2221 if (NarrowVT == WideVT) 2222 return Op; 2223 2224 int64_t BitSize = NarrowVT.getSizeInBits(); 2225 SDValue ChainIn = Node->getChain(); 2226 SDValue Addr = Node->getBasePtr(); 2227 SDValue Src2 = Node->getVal(); 2228 MachineMemOperand *MMO = Node->getMemOperand(); 2229 SDLoc DL(Node); 2230 EVT PtrVT = Addr.getValueType(); 2231 2232 // Convert atomic subtracts of constants into additions. 2233 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) 2234 if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) { 2235 Opcode = SystemZISD::ATOMIC_LOADW_ADD; 2236 Src2 = DAG.getConstant(-Const->getSExtValue(), Src2.getValueType()); 2237 } 2238 2239 // Get the address of the containing word. 2240 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 2241 DAG.getConstant(-4, PtrVT)); 2242 2243 // Get the number of bits that the word must be rotated left in order 2244 // to bring the field to the top bits of a GR32. 2245 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 2246 DAG.getConstant(3, PtrVT)); 2247 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 2248 2249 // Get the complementing shift amount, for rotating a field in the top 2250 // bits back to its proper position. 2251 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 2252 DAG.getConstant(0, WideVT), BitShift); 2253 2254 // Extend the source operand to 32 bits and prepare it for the inner loop. 2255 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other 2256 // operations require the source to be shifted in advance. (This shift 2257 // can be folded if the source is constant.) For AND and NAND, the lower 2258 // bits must be set, while for other opcodes they should be left clear. 2259 if (Opcode != SystemZISD::ATOMIC_SWAPW) 2260 Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2, 2261 DAG.getConstant(32 - BitSize, WideVT)); 2262 if (Opcode == SystemZISD::ATOMIC_LOADW_AND || 2263 Opcode == SystemZISD::ATOMIC_LOADW_NAND) 2264 Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2, 2265 DAG.getConstant(uint32_t(-1) >> BitSize, WideVT)); 2266 2267 // Construct the ATOMIC_LOADW_* node. 2268 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2269 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift, 2270 DAG.getConstant(BitSize, WideVT) }; 2271 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops, 2272 NarrowVT, MMO); 2273 2274 // Rotate the result of the final CS so that the field is in the lower 2275 // bits of a GR32, then truncate it. 2276 SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift, 2277 DAG.getConstant(BitSize, WideVT)); 2278 SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift); 2279 2280 SDValue RetOps[2] = { Result, AtomicOp.getValue(1) }; 2281 return DAG.getMergeValues(RetOps, DL); 2282} 2283 2284// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations 2285// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit 2286// operations into additions. 2287SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op, 2288 SelectionDAG &DAG) const { 2289 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2290 EVT MemVT = Node->getMemoryVT(); 2291 if (MemVT == MVT::i32 || MemVT == MVT::i64) { 2292 // A full-width operation. 2293 assert(Op.getValueType() == MemVT && "Mismatched VTs"); 2294 SDValue Src2 = Node->getVal(); 2295 SDValue NegSrc2; 2296 SDLoc DL(Src2); 2297 2298 if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) { 2299 // Use an addition if the operand is constant and either LAA(G) is 2300 // available or the negative value is in the range of A(G)FHI. 2301 int64_t Value = (-Op2->getAPIntValue()).getSExtValue(); 2302 if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1()) 2303 NegSrc2 = DAG.getConstant(Value, MemVT); 2304 } else if (Subtarget.hasInterlockedAccess1()) 2305 // Use LAA(G) if available. 2306 NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, MemVT), 2307 Src2); 2308 2309 if (NegSrc2.getNode()) 2310 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, 2311 Node->getChain(), Node->getBasePtr(), NegSrc2, 2312 Node->getMemOperand(), Node->getOrdering(), 2313 Node->getSynchScope()); 2314 2315 // Use the node as-is. 2316 return Op; 2317 } 2318 2319 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); 2320} 2321 2322// Node is an 8- or 16-bit ATOMIC_CMP_SWAP operation. Lower the first two 2323// into a fullword ATOMIC_CMP_SWAPW operation. 2324SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, 2325 SelectionDAG &DAG) const { 2326 auto *Node = cast<AtomicSDNode>(Op.getNode()); 2327 2328 // We have native support for 32-bit compare and swap. 2329 EVT NarrowVT = Node->getMemoryVT(); 2330 EVT WideVT = MVT::i32; 2331 if (NarrowVT == WideVT) 2332 return Op; 2333 2334 int64_t BitSize = NarrowVT.getSizeInBits(); 2335 SDValue ChainIn = Node->getOperand(0); 2336 SDValue Addr = Node->getOperand(1); 2337 SDValue CmpVal = Node->getOperand(2); 2338 SDValue SwapVal = Node->getOperand(3); 2339 MachineMemOperand *MMO = Node->getMemOperand(); 2340 SDLoc DL(Node); 2341 EVT PtrVT = Addr.getValueType(); 2342 2343 // Get the address of the containing word. 2344 SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, 2345 DAG.getConstant(-4, PtrVT)); 2346 2347 // Get the number of bits that the word must be rotated left in order 2348 // to bring the field to the top bits of a GR32. 2349 SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, 2350 DAG.getConstant(3, PtrVT)); 2351 BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); 2352 2353 // Get the complementing shift amount, for rotating a field in the top 2354 // bits back to its proper position. 2355 SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, 2356 DAG.getConstant(0, WideVT), BitShift); 2357 2358 // Construct the ATOMIC_CMP_SWAPW node. 2359 SDVTList VTList = DAG.getVTList(WideVT, MVT::Other); 2360 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift, 2361 NegBitShift, DAG.getConstant(BitSize, WideVT) }; 2362 SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL, 2363 VTList, Ops, NarrowVT, MMO); 2364 return AtomicOp; 2365} 2366 2367SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op, 2368 SelectionDAG &DAG) const { 2369 MachineFunction &MF = DAG.getMachineFunction(); 2370 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2371 return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op), 2372 SystemZ::R15D, Op.getValueType()); 2373} 2374 2375SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op, 2376 SelectionDAG &DAG) const { 2377 MachineFunction &MF = DAG.getMachineFunction(); 2378 MF.getInfo<SystemZMachineFunctionInfo>()->setManipulatesSP(true); 2379 return DAG.getCopyToReg(Op.getOperand(0), SDLoc(Op), 2380 SystemZ::R15D, Op.getOperand(1)); 2381} 2382 2383SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op, 2384 SelectionDAG &DAG) const { 2385 bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue(); 2386 if (!IsData) 2387 // Just preserve the chain. 2388 return Op.getOperand(0); 2389 2390 bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); 2391 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ; 2392 auto *Node = cast<MemIntrinsicSDNode>(Op.getNode()); 2393 SDValue Ops[] = { 2394 Op.getOperand(0), 2395 DAG.getConstant(Code, MVT::i32), 2396 Op.getOperand(1) 2397 }; 2398 return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, SDLoc(Op), 2399 Node->getVTList(), Ops, 2400 Node->getMemoryVT(), Node->getMemOperand()); 2401} 2402 2403SDValue SystemZTargetLowering::LowerOperation(SDValue Op, 2404 SelectionDAG &DAG) const { 2405 switch (Op.getOpcode()) { 2406 case ISD::BR_CC: 2407 return lowerBR_CC(Op, DAG); 2408 case ISD::SELECT_CC: 2409 return lowerSELECT_CC(Op, DAG); 2410 case ISD::SETCC: 2411 return lowerSETCC(Op, DAG); 2412 case ISD::GlobalAddress: 2413 return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG); 2414 case ISD::GlobalTLSAddress: 2415 return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG); 2416 case ISD::BlockAddress: 2417 return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG); 2418 case ISD::JumpTable: 2419 return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG); 2420 case ISD::ConstantPool: 2421 return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG); 2422 case ISD::BITCAST: 2423 return lowerBITCAST(Op, DAG); 2424 case ISD::VASTART: 2425 return lowerVASTART(Op, DAG); 2426 case ISD::VACOPY: 2427 return lowerVACOPY(Op, DAG); 2428 case ISD::DYNAMIC_STACKALLOC: 2429 return lowerDYNAMIC_STACKALLOC(Op, DAG); 2430 case ISD::SMUL_LOHI: 2431 return lowerSMUL_LOHI(Op, DAG); 2432 case ISD::UMUL_LOHI: 2433 return lowerUMUL_LOHI(Op, DAG); 2434 case ISD::SDIVREM: 2435 return lowerSDIVREM(Op, DAG); 2436 case ISD::UDIVREM: 2437 return lowerUDIVREM(Op, DAG); 2438 case ISD::OR: 2439 return lowerOR(Op, DAG); 2440 case ISD::ATOMIC_SWAP: 2441 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW); 2442 case ISD::ATOMIC_STORE: 2443 return lowerATOMIC_STORE(Op, DAG); 2444 case ISD::ATOMIC_LOAD: 2445 return lowerATOMIC_LOAD(Op, DAG); 2446 case ISD::ATOMIC_LOAD_ADD: 2447 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD); 2448 case ISD::ATOMIC_LOAD_SUB: 2449 return lowerATOMIC_LOAD_SUB(Op, DAG); 2450 case ISD::ATOMIC_LOAD_AND: 2451 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND); 2452 case ISD::ATOMIC_LOAD_OR: 2453 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR); 2454 case ISD::ATOMIC_LOAD_XOR: 2455 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR); 2456 case ISD::ATOMIC_LOAD_NAND: 2457 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND); 2458 case ISD::ATOMIC_LOAD_MIN: 2459 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN); 2460 case ISD::ATOMIC_LOAD_MAX: 2461 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX); 2462 case ISD::ATOMIC_LOAD_UMIN: 2463 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN); 2464 case ISD::ATOMIC_LOAD_UMAX: 2465 return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX); 2466 case ISD::ATOMIC_CMP_SWAP: 2467 return lowerATOMIC_CMP_SWAP(Op, DAG); 2468 case ISD::STACKSAVE: 2469 return lowerSTACKSAVE(Op, DAG); 2470 case ISD::STACKRESTORE: 2471 return lowerSTACKRESTORE(Op, DAG); 2472 case ISD::PREFETCH: 2473 return lowerPREFETCH(Op, DAG); 2474 default: 2475 llvm_unreachable("Unexpected node to lower"); 2476 } 2477} 2478 2479const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { 2480#define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME 2481 switch (Opcode) { 2482 OPCODE(RET_FLAG); 2483 OPCODE(CALL); 2484 OPCODE(SIBCALL); 2485 OPCODE(PCREL_WRAPPER); 2486 OPCODE(PCREL_OFFSET); 2487 OPCODE(IABS); 2488 OPCODE(ICMP); 2489 OPCODE(FCMP); 2490 OPCODE(TM); 2491 OPCODE(BR_CCMASK); 2492 OPCODE(SELECT_CCMASK); 2493 OPCODE(ADJDYNALLOC); 2494 OPCODE(EXTRACT_ACCESS); 2495 OPCODE(UMUL_LOHI64); 2496 OPCODE(SDIVREM64); 2497 OPCODE(UDIVREM32); 2498 OPCODE(UDIVREM64); 2499 OPCODE(MVC); 2500 OPCODE(MVC_LOOP); 2501 OPCODE(NC); 2502 OPCODE(NC_LOOP); 2503 OPCODE(OC); 2504 OPCODE(OC_LOOP); 2505 OPCODE(XC); 2506 OPCODE(XC_LOOP); 2507 OPCODE(CLC); 2508 OPCODE(CLC_LOOP); 2509 OPCODE(STRCMP); 2510 OPCODE(STPCPY); 2511 OPCODE(SEARCH_STRING); 2512 OPCODE(IPM); 2513 OPCODE(SERIALIZE); 2514 OPCODE(ATOMIC_SWAPW); 2515 OPCODE(ATOMIC_LOADW_ADD); 2516 OPCODE(ATOMIC_LOADW_SUB); 2517 OPCODE(ATOMIC_LOADW_AND); 2518 OPCODE(ATOMIC_LOADW_OR); 2519 OPCODE(ATOMIC_LOADW_XOR); 2520 OPCODE(ATOMIC_LOADW_NAND); 2521 OPCODE(ATOMIC_LOADW_MIN); 2522 OPCODE(ATOMIC_LOADW_MAX); 2523 OPCODE(ATOMIC_LOADW_UMIN); 2524 OPCODE(ATOMIC_LOADW_UMAX); 2525 OPCODE(ATOMIC_CMP_SWAPW); 2526 OPCODE(PREFETCH); 2527 } 2528 return nullptr; 2529#undef OPCODE 2530} 2531 2532SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, 2533 DAGCombinerInfo &DCI) const { 2534 SelectionDAG &DAG = DCI.DAG; 2535 unsigned Opcode = N->getOpcode(); 2536 if (Opcode == ISD::SIGN_EXTEND) { 2537 // Convert (sext (ashr (shl X, C1), C2)) to 2538 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as 2539 // cheap as narrower ones. 2540 SDValue N0 = N->getOperand(0); 2541 EVT VT = N->getValueType(0); 2542 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) { 2543 auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)); 2544 SDValue Inner = N0.getOperand(0); 2545 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) { 2546 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) { 2547 unsigned Extra = (VT.getSizeInBits() - 2548 N0.getValueType().getSizeInBits()); 2549 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra; 2550 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra; 2551 EVT ShiftVT = N0.getOperand(1).getValueType(); 2552 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT, 2553 Inner.getOperand(0)); 2554 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext, 2555 DAG.getConstant(NewShlAmt, ShiftVT)); 2556 return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, 2557 DAG.getConstant(NewSraAmt, ShiftVT)); 2558 } 2559 } 2560 } 2561 } 2562 return SDValue(); 2563} 2564 2565//===----------------------------------------------------------------------===// 2566// Custom insertion 2567//===----------------------------------------------------------------------===// 2568 2569// Create a new basic block after MBB. 2570static MachineBasicBlock *emitBlockAfter(MachineBasicBlock *MBB) { 2571 MachineFunction &MF = *MBB->getParent(); 2572 MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock()); 2573 MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB); 2574 return NewMBB; 2575} 2576 2577// Split MBB after MI and return the new block (the one that contains 2578// instructions after MI). 2579static MachineBasicBlock *splitBlockAfter(MachineInstr *MI, 2580 MachineBasicBlock *MBB) { 2581 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2582 NewMBB->splice(NewMBB->begin(), MBB, 2583 std::next(MachineBasicBlock::iterator(MI)), MBB->end()); 2584 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2585 return NewMBB; 2586} 2587 2588// Split MBB before MI and return the new block (the one that contains MI). 2589static MachineBasicBlock *splitBlockBefore(MachineInstr *MI, 2590 MachineBasicBlock *MBB) { 2591 MachineBasicBlock *NewMBB = emitBlockAfter(MBB); 2592 NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end()); 2593 NewMBB->transferSuccessorsAndUpdatePHIs(MBB); 2594 return NewMBB; 2595} 2596 2597// Force base value Base into a register before MI. Return the register. 2598static unsigned forceReg(MachineInstr *MI, MachineOperand &Base, 2599 const SystemZInstrInfo *TII) { 2600 if (Base.isReg()) 2601 return Base.getReg(); 2602 2603 MachineBasicBlock *MBB = MI->getParent(); 2604 MachineFunction &MF = *MBB->getParent(); 2605 MachineRegisterInfo &MRI = MF.getRegInfo(); 2606 2607 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 2608 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LA), Reg) 2609 .addOperand(Base).addImm(0).addReg(0); 2610 return Reg; 2611} 2612 2613// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI. 2614MachineBasicBlock * 2615SystemZTargetLowering::emitSelect(MachineInstr *MI, 2616 MachineBasicBlock *MBB) const { 2617 const SystemZInstrInfo *TII = static_cast<const SystemZInstrInfo *>( 2618 MBB->getParent()->getSubtarget().getInstrInfo()); 2619 2620 unsigned DestReg = MI->getOperand(0).getReg(); 2621 unsigned TrueReg = MI->getOperand(1).getReg(); 2622 unsigned FalseReg = MI->getOperand(2).getReg(); 2623 unsigned CCValid = MI->getOperand(3).getImm(); 2624 unsigned CCMask = MI->getOperand(4).getImm(); 2625 DebugLoc DL = MI->getDebugLoc(); 2626 2627 MachineBasicBlock *StartMBB = MBB; 2628 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2629 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2630 2631 // StartMBB: 2632 // BRC CCMask, JoinMBB 2633 // # fallthrough to FalseMBB 2634 MBB = StartMBB; 2635 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2636 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2637 MBB->addSuccessor(JoinMBB); 2638 MBB->addSuccessor(FalseMBB); 2639 2640 // FalseMBB: 2641 // # fallthrough to JoinMBB 2642 MBB = FalseMBB; 2643 MBB->addSuccessor(JoinMBB); 2644 2645 // JoinMBB: 2646 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ] 2647 // ... 2648 MBB = JoinMBB; 2649 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg) 2650 .addReg(TrueReg).addMBB(StartMBB) 2651 .addReg(FalseReg).addMBB(FalseMBB); 2652 2653 MI->eraseFromParent(); 2654 return JoinMBB; 2655} 2656 2657// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI. 2658// StoreOpcode is the store to use and Invert says whether the store should 2659// happen when the condition is false rather than true. If a STORE ON 2660// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0. 2661MachineBasicBlock * 2662SystemZTargetLowering::emitCondStore(MachineInstr *MI, 2663 MachineBasicBlock *MBB, 2664 unsigned StoreOpcode, unsigned STOCOpcode, 2665 bool Invert) const { 2666 const SystemZInstrInfo *TII = static_cast<const SystemZInstrInfo *>( 2667 MBB->getParent()->getSubtarget().getInstrInfo()); 2668 2669 unsigned SrcReg = MI->getOperand(0).getReg(); 2670 MachineOperand Base = MI->getOperand(1); 2671 int64_t Disp = MI->getOperand(2).getImm(); 2672 unsigned IndexReg = MI->getOperand(3).getReg(); 2673 unsigned CCValid = MI->getOperand(4).getImm(); 2674 unsigned CCMask = MI->getOperand(5).getImm(); 2675 DebugLoc DL = MI->getDebugLoc(); 2676 2677 StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp); 2678 2679 // Use STOCOpcode if possible. We could use different store patterns in 2680 // order to avoid matching the index register, but the performance trade-offs 2681 // might be more complicated in that case. 2682 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) { 2683 if (Invert) 2684 CCMask ^= CCValid; 2685 BuildMI(*MBB, MI, DL, TII->get(STOCOpcode)) 2686 .addReg(SrcReg).addOperand(Base).addImm(Disp) 2687 .addImm(CCValid).addImm(CCMask); 2688 MI->eraseFromParent(); 2689 return MBB; 2690 } 2691 2692 // Get the condition needed to branch around the store. 2693 if (!Invert) 2694 CCMask ^= CCValid; 2695 2696 MachineBasicBlock *StartMBB = MBB; 2697 MachineBasicBlock *JoinMBB = splitBlockBefore(MI, MBB); 2698 MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB); 2699 2700 // StartMBB: 2701 // BRC CCMask, JoinMBB 2702 // # fallthrough to FalseMBB 2703 MBB = StartMBB; 2704 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2705 .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB); 2706 MBB->addSuccessor(JoinMBB); 2707 MBB->addSuccessor(FalseMBB); 2708 2709 // FalseMBB: 2710 // store %SrcReg, %Disp(%Index,%Base) 2711 // # fallthrough to JoinMBB 2712 MBB = FalseMBB; 2713 BuildMI(MBB, DL, TII->get(StoreOpcode)) 2714 .addReg(SrcReg).addOperand(Base).addImm(Disp).addReg(IndexReg); 2715 MBB->addSuccessor(JoinMBB); 2716 2717 MI->eraseFromParent(); 2718 return JoinMBB; 2719} 2720 2721// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* 2722// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that 2723// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. 2724// BitSize is the width of the field in bits, or 0 if this is a partword 2725// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize 2726// is one of the operands. Invert says whether the field should be 2727// inverted after performing BinOpcode (e.g. for NAND). 2728MachineBasicBlock * 2729SystemZTargetLowering::emitAtomicLoadBinary(MachineInstr *MI, 2730 MachineBasicBlock *MBB, 2731 unsigned BinOpcode, 2732 unsigned BitSize, 2733 bool Invert) const { 2734 MachineFunction &MF = *MBB->getParent(); 2735 const SystemZInstrInfo *TII = 2736 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 2737 MachineRegisterInfo &MRI = MF.getRegInfo(); 2738 bool IsSubWord = (BitSize < 32); 2739 2740 // Extract the operands. Base can be a register or a frame index. 2741 // Src2 can be a register or immediate. 2742 unsigned Dest = MI->getOperand(0).getReg(); 2743 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2744 int64_t Disp = MI->getOperand(2).getImm(); 2745 MachineOperand Src2 = earlyUseOperand(MI->getOperand(3)); 2746 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2747 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2748 DebugLoc DL = MI->getDebugLoc(); 2749 if (IsSubWord) 2750 BitSize = MI->getOperand(6).getImm(); 2751 2752 // Subword operations use 32-bit registers. 2753 const TargetRegisterClass *RC = (BitSize <= 32 ? 2754 &SystemZ::GR32BitRegClass : 2755 &SystemZ::GR64BitRegClass); 2756 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2757 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2758 2759 // Get the right opcodes for the displacement. 2760 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2761 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2762 assert(LOpcode && CSOpcode && "Displacement out of range"); 2763 2764 // Create virtual registers for temporary results. 2765 unsigned OrigVal = MRI.createVirtualRegister(RC); 2766 unsigned OldVal = MRI.createVirtualRegister(RC); 2767 unsigned NewVal = (BinOpcode || IsSubWord ? 2768 MRI.createVirtualRegister(RC) : Src2.getReg()); 2769 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2770 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2771 2772 // Insert a basic block for the main loop. 2773 MachineBasicBlock *StartMBB = MBB; 2774 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2775 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2776 2777 // StartMBB: 2778 // ... 2779 // %OrigVal = L Disp(%Base) 2780 // # fall through to LoopMMB 2781 MBB = StartMBB; 2782 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2783 .addOperand(Base).addImm(Disp).addReg(0); 2784 MBB->addSuccessor(LoopMBB); 2785 2786 // LoopMBB: 2787 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ] 2788 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2789 // %RotatedNewVal = OP %RotatedOldVal, %Src2 2790 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2791 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2792 // JNE LoopMBB 2793 // # fall through to DoneMMB 2794 MBB = LoopMBB; 2795 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2796 .addReg(OrigVal).addMBB(StartMBB) 2797 .addReg(Dest).addMBB(LoopMBB); 2798 if (IsSubWord) 2799 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2800 .addReg(OldVal).addReg(BitShift).addImm(0); 2801 if (Invert) { 2802 // Perform the operation normally and then invert every bit of the field. 2803 unsigned Tmp = MRI.createVirtualRegister(RC); 2804 BuildMI(MBB, DL, TII->get(BinOpcode), Tmp) 2805 .addReg(RotatedOldVal).addOperand(Src2); 2806 if (BitSize <= 32) 2807 // XILF with the upper BitSize bits set. 2808 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) 2809 .addReg(Tmp).addImm(-1U << (32 - BitSize)); 2810 else { 2811 // Use LCGR and add -1 to the result, which is more compact than 2812 // an XILF, XILH pair. 2813 unsigned Tmp2 = MRI.createVirtualRegister(RC); 2814 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); 2815 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) 2816 .addReg(Tmp2).addImm(-1); 2817 } 2818 } else if (BinOpcode) 2819 // A simply binary operation. 2820 BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) 2821 .addReg(RotatedOldVal).addOperand(Src2); 2822 else if (IsSubWord) 2823 // Use RISBG to rotate Src2 into position and use it to replace the 2824 // field in RotatedOldVal. 2825 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) 2826 .addReg(RotatedOldVal).addReg(Src2.getReg()) 2827 .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); 2828 if (IsSubWord) 2829 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2830 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2831 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2832 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2833 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2834 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2835 MBB->addSuccessor(LoopMBB); 2836 MBB->addSuccessor(DoneMBB); 2837 2838 MI->eraseFromParent(); 2839 return DoneMBB; 2840} 2841 2842// Implement EmitInstrWithCustomInserter for pseudo 2843// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the 2844// instruction that should be used to compare the current field with the 2845// minimum or maximum value. KeepOldMask is the BRC condition-code mask 2846// for when the current field should be kept. BitSize is the width of 2847// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. 2848MachineBasicBlock * 2849SystemZTargetLowering::emitAtomicLoadMinMax(MachineInstr *MI, 2850 MachineBasicBlock *MBB, 2851 unsigned CompareOpcode, 2852 unsigned KeepOldMask, 2853 unsigned BitSize) const { 2854 MachineFunction &MF = *MBB->getParent(); 2855 const SystemZInstrInfo *TII = 2856 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 2857 MachineRegisterInfo &MRI = MF.getRegInfo(); 2858 bool IsSubWord = (BitSize < 32); 2859 2860 // Extract the operands. Base can be a register or a frame index. 2861 unsigned Dest = MI->getOperand(0).getReg(); 2862 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2863 int64_t Disp = MI->getOperand(2).getImm(); 2864 unsigned Src2 = MI->getOperand(3).getReg(); 2865 unsigned BitShift = (IsSubWord ? MI->getOperand(4).getReg() : 0); 2866 unsigned NegBitShift = (IsSubWord ? MI->getOperand(5).getReg() : 0); 2867 DebugLoc DL = MI->getDebugLoc(); 2868 if (IsSubWord) 2869 BitSize = MI->getOperand(6).getImm(); 2870 2871 // Subword operations use 32-bit registers. 2872 const TargetRegisterClass *RC = (BitSize <= 32 ? 2873 &SystemZ::GR32BitRegClass : 2874 &SystemZ::GR64BitRegClass); 2875 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; 2876 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; 2877 2878 // Get the right opcodes for the displacement. 2879 LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); 2880 CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); 2881 assert(LOpcode && CSOpcode && "Displacement out of range"); 2882 2883 // Create virtual registers for temporary results. 2884 unsigned OrigVal = MRI.createVirtualRegister(RC); 2885 unsigned OldVal = MRI.createVirtualRegister(RC); 2886 unsigned NewVal = MRI.createVirtualRegister(RC); 2887 unsigned RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); 2888 unsigned RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); 2889 unsigned RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); 2890 2891 // Insert 3 basic blocks for the loop. 2892 MachineBasicBlock *StartMBB = MBB; 2893 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 2894 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 2895 MachineBasicBlock *UseAltMBB = emitBlockAfter(LoopMBB); 2896 MachineBasicBlock *UpdateMBB = emitBlockAfter(UseAltMBB); 2897 2898 // StartMBB: 2899 // ... 2900 // %OrigVal = L Disp(%Base) 2901 // # fall through to LoopMMB 2902 MBB = StartMBB; 2903 BuildMI(MBB, DL, TII->get(LOpcode), OrigVal) 2904 .addOperand(Base).addImm(Disp).addReg(0); 2905 MBB->addSuccessor(LoopMBB); 2906 2907 // LoopMBB: 2908 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ] 2909 // %RotatedOldVal = RLL %OldVal, 0(%BitShift) 2910 // CompareOpcode %RotatedOldVal, %Src2 2911 // BRC KeepOldMask, UpdateMBB 2912 MBB = LoopMBB; 2913 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 2914 .addReg(OrigVal).addMBB(StartMBB) 2915 .addReg(Dest).addMBB(UpdateMBB); 2916 if (IsSubWord) 2917 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) 2918 .addReg(OldVal).addReg(BitShift).addImm(0); 2919 BuildMI(MBB, DL, TII->get(CompareOpcode)) 2920 .addReg(RotatedOldVal).addReg(Src2); 2921 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2922 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB); 2923 MBB->addSuccessor(UpdateMBB); 2924 MBB->addSuccessor(UseAltMBB); 2925 2926 // UseAltMBB: 2927 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 2928 // # fall through to UpdateMMB 2929 MBB = UseAltMBB; 2930 if (IsSubWord) 2931 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) 2932 .addReg(RotatedOldVal).addReg(Src2) 2933 .addImm(32).addImm(31 + BitSize).addImm(0); 2934 MBB->addSuccessor(UpdateMBB); 2935 2936 // UpdateMBB: 2937 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ], 2938 // [ %RotatedAltVal, UseAltMBB ] 2939 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift) 2940 // %Dest = CS %OldVal, %NewVal, Disp(%Base) 2941 // JNE LoopMBB 2942 // # fall through to DoneMMB 2943 MBB = UpdateMBB; 2944 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) 2945 .addReg(RotatedOldVal).addMBB(LoopMBB) 2946 .addReg(RotatedAltVal).addMBB(UseAltMBB); 2947 if (IsSubWord) 2948 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) 2949 .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); 2950 BuildMI(MBB, DL, TII->get(CSOpcode), Dest) 2951 .addReg(OldVal).addReg(NewVal).addOperand(Base).addImm(Disp); 2952 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 2953 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 2954 MBB->addSuccessor(LoopMBB); 2955 MBB->addSuccessor(DoneMBB); 2956 2957 MI->eraseFromParent(); 2958 return DoneMBB; 2959} 2960 2961// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW 2962// instruction MI. 2963MachineBasicBlock * 2964SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr *MI, 2965 MachineBasicBlock *MBB) const { 2966 MachineFunction &MF = *MBB->getParent(); 2967 const SystemZInstrInfo *TII = 2968 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 2969 MachineRegisterInfo &MRI = MF.getRegInfo(); 2970 2971 // Extract the operands. Base can be a register or a frame index. 2972 unsigned Dest = MI->getOperand(0).getReg(); 2973 MachineOperand Base = earlyUseOperand(MI->getOperand(1)); 2974 int64_t Disp = MI->getOperand(2).getImm(); 2975 unsigned OrigCmpVal = MI->getOperand(3).getReg(); 2976 unsigned OrigSwapVal = MI->getOperand(4).getReg(); 2977 unsigned BitShift = MI->getOperand(5).getReg(); 2978 unsigned NegBitShift = MI->getOperand(6).getReg(); 2979 int64_t BitSize = MI->getOperand(7).getImm(); 2980 DebugLoc DL = MI->getDebugLoc(); 2981 2982 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass; 2983 2984 // Get the right opcodes for the displacement. 2985 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); 2986 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); 2987 assert(LOpcode && CSOpcode && "Displacement out of range"); 2988 2989 // Create virtual registers for temporary results. 2990 unsigned OrigOldVal = MRI.createVirtualRegister(RC); 2991 unsigned OldVal = MRI.createVirtualRegister(RC); 2992 unsigned CmpVal = MRI.createVirtualRegister(RC); 2993 unsigned SwapVal = MRI.createVirtualRegister(RC); 2994 unsigned StoreVal = MRI.createVirtualRegister(RC); 2995 unsigned RetryOldVal = MRI.createVirtualRegister(RC); 2996 unsigned RetryCmpVal = MRI.createVirtualRegister(RC); 2997 unsigned RetrySwapVal = MRI.createVirtualRegister(RC); 2998 2999 // Insert 2 basic blocks for the loop. 3000 MachineBasicBlock *StartMBB = MBB; 3001 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 3002 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 3003 MachineBasicBlock *SetMBB = emitBlockAfter(LoopMBB); 3004 3005 // StartMBB: 3006 // ... 3007 // %OrigOldVal = L Disp(%Base) 3008 // # fall through to LoopMMB 3009 MBB = StartMBB; 3010 BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal) 3011 .addOperand(Base).addImm(Disp).addReg(0); 3012 MBB->addSuccessor(LoopMBB); 3013 3014 // LoopMBB: 3015 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ] 3016 // %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ] 3017 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ] 3018 // %Dest = RLL %OldVal, BitSize(%BitShift) 3019 // ^^ The low BitSize bits contain the field 3020 // of interest. 3021 // %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0 3022 // ^^ Replace the upper 32-BitSize bits of the 3023 // comparison value with those that we loaded, 3024 // so that we can use a full word comparison. 3025 // CR %Dest, %RetryCmpVal 3026 // JNE DoneMBB 3027 // # Fall through to SetMBB 3028 MBB = LoopMBB; 3029 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) 3030 .addReg(OrigOldVal).addMBB(StartMBB) 3031 .addReg(RetryOldVal).addMBB(SetMBB); 3032 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal) 3033 .addReg(OrigCmpVal).addMBB(StartMBB) 3034 .addReg(RetryCmpVal).addMBB(SetMBB); 3035 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal) 3036 .addReg(OrigSwapVal).addMBB(StartMBB) 3037 .addReg(RetrySwapVal).addMBB(SetMBB); 3038 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest) 3039 .addReg(OldVal).addReg(BitShift).addImm(BitSize); 3040 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal) 3041 .addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 3042 BuildMI(MBB, DL, TII->get(SystemZ::CR)) 3043 .addReg(Dest).addReg(RetryCmpVal); 3044 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3045 .addImm(SystemZ::CCMASK_ICMP) 3046 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB); 3047 MBB->addSuccessor(DoneMBB); 3048 MBB->addSuccessor(SetMBB); 3049 3050 // SetMBB: 3051 // %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0 3052 // ^^ Replace the upper 32-BitSize bits of the new 3053 // value with those that we loaded. 3054 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift) 3055 // ^^ Rotate the new field to its proper position. 3056 // %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base) 3057 // JNE LoopMBB 3058 // # fall through to ExitMMB 3059 MBB = SetMBB; 3060 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal) 3061 .addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0); 3062 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal) 3063 .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize); 3064 BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal) 3065 .addReg(OldVal).addReg(StoreVal).addOperand(Base).addImm(Disp); 3066 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3067 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB); 3068 MBB->addSuccessor(LoopMBB); 3069 MBB->addSuccessor(DoneMBB); 3070 3071 MI->eraseFromParent(); 3072 return DoneMBB; 3073} 3074 3075// Emit an extension from a GR32 or GR64 to a GR128. ClearEven is true 3076// if the high register of the GR128 value must be cleared or false if 3077// it's "don't care". SubReg is subreg_l32 when extending a GR32 3078// and subreg_l64 when extending a GR64. 3079MachineBasicBlock * 3080SystemZTargetLowering::emitExt128(MachineInstr *MI, 3081 MachineBasicBlock *MBB, 3082 bool ClearEven, unsigned SubReg) const { 3083 MachineFunction &MF = *MBB->getParent(); 3084 const SystemZInstrInfo *TII = 3085 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 3086 MachineRegisterInfo &MRI = MF.getRegInfo(); 3087 DebugLoc DL = MI->getDebugLoc(); 3088 3089 unsigned Dest = MI->getOperand(0).getReg(); 3090 unsigned Src = MI->getOperand(1).getReg(); 3091 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 3092 3093 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128); 3094 if (ClearEven) { 3095 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass); 3096 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass); 3097 3098 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64) 3099 .addImm(0); 3100 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128) 3101 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64); 3102 In128 = NewIn128; 3103 } 3104 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest) 3105 .addReg(In128).addReg(Src).addImm(SubReg); 3106 3107 MI->eraseFromParent(); 3108 return MBB; 3109} 3110 3111MachineBasicBlock * 3112SystemZTargetLowering::emitMemMemWrapper(MachineInstr *MI, 3113 MachineBasicBlock *MBB, 3114 unsigned Opcode) const { 3115 MachineFunction &MF = *MBB->getParent(); 3116 const SystemZInstrInfo *TII = 3117 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 3118 MachineRegisterInfo &MRI = MF.getRegInfo(); 3119 DebugLoc DL = MI->getDebugLoc(); 3120 3121 MachineOperand DestBase = earlyUseOperand(MI->getOperand(0)); 3122 uint64_t DestDisp = MI->getOperand(1).getImm(); 3123 MachineOperand SrcBase = earlyUseOperand(MI->getOperand(2)); 3124 uint64_t SrcDisp = MI->getOperand(3).getImm(); 3125 uint64_t Length = MI->getOperand(4).getImm(); 3126 3127 // When generating more than one CLC, all but the last will need to 3128 // branch to the end when a difference is found. 3129 MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ? 3130 splitBlockAfter(MI, MBB) : nullptr); 3131 3132 // Check for the loop form, in which operand 5 is the trip count. 3133 if (MI->getNumExplicitOperands() > 5) { 3134 bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); 3135 3136 uint64_t StartCountReg = MI->getOperand(5).getReg(); 3137 uint64_t StartSrcReg = forceReg(MI, SrcBase, TII); 3138 uint64_t StartDestReg = (HaveSingleBase ? StartSrcReg : 3139 forceReg(MI, DestBase, TII)); 3140 3141 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass; 3142 uint64_t ThisSrcReg = MRI.createVirtualRegister(RC); 3143 uint64_t ThisDestReg = (HaveSingleBase ? ThisSrcReg : 3144 MRI.createVirtualRegister(RC)); 3145 uint64_t NextSrcReg = MRI.createVirtualRegister(RC); 3146 uint64_t NextDestReg = (HaveSingleBase ? NextSrcReg : 3147 MRI.createVirtualRegister(RC)); 3148 3149 RC = &SystemZ::GR64BitRegClass; 3150 uint64_t ThisCountReg = MRI.createVirtualRegister(RC); 3151 uint64_t NextCountReg = MRI.createVirtualRegister(RC); 3152 3153 MachineBasicBlock *StartMBB = MBB; 3154 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 3155 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 3156 MachineBasicBlock *NextMBB = (EndMBB ? emitBlockAfter(LoopMBB) : LoopMBB); 3157 3158 // StartMBB: 3159 // # fall through to LoopMMB 3160 MBB->addSuccessor(LoopMBB); 3161 3162 // LoopMBB: 3163 // %ThisDestReg = phi [ %StartDestReg, StartMBB ], 3164 // [ %NextDestReg, NextMBB ] 3165 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ], 3166 // [ %NextSrcReg, NextMBB ] 3167 // %ThisCountReg = phi [ %StartCountReg, StartMBB ], 3168 // [ %NextCountReg, NextMBB ] 3169 // ( PFD 2, 768+DestDisp(%ThisDestReg) ) 3170 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg) 3171 // ( JLH EndMBB ) 3172 // 3173 // The prefetch is used only for MVC. The JLH is used only for CLC. 3174 MBB = LoopMBB; 3175 3176 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg) 3177 .addReg(StartDestReg).addMBB(StartMBB) 3178 .addReg(NextDestReg).addMBB(NextMBB); 3179 if (!HaveSingleBase) 3180 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg) 3181 .addReg(StartSrcReg).addMBB(StartMBB) 3182 .addReg(NextSrcReg).addMBB(NextMBB); 3183 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg) 3184 .addReg(StartCountReg).addMBB(StartMBB) 3185 .addReg(NextCountReg).addMBB(NextMBB); 3186 if (Opcode == SystemZ::MVC) 3187 BuildMI(MBB, DL, TII->get(SystemZ::PFD)) 3188 .addImm(SystemZ::PFD_WRITE) 3189 .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); 3190 BuildMI(MBB, DL, TII->get(Opcode)) 3191 .addReg(ThisDestReg).addImm(DestDisp).addImm(256) 3192 .addReg(ThisSrcReg).addImm(SrcDisp); 3193 if (EndMBB) { 3194 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3195 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3196 .addMBB(EndMBB); 3197 MBB->addSuccessor(EndMBB); 3198 MBB->addSuccessor(NextMBB); 3199 } 3200 3201 // NextMBB: 3202 // %NextDestReg = LA 256(%ThisDestReg) 3203 // %NextSrcReg = LA 256(%ThisSrcReg) 3204 // %NextCountReg = AGHI %ThisCountReg, -1 3205 // CGHI %NextCountReg, 0 3206 // JLH LoopMBB 3207 // # fall through to DoneMMB 3208 // 3209 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes. 3210 MBB = NextMBB; 3211 3212 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg) 3213 .addReg(ThisDestReg).addImm(256).addReg(0); 3214 if (!HaveSingleBase) 3215 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg) 3216 .addReg(ThisSrcReg).addImm(256).addReg(0); 3217 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg) 3218 .addReg(ThisCountReg).addImm(-1); 3219 BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) 3220 .addReg(NextCountReg).addImm(0); 3221 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3222 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3223 .addMBB(LoopMBB); 3224 MBB->addSuccessor(LoopMBB); 3225 MBB->addSuccessor(DoneMBB); 3226 3227 DestBase = MachineOperand::CreateReg(NextDestReg, false); 3228 SrcBase = MachineOperand::CreateReg(NextSrcReg, false); 3229 Length &= 255; 3230 MBB = DoneMBB; 3231 } 3232 // Handle any remaining bytes with straight-line code. 3233 while (Length > 0) { 3234 uint64_t ThisLength = std::min(Length, uint64_t(256)); 3235 // The previous iteration might have created out-of-range displacements. 3236 // Apply them using LAY if so. 3237 if (!isUInt<12>(DestDisp)) { 3238 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 3239 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 3240 .addOperand(DestBase).addImm(DestDisp).addReg(0); 3241 DestBase = MachineOperand::CreateReg(Reg, false); 3242 DestDisp = 0; 3243 } 3244 if (!isUInt<12>(SrcDisp)) { 3245 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); 3246 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg) 3247 .addOperand(SrcBase).addImm(SrcDisp).addReg(0); 3248 SrcBase = MachineOperand::CreateReg(Reg, false); 3249 SrcDisp = 0; 3250 } 3251 BuildMI(*MBB, MI, DL, TII->get(Opcode)) 3252 .addOperand(DestBase).addImm(DestDisp).addImm(ThisLength) 3253 .addOperand(SrcBase).addImm(SrcDisp); 3254 DestDisp += ThisLength; 3255 SrcDisp += ThisLength; 3256 Length -= ThisLength; 3257 // If there's another CLC to go, branch to the end if a difference 3258 // was found. 3259 if (EndMBB && Length > 0) { 3260 MachineBasicBlock *NextMBB = splitBlockBefore(MI, MBB); 3261 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3262 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) 3263 .addMBB(EndMBB); 3264 MBB->addSuccessor(EndMBB); 3265 MBB->addSuccessor(NextMBB); 3266 MBB = NextMBB; 3267 } 3268 } 3269 if (EndMBB) { 3270 MBB->addSuccessor(EndMBB); 3271 MBB = EndMBB; 3272 MBB->addLiveIn(SystemZ::CC); 3273 } 3274 3275 MI->eraseFromParent(); 3276 return MBB; 3277} 3278 3279// Decompose string pseudo-instruction MI into a loop that continually performs 3280// Opcode until CC != 3. 3281MachineBasicBlock * 3282SystemZTargetLowering::emitStringWrapper(MachineInstr *MI, 3283 MachineBasicBlock *MBB, 3284 unsigned Opcode) const { 3285 MachineFunction &MF = *MBB->getParent(); 3286 const SystemZInstrInfo *TII = 3287 static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); 3288 MachineRegisterInfo &MRI = MF.getRegInfo(); 3289 DebugLoc DL = MI->getDebugLoc(); 3290 3291 uint64_t End1Reg = MI->getOperand(0).getReg(); 3292 uint64_t Start1Reg = MI->getOperand(1).getReg(); 3293 uint64_t Start2Reg = MI->getOperand(2).getReg(); 3294 uint64_t CharReg = MI->getOperand(3).getReg(); 3295 3296 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass; 3297 uint64_t This1Reg = MRI.createVirtualRegister(RC); 3298 uint64_t This2Reg = MRI.createVirtualRegister(RC); 3299 uint64_t End2Reg = MRI.createVirtualRegister(RC); 3300 3301 MachineBasicBlock *StartMBB = MBB; 3302 MachineBasicBlock *DoneMBB = splitBlockBefore(MI, MBB); 3303 MachineBasicBlock *LoopMBB = emitBlockAfter(StartMBB); 3304 3305 // StartMBB: 3306 // # fall through to LoopMMB 3307 MBB->addSuccessor(LoopMBB); 3308 3309 // LoopMBB: 3310 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ] 3311 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ] 3312 // R0L = %CharReg 3313 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L 3314 // JO LoopMBB 3315 // # fall through to DoneMMB 3316 // 3317 // The load of R0L can be hoisted by post-RA LICM. 3318 MBB = LoopMBB; 3319 3320 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg) 3321 .addReg(Start1Reg).addMBB(StartMBB) 3322 .addReg(End1Reg).addMBB(LoopMBB); 3323 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg) 3324 .addReg(Start2Reg).addMBB(StartMBB) 3325 .addReg(End2Reg).addMBB(LoopMBB); 3326 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg); 3327 BuildMI(MBB, DL, TII->get(Opcode)) 3328 .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define) 3329 .addReg(This1Reg).addReg(This2Reg); 3330 BuildMI(MBB, DL, TII->get(SystemZ::BRC)) 3331 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB); 3332 MBB->addSuccessor(LoopMBB); 3333 MBB->addSuccessor(DoneMBB); 3334 3335 DoneMBB->addLiveIn(SystemZ::CC); 3336 3337 MI->eraseFromParent(); 3338 return DoneMBB; 3339} 3340 3341MachineBasicBlock *SystemZTargetLowering:: 3342EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *MBB) const { 3343 switch (MI->getOpcode()) { 3344 case SystemZ::Select32Mux: 3345 case SystemZ::Select32: 3346 case SystemZ::SelectF32: 3347 case SystemZ::Select64: 3348 case SystemZ::SelectF64: 3349 case SystemZ::SelectF128: 3350 return emitSelect(MI, MBB); 3351 3352 case SystemZ::CondStore8Mux: 3353 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false); 3354 case SystemZ::CondStore8MuxInv: 3355 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true); 3356 case SystemZ::CondStore16Mux: 3357 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false); 3358 case SystemZ::CondStore16MuxInv: 3359 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true); 3360 case SystemZ::CondStore8: 3361 return emitCondStore(MI, MBB, SystemZ::STC, 0, false); 3362 case SystemZ::CondStore8Inv: 3363 return emitCondStore(MI, MBB, SystemZ::STC, 0, true); 3364 case SystemZ::CondStore16: 3365 return emitCondStore(MI, MBB, SystemZ::STH, 0, false); 3366 case SystemZ::CondStore16Inv: 3367 return emitCondStore(MI, MBB, SystemZ::STH, 0, true); 3368 case SystemZ::CondStore32: 3369 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false); 3370 case SystemZ::CondStore32Inv: 3371 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true); 3372 case SystemZ::CondStore64: 3373 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false); 3374 case SystemZ::CondStore64Inv: 3375 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true); 3376 case SystemZ::CondStoreF32: 3377 return emitCondStore(MI, MBB, SystemZ::STE, 0, false); 3378 case SystemZ::CondStoreF32Inv: 3379 return emitCondStore(MI, MBB, SystemZ::STE, 0, true); 3380 case SystemZ::CondStoreF64: 3381 return emitCondStore(MI, MBB, SystemZ::STD, 0, false); 3382 case SystemZ::CondStoreF64Inv: 3383 return emitCondStore(MI, MBB, SystemZ::STD, 0, true); 3384 3385 case SystemZ::AEXT128_64: 3386 return emitExt128(MI, MBB, false, SystemZ::subreg_l64); 3387 case SystemZ::ZEXT128_32: 3388 return emitExt128(MI, MBB, true, SystemZ::subreg_l32); 3389 case SystemZ::ZEXT128_64: 3390 return emitExt128(MI, MBB, true, SystemZ::subreg_l64); 3391 3392 case SystemZ::ATOMIC_SWAPW: 3393 return emitAtomicLoadBinary(MI, MBB, 0, 0); 3394 case SystemZ::ATOMIC_SWAP_32: 3395 return emitAtomicLoadBinary(MI, MBB, 0, 32); 3396 case SystemZ::ATOMIC_SWAP_64: 3397 return emitAtomicLoadBinary(MI, MBB, 0, 64); 3398 3399 case SystemZ::ATOMIC_LOADW_AR: 3400 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); 3401 case SystemZ::ATOMIC_LOADW_AFI: 3402 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); 3403 case SystemZ::ATOMIC_LOAD_AR: 3404 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); 3405 case SystemZ::ATOMIC_LOAD_AHI: 3406 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); 3407 case SystemZ::ATOMIC_LOAD_AFI: 3408 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); 3409 case SystemZ::ATOMIC_LOAD_AGR: 3410 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); 3411 case SystemZ::ATOMIC_LOAD_AGHI: 3412 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); 3413 case SystemZ::ATOMIC_LOAD_AGFI: 3414 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); 3415 3416 case SystemZ::ATOMIC_LOADW_SR: 3417 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); 3418 case SystemZ::ATOMIC_LOAD_SR: 3419 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); 3420 case SystemZ::ATOMIC_LOAD_SGR: 3421 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); 3422 3423 case SystemZ::ATOMIC_LOADW_NR: 3424 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); 3425 case SystemZ::ATOMIC_LOADW_NILH: 3426 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); 3427 case SystemZ::ATOMIC_LOAD_NR: 3428 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); 3429 case SystemZ::ATOMIC_LOAD_NILL: 3430 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); 3431 case SystemZ::ATOMIC_LOAD_NILH: 3432 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); 3433 case SystemZ::ATOMIC_LOAD_NILF: 3434 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); 3435 case SystemZ::ATOMIC_LOAD_NGR: 3436 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); 3437 case SystemZ::ATOMIC_LOAD_NILL64: 3438 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); 3439 case SystemZ::ATOMIC_LOAD_NILH64: 3440 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); 3441 case SystemZ::ATOMIC_LOAD_NIHL64: 3442 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); 3443 case SystemZ::ATOMIC_LOAD_NIHH64: 3444 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); 3445 case SystemZ::ATOMIC_LOAD_NILF64: 3446 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); 3447 case SystemZ::ATOMIC_LOAD_NIHF64: 3448 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); 3449 3450 case SystemZ::ATOMIC_LOADW_OR: 3451 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); 3452 case SystemZ::ATOMIC_LOADW_OILH: 3453 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); 3454 case SystemZ::ATOMIC_LOAD_OR: 3455 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); 3456 case SystemZ::ATOMIC_LOAD_OILL: 3457 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); 3458 case SystemZ::ATOMIC_LOAD_OILH: 3459 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); 3460 case SystemZ::ATOMIC_LOAD_OILF: 3461 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); 3462 case SystemZ::ATOMIC_LOAD_OGR: 3463 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); 3464 case SystemZ::ATOMIC_LOAD_OILL64: 3465 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); 3466 case SystemZ::ATOMIC_LOAD_OILH64: 3467 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); 3468 case SystemZ::ATOMIC_LOAD_OIHL64: 3469 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); 3470 case SystemZ::ATOMIC_LOAD_OIHH64: 3471 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); 3472 case SystemZ::ATOMIC_LOAD_OILF64: 3473 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); 3474 case SystemZ::ATOMIC_LOAD_OIHF64: 3475 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); 3476 3477 case SystemZ::ATOMIC_LOADW_XR: 3478 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); 3479 case SystemZ::ATOMIC_LOADW_XILF: 3480 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); 3481 case SystemZ::ATOMIC_LOAD_XR: 3482 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); 3483 case SystemZ::ATOMIC_LOAD_XILF: 3484 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); 3485 case SystemZ::ATOMIC_LOAD_XGR: 3486 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); 3487 case SystemZ::ATOMIC_LOAD_XILF64: 3488 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); 3489 case SystemZ::ATOMIC_LOAD_XIHF64: 3490 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); 3491 3492 case SystemZ::ATOMIC_LOADW_NRi: 3493 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); 3494 case SystemZ::ATOMIC_LOADW_NILHi: 3495 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); 3496 case SystemZ::ATOMIC_LOAD_NRi: 3497 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); 3498 case SystemZ::ATOMIC_LOAD_NILLi: 3499 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); 3500 case SystemZ::ATOMIC_LOAD_NILHi: 3501 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); 3502 case SystemZ::ATOMIC_LOAD_NILFi: 3503 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); 3504 case SystemZ::ATOMIC_LOAD_NGRi: 3505 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); 3506 case SystemZ::ATOMIC_LOAD_NILL64i: 3507 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); 3508 case SystemZ::ATOMIC_LOAD_NILH64i: 3509 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); 3510 case SystemZ::ATOMIC_LOAD_NIHL64i: 3511 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); 3512 case SystemZ::ATOMIC_LOAD_NIHH64i: 3513 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); 3514 case SystemZ::ATOMIC_LOAD_NILF64i: 3515 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); 3516 case SystemZ::ATOMIC_LOAD_NIHF64i: 3517 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); 3518 3519 case SystemZ::ATOMIC_LOADW_MIN: 3520 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3521 SystemZ::CCMASK_CMP_LE, 0); 3522 case SystemZ::ATOMIC_LOAD_MIN_32: 3523 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3524 SystemZ::CCMASK_CMP_LE, 32); 3525 case SystemZ::ATOMIC_LOAD_MIN_64: 3526 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3527 SystemZ::CCMASK_CMP_LE, 64); 3528 3529 case SystemZ::ATOMIC_LOADW_MAX: 3530 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3531 SystemZ::CCMASK_CMP_GE, 0); 3532 case SystemZ::ATOMIC_LOAD_MAX_32: 3533 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, 3534 SystemZ::CCMASK_CMP_GE, 32); 3535 case SystemZ::ATOMIC_LOAD_MAX_64: 3536 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, 3537 SystemZ::CCMASK_CMP_GE, 64); 3538 3539 case SystemZ::ATOMIC_LOADW_UMIN: 3540 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3541 SystemZ::CCMASK_CMP_LE, 0); 3542 case SystemZ::ATOMIC_LOAD_UMIN_32: 3543 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3544 SystemZ::CCMASK_CMP_LE, 32); 3545 case SystemZ::ATOMIC_LOAD_UMIN_64: 3546 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3547 SystemZ::CCMASK_CMP_LE, 64); 3548 3549 case SystemZ::ATOMIC_LOADW_UMAX: 3550 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3551 SystemZ::CCMASK_CMP_GE, 0); 3552 case SystemZ::ATOMIC_LOAD_UMAX_32: 3553 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, 3554 SystemZ::CCMASK_CMP_GE, 32); 3555 case SystemZ::ATOMIC_LOAD_UMAX_64: 3556 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, 3557 SystemZ::CCMASK_CMP_GE, 64); 3558 3559 case SystemZ::ATOMIC_CMP_SWAPW: 3560 return emitAtomicCmpSwapW(MI, MBB); 3561 case SystemZ::MVCSequence: 3562 case SystemZ::MVCLoop: 3563 return emitMemMemWrapper(MI, MBB, SystemZ::MVC); 3564 case SystemZ::NCSequence: 3565 case SystemZ::NCLoop: 3566 return emitMemMemWrapper(MI, MBB, SystemZ::NC); 3567 case SystemZ::OCSequence: 3568 case SystemZ::OCLoop: 3569 return emitMemMemWrapper(MI, MBB, SystemZ::OC); 3570 case SystemZ::XCSequence: 3571 case SystemZ::XCLoop: 3572 return emitMemMemWrapper(MI, MBB, SystemZ::XC); 3573 case SystemZ::CLCSequence: 3574 case SystemZ::CLCLoop: 3575 return emitMemMemWrapper(MI, MBB, SystemZ::CLC); 3576 case SystemZ::CLSTLoop: 3577 return emitStringWrapper(MI, MBB, SystemZ::CLST); 3578 case SystemZ::MVSTLoop: 3579 return emitStringWrapper(MI, MBB, SystemZ::MVST); 3580 case SystemZ::SRSTLoop: 3581 return emitStringWrapper(MI, MBB, SystemZ::SRST); 3582 default: 3583 llvm_unreachable("Unexpected instr type to insert"); 3584 } 3585} 3586