1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file performs vector type splitting and scalarization for LegalizeTypes. 10// Scalarization is the act of changing a computation in an illegal one-element 11// vector type to be a computation in its scalar element type. For example, 12// implementing <1 x f32> arithmetic in a scalar f32 register. This is needed 13// as a base case when scalarizing vector arithmetic like <4 x f32>, which 14// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 15// types. 16// Splitting is the act of changing a computation in an invalid vector type to 17// be a computation in two vectors of half the size. For example, implementing 18// <128 x f32> operations in terms of two <64 x f32> operations. 19// 20//===----------------------------------------------------------------------===// 21 22#include "LegalizeTypes.h" 23#include "llvm/Analysis/MemoryLocation.h" 24#include "llvm/IR/DataLayout.h" 25#include "llvm/Support/ErrorHandling.h" 26#include "llvm/Support/TypeSize.h" 27#include "llvm/Support/raw_ostream.h" 28using namespace llvm; 29 30#define DEBUG_TYPE "legalize-types" 31 32//===----------------------------------------------------------------------===// 33// Result Vector Scalarization: <1 x ty> -> ty. 34//===----------------------------------------------------------------------===// 35 36void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { 37 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG); 38 dbgs() << "\n"); 39 SDValue R = SDValue(); 40 41 switch (N->getOpcode()) { 42 default: 43#ifndef NDEBUG 44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": "; 45 N->dump(&DAG); 46 dbgs() << "\n"; 47#endif 48 report_fatal_error("Do not know how to scalarize the result of this " 49 "operator!\n"); 50 51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break; 52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; 53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; 54 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; 55 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; 56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; 57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break; 58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break; 59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break; 60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break; 61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break; 62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break; 63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break; 64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break; 65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break; 66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break; 67 case ISD::ANY_EXTEND_VECTOR_INREG: 68 case ISD::SIGN_EXTEND_VECTOR_INREG: 69 case ISD::ZERO_EXTEND_VECTOR_INREG: 70 R = ScalarizeVecRes_VecInregOp(N); 71 break; 72 case ISD::ABS: 73 case ISD::ANY_EXTEND: 74 case ISD::BITREVERSE: 75 case ISD::BSWAP: 76 case ISD::CTLZ: 77 case ISD::CTLZ_ZERO_UNDEF: 78 case ISD::CTPOP: 79 case ISD::CTTZ: 80 case ISD::CTTZ_ZERO_UNDEF: 81 case ISD::FABS: 82 case ISD::FCEIL: 83 case ISD::FCOS: 84 case ISD::FEXP: 85 case ISD::FEXP2: 86 case ISD::FFLOOR: 87 case ISD::FLOG: 88 case ISD::FLOG10: 89 case ISD::FLOG2: 90 case ISD::FNEARBYINT: 91 case ISD::FNEG: 92 case ISD::FREEZE: 93 case ISD::FP_EXTEND: 94 case ISD::FP_TO_SINT: 95 case ISD::FP_TO_UINT: 96 case ISD::FRINT: 97 case ISD::FROUND: 98 case ISD::FROUNDEVEN: 99 case ISD::FSIN: 100 case ISD::FSQRT: 101 case ISD::FTRUNC: 102 case ISD::SIGN_EXTEND: 103 case ISD::SINT_TO_FP: 104 case ISD::TRUNCATE: 105 case ISD::UINT_TO_FP: 106 case ISD::ZERO_EXTEND: 107 case ISD::FCANONICALIZE: 108 R = ScalarizeVecRes_UnaryOp(N); 109 break; 110 111 case ISD::ADD: 112 case ISD::AND: 113 case ISD::FADD: 114 case ISD::FCOPYSIGN: 115 case ISD::FDIV: 116 case ISD::FMUL: 117 case ISD::FMINNUM: 118 case ISD::FMAXNUM: 119 case ISD::FMINNUM_IEEE: 120 case ISD::FMAXNUM_IEEE: 121 case ISD::FMINIMUM: 122 case ISD::FMAXIMUM: 123 case ISD::SMIN: 124 case ISD::SMAX: 125 case ISD::UMIN: 126 case ISD::UMAX: 127 128 case ISD::SADDSAT: 129 case ISD::UADDSAT: 130 case ISD::SSUBSAT: 131 case ISD::USUBSAT: 132 case ISD::SSHLSAT: 133 case ISD::USHLSAT: 134 135 case ISD::FPOW: 136 case ISD::FREM: 137 case ISD::FSUB: 138 case ISD::MUL: 139 case ISD::OR: 140 case ISD::SDIV: 141 case ISD::SREM: 142 case ISD::SUB: 143 case ISD::UDIV: 144 case ISD::UREM: 145 case ISD::XOR: 146 case ISD::SHL: 147 case ISD::SRA: 148 case ISD::SRL: 149 case ISD::ROTL: 150 case ISD::ROTR: 151 R = ScalarizeVecRes_BinOp(N); 152 break; 153 case ISD::FMA: 154 case ISD::FSHL: 155 case ISD::FSHR: 156 R = ScalarizeVecRes_TernaryOp(N); 157 break; 158 159#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 160 case ISD::STRICT_##DAGN: 161#include "llvm/IR/ConstrainedOps.def" 162 R = ScalarizeVecRes_StrictFPOp(N); 163 break; 164 165 case ISD::FP_TO_UINT_SAT: 166 case ISD::FP_TO_SINT_SAT: 167 R = ScalarizeVecRes_FP_TO_XINT_SAT(N); 168 break; 169 170 case ISD::UADDO: 171 case ISD::SADDO: 172 case ISD::USUBO: 173 case ISD::SSUBO: 174 case ISD::UMULO: 175 case ISD::SMULO: 176 R = ScalarizeVecRes_OverflowOp(N, ResNo); 177 break; 178 case ISD::SMULFIX: 179 case ISD::SMULFIXSAT: 180 case ISD::UMULFIX: 181 case ISD::UMULFIXSAT: 182 case ISD::SDIVFIX: 183 case ISD::SDIVFIXSAT: 184 case ISD::UDIVFIX: 185 case ISD::UDIVFIXSAT: 186 R = ScalarizeVecRes_FIX(N); 187 break; 188 } 189 190 // If R is null, the sub-method took care of registering the result. 191 if (R.getNode()) 192 SetScalarizedVector(SDValue(N, ResNo), R); 193} 194 195SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { 196 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 197 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 198 return DAG.getNode(N->getOpcode(), SDLoc(N), 199 LHS.getValueType(), LHS, RHS, N->getFlags()); 200} 201 202SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { 203 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 204 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 205 SDValue Op2 = GetScalarizedVector(N->getOperand(2)); 206 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 207 Op2, N->getFlags()); 208} 209 210SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { 211 SDValue Op0 = GetScalarizedVector(N->getOperand(0)); 212 SDValue Op1 = GetScalarizedVector(N->getOperand(1)); 213 SDValue Op2 = N->getOperand(2); 214 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1, 215 Op2, N->getFlags()); 216} 217 218SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { 219 EVT VT = N->getValueType(0).getVectorElementType(); 220 unsigned NumOpers = N->getNumOperands(); 221 SDValue Chain = N->getOperand(0); 222 EVT ValueVTs[] = {VT, MVT::Other}; 223 SDLoc dl(N); 224 225 SmallVector<SDValue, 4> Opers(NumOpers); 226 227 // The Chain is the first operand. 228 Opers[0] = Chain; 229 230 // Now process the remaining operands. 231 for (unsigned i = 1; i < NumOpers; ++i) { 232 SDValue Oper = N->getOperand(i); 233 234 if (Oper.getValueType().isVector()) 235 Oper = GetScalarizedVector(Oper); 236 237 Opers[i] = Oper; 238 } 239 240 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs), 241 Opers, N->getFlags()); 242 243 // Legalize the chain result - switch anything that used the old chain to 244 // use the new one. 245 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 246 return Result; 247} 248 249SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, 250 unsigned ResNo) { 251 SDLoc DL(N); 252 EVT ResVT = N->getValueType(0); 253 EVT OvVT = N->getValueType(1); 254 255 SDValue ScalarLHS, ScalarRHS; 256 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) { 257 ScalarLHS = GetScalarizedVector(N->getOperand(0)); 258 ScalarRHS = GetScalarizedVector(N->getOperand(1)); 259 } else { 260 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS; 261 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS); 262 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS); 263 ScalarLHS = ElemsLHS[0]; 264 ScalarRHS = ElemsRHS[0]; 265 } 266 267 SDVTList ScalarVTs = DAG.getVTList( 268 ResVT.getVectorElementType(), OvVT.getVectorElementType()); 269 SDNode *ScalarNode = DAG.getNode( 270 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); 271 ScalarNode->setFlags(N->getFlags()); 272 273 // Replace the other vector result not being explicitly scalarized here. 274 unsigned OtherNo = 1 - ResNo; 275 EVT OtherVT = N->getValueType(OtherNo); 276 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { 277 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); 278 } else { 279 SDValue OtherVal = DAG.getNode( 280 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo)); 281 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 282 } 283 284 return SDValue(ScalarNode, ResNo); 285} 286 287SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, 288 unsigned ResNo) { 289 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 290 return GetScalarizedVector(Op); 291} 292 293SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { 294 SDValue Op = N->getOperand(0); 295 if (Op.getValueType().isVector() 296 && Op.getValueType().getVectorNumElements() == 1 297 && !isSimpleLegalType(Op.getValueType())) 298 Op = GetScalarizedVector(Op); 299 EVT NewVT = N->getValueType(0).getVectorElementType(); 300 return DAG.getNode(ISD::BITCAST, SDLoc(N), 301 NewVT, Op); 302} 303 304SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { 305 EVT EltVT = N->getValueType(0).getVectorElementType(); 306 SDValue InOp = N->getOperand(0); 307 // The BUILD_VECTOR operands may be of wider element types and 308 // we may need to truncate them back to the requested return type. 309 if (EltVT.isInteger()) 310 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 311 return InOp; 312} 313 314SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 315 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 316 N->getValueType(0).getVectorElementType(), 317 N->getOperand(0), N->getOperand(1)); 318} 319 320SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { 321 SDLoc DL(N); 322 SDValue Op = N->getOperand(0); 323 EVT OpVT = Op.getValueType(); 324 // The result needs scalarizing, but it's not a given that the source does. 325 // See similar logic in ScalarizeVecRes_UnaryOp. 326 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 327 Op = GetScalarizedVector(Op); 328 } else { 329 EVT VT = OpVT.getVectorElementType(); 330 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 331 DAG.getVectorIdxConstant(0, DL)); 332 } 333 return DAG.getNode(ISD::FP_ROUND, DL, 334 N->getValueType(0).getVectorElementType(), Op, 335 N->getOperand(1)); 336} 337 338SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { 339 SDValue Op = GetScalarizedVector(N->getOperand(0)); 340 return DAG.getNode(ISD::FPOWI, SDLoc(N), 341 Op.getValueType(), Op, N->getOperand(1)); 342} 343 344SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { 345 // The value to insert may have a wider type than the vector element type, 346 // so be sure to truncate it to the element type if necessary. 347 SDValue Op = N->getOperand(1); 348 EVT EltVT = N->getValueType(0).getVectorElementType(); 349 if (Op.getValueType() != EltVT) 350 // FIXME: Can this happen for floating point types? 351 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op); 352 return Op; 353} 354 355SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { 356 assert(N->isUnindexed() && "Indexed vector load?"); 357 358 SDValue Result = DAG.getLoad( 359 ISD::UNINDEXED, N->getExtensionType(), 360 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(), 361 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()), 362 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(), 363 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo()); 364 365 // Legalize the chain result - switch anything that used the old chain to 366 // use the new one. 367 ReplaceValueWith(SDValue(N, 1), Result.getValue(1)); 368 return Result; 369} 370 371SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { 372 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp. 373 EVT DestVT = N->getValueType(0).getVectorElementType(); 374 SDValue Op = N->getOperand(0); 375 EVT OpVT = Op.getValueType(); 376 SDLoc DL(N); 377 // The result needs scalarizing, but it's not a given that the source does. 378 // This is a workaround for targets where it's impossible to scalarize the 379 // result of a conversion, because the source type is legal. 380 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32} 381 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is 382 // legal and was not scalarized. 383 // See the similar logic in ScalarizeVecRes_SETCC 384 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 385 Op = GetScalarizedVector(Op); 386 } else { 387 EVT VT = OpVT.getVectorElementType(); 388 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, 389 DAG.getVectorIdxConstant(0, DL)); 390 } 391 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags()); 392} 393 394SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { 395 EVT EltVT = N->getValueType(0).getVectorElementType(); 396 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType(); 397 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 398 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT, 399 LHS, DAG.getValueType(ExtVT)); 400} 401 402SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { 403 SDLoc DL(N); 404 SDValue Op = N->getOperand(0); 405 406 EVT OpVT = Op.getValueType(); 407 EVT OpEltVT = OpVT.getVectorElementType(); 408 EVT EltVT = N->getValueType(0).getVectorElementType(); 409 410 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 411 Op = GetScalarizedVector(Op); 412 } else { 413 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op, 414 DAG.getVectorIdxConstant(0, DL)); 415 } 416 417 switch (N->getOpcode()) { 418 case ISD::ANY_EXTEND_VECTOR_INREG: 419 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op); 420 case ISD::SIGN_EXTEND_VECTOR_INREG: 421 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op); 422 case ISD::ZERO_EXTEND_VECTOR_INREG: 423 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op); 424 } 425 426 llvm_unreachable("Illegal extend_vector_inreg opcode"); 427} 428 429SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { 430 // If the operand is wider than the vector element type then it is implicitly 431 // truncated. Make that explicit here. 432 EVT EltVT = N->getValueType(0).getVectorElementType(); 433 SDValue InOp = N->getOperand(0); 434 if (InOp.getValueType() != EltVT) 435 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp); 436 return InOp; 437} 438 439SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { 440 SDValue Cond = N->getOperand(0); 441 EVT OpVT = Cond.getValueType(); 442 SDLoc DL(N); 443 // The vselect result and true/value operands needs scalarizing, but it's 444 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal. 445 // See the similar logic in ScalarizeVecRes_SETCC 446 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 447 Cond = GetScalarizedVector(Cond); 448 } else { 449 EVT VT = OpVT.getVectorElementType(); 450 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond, 451 DAG.getVectorIdxConstant(0, DL)); 452 } 453 454 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 455 TargetLowering::BooleanContent ScalarBool = 456 TLI.getBooleanContents(false, false); 457 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false); 458 459 // If integer and float booleans have different contents then we can't 460 // reliably optimize in all cases. There is a full explanation for this in 461 // DAGCombiner::visitSELECT() where the same issue affects folding 462 // (select C, 0, 1) to (xor C, 1). 463 if (TLI.getBooleanContents(false, false) != 464 TLI.getBooleanContents(false, true)) { 465 // At least try the common case where the boolean is generated by a 466 // comparison. 467 if (Cond->getOpcode() == ISD::SETCC) { 468 EVT OpVT = Cond->getOperand(0).getValueType(); 469 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType()); 470 VecBool = TLI.getBooleanContents(OpVT); 471 } else 472 ScalarBool = TargetLowering::UndefinedBooleanContent; 473 } 474 475 EVT CondVT = Cond.getValueType(); 476 if (ScalarBool != VecBool) { 477 switch (ScalarBool) { 478 case TargetLowering::UndefinedBooleanContent: 479 break; 480 case TargetLowering::ZeroOrOneBooleanContent: 481 assert(VecBool == TargetLowering::UndefinedBooleanContent || 482 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent); 483 // Vector read from all ones, scalar expects a single 1 so mask. 484 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT, 485 Cond, DAG.getConstant(1, SDLoc(N), CondVT)); 486 break; 487 case TargetLowering::ZeroOrNegativeOneBooleanContent: 488 assert(VecBool == TargetLowering::UndefinedBooleanContent || 489 VecBool == TargetLowering::ZeroOrOneBooleanContent); 490 // Vector reads from a one, scalar from all ones so sign extend. 491 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT, 492 Cond, DAG.getValueType(MVT::i1)); 493 break; 494 } 495 } 496 497 // Truncate the condition if needed 498 auto BoolVT = getSetCCResultType(CondVT); 499 if (BoolVT.bitsLT(CondVT)) 500 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond); 501 502 return DAG.getSelect(SDLoc(N), 503 LHS.getValueType(), Cond, LHS, 504 GetScalarizedVector(N->getOperand(2))); 505} 506 507SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { 508 SDValue LHS = GetScalarizedVector(N->getOperand(1)); 509 return DAG.getSelect(SDLoc(N), 510 LHS.getValueType(), N->getOperand(0), LHS, 511 GetScalarizedVector(N->getOperand(2))); 512} 513 514SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { 515 SDValue LHS = GetScalarizedVector(N->getOperand(2)); 516 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(), 517 N->getOperand(0), N->getOperand(1), 518 LHS, GetScalarizedVector(N->getOperand(3)), 519 N->getOperand(4)); 520} 521 522SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { 523 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 524} 525 526SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { 527 // Figure out if the scalar is the LHS or RHS and return it. 528 SDValue Arg = N->getOperand(2).getOperand(0); 529 if (Arg.isUndef()) 530 return DAG.getUNDEF(N->getValueType(0).getVectorElementType()); 531 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue(); 532 return GetScalarizedVector(N->getOperand(Op)); 533} 534 535SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { 536 SDValue Src = N->getOperand(0); 537 EVT SrcVT = Src.getValueType(); 538 SDLoc dl(N); 539 540 // Handle case where result is scalarized but operand is not 541 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector) 542 Src = GetScalarizedVector(Src); 543 else 544 Src = DAG.getNode( 545 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src, 546 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 547 548 EVT DstVT = N->getValueType(0).getVectorElementType(); 549 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1)); 550} 551 552SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { 553 assert(N->getValueType(0).isVector() && 554 N->getOperand(0).getValueType().isVector() && 555 "Operand types must be vectors"); 556 SDValue LHS = N->getOperand(0); 557 SDValue RHS = N->getOperand(1); 558 EVT OpVT = LHS.getValueType(); 559 EVT NVT = N->getValueType(0).getVectorElementType(); 560 SDLoc DL(N); 561 562 // The result needs scalarizing, but it's not a given that the source does. 563 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) { 564 LHS = GetScalarizedVector(LHS); 565 RHS = GetScalarizedVector(RHS); 566 } else { 567 EVT VT = OpVT.getVectorElementType(); 568 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS, 569 DAG.getVectorIdxConstant(0, DL)); 570 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS, 571 DAG.getVectorIdxConstant(0, DL)); 572 } 573 574 // Turn it into a scalar SETCC. 575 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 576 N->getOperand(2)); 577 // Vectors may have a different boolean contents to scalars. Promote the 578 // value appropriately. 579 ISD::NodeType ExtendCode = 580 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 581 return DAG.getNode(ExtendCode, DL, NVT, Res); 582} 583 584 585//===----------------------------------------------------------------------===// 586// Operand Vector Scalarization <1 x ty> -> ty. 587//===----------------------------------------------------------------------===// 588 589bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { 590 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG); 591 dbgs() << "\n"); 592 SDValue Res = SDValue(); 593 594 switch (N->getOpcode()) { 595 default: 596#ifndef NDEBUG 597 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": "; 598 N->dump(&DAG); 599 dbgs() << "\n"; 600#endif 601 report_fatal_error("Do not know how to scalarize this operator's " 602 "operand!\n"); 603 case ISD::BITCAST: 604 Res = ScalarizeVecOp_BITCAST(N); 605 break; 606 case ISD::ANY_EXTEND: 607 case ISD::ZERO_EXTEND: 608 case ISD::SIGN_EXTEND: 609 case ISD::TRUNCATE: 610 case ISD::FP_TO_SINT: 611 case ISD::FP_TO_UINT: 612 case ISD::SINT_TO_FP: 613 case ISD::UINT_TO_FP: 614 Res = ScalarizeVecOp_UnaryOp(N); 615 break; 616 case ISD::STRICT_SINT_TO_FP: 617 case ISD::STRICT_UINT_TO_FP: 618 case ISD::STRICT_FP_TO_SINT: 619 case ISD::STRICT_FP_TO_UINT: 620 Res = ScalarizeVecOp_UnaryOp_StrictFP(N); 621 break; 622 case ISD::CONCAT_VECTORS: 623 Res = ScalarizeVecOp_CONCAT_VECTORS(N); 624 break; 625 case ISD::EXTRACT_VECTOR_ELT: 626 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); 627 break; 628 case ISD::VSELECT: 629 Res = ScalarizeVecOp_VSELECT(N); 630 break; 631 case ISD::SETCC: 632 Res = ScalarizeVecOp_VSETCC(N); 633 break; 634 case ISD::STORE: 635 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); 636 break; 637 case ISD::STRICT_FP_ROUND: 638 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); 639 break; 640 case ISD::FP_ROUND: 641 Res = ScalarizeVecOp_FP_ROUND(N, OpNo); 642 break; 643 case ISD::STRICT_FP_EXTEND: 644 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N); 645 break; 646 case ISD::FP_EXTEND: 647 Res = ScalarizeVecOp_FP_EXTEND(N); 648 break; 649 case ISD::VECREDUCE_FADD: 650 case ISD::VECREDUCE_FMUL: 651 case ISD::VECREDUCE_ADD: 652 case ISD::VECREDUCE_MUL: 653 case ISD::VECREDUCE_AND: 654 case ISD::VECREDUCE_OR: 655 case ISD::VECREDUCE_XOR: 656 case ISD::VECREDUCE_SMAX: 657 case ISD::VECREDUCE_SMIN: 658 case ISD::VECREDUCE_UMAX: 659 case ISD::VECREDUCE_UMIN: 660 case ISD::VECREDUCE_FMAX: 661 case ISD::VECREDUCE_FMIN: 662 Res = ScalarizeVecOp_VECREDUCE(N); 663 break; 664 case ISD::VECREDUCE_SEQ_FADD: 665 case ISD::VECREDUCE_SEQ_FMUL: 666 Res = ScalarizeVecOp_VECREDUCE_SEQ(N); 667 break; 668 } 669 670 // If the result is null, the sub-method took care of registering results etc. 671 if (!Res.getNode()) return false; 672 673 // If the result is N, the sub-method updated N in place. Tell the legalizer 674 // core about this. 675 if (Res.getNode() == N) 676 return true; 677 678 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 679 "Invalid operand expansion"); 680 681 ReplaceValueWith(SDValue(N, 0), Res); 682 return false; 683} 684 685/// If the value to convert is a vector that needs to be scalarized, it must be 686/// <1 x ty>. Convert the element instead. 687SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { 688 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 689 return DAG.getNode(ISD::BITCAST, SDLoc(N), 690 N->getValueType(0), Elt); 691} 692 693/// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 694/// Do the operation on the element instead. 695SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { 696 assert(N->getValueType(0).getVectorNumElements() == 1 && 697 "Unexpected vector type!"); 698 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 699 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N), 700 N->getValueType(0).getScalarType(), Elt); 701 // Revectorize the result so the types line up with what the uses of this 702 // expression expect. 703 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op); 704} 705 706/// If the input is a vector that needs to be scalarized, it must be <1 x ty>. 707/// Do the strict FP operation on the element instead. 708SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { 709 assert(N->getValueType(0).getVectorNumElements() == 1 && 710 "Unexpected vector type!"); 711 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 712 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), 713 { N->getValueType(0).getScalarType(), MVT::Other }, 714 { N->getOperand(0), Elt }); 715 // Legalize the chain result - switch anything that used the old chain to 716 // use the new one. 717 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 718 // Revectorize the result so the types line up with what the uses of this 719 // expression expect. 720 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 721 722 // Do our own replacement and return SDValue() to tell the caller that we 723 // handled all replacements since caller can only handle a single result. 724 ReplaceValueWith(SDValue(N, 0), Res); 725 return SDValue(); 726} 727 728/// The vectors to concatenate have length one - use a BUILD_VECTOR instead. 729SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { 730 SmallVector<SDValue, 8> Ops(N->getNumOperands()); 731 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) 732 Ops[i] = GetScalarizedVector(N->getOperand(i)); 733 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops); 734} 735 736/// If the input is a vector that needs to be scalarized, it must be <1 x ty>, 737/// so just return the element, ignoring the index. 738SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 739 EVT VT = N->getValueType(0); 740 SDValue Res = GetScalarizedVector(N->getOperand(0)); 741 if (Res.getValueType() != VT) 742 Res = VT.isFloatingPoint() 743 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res) 744 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res); 745 return Res; 746} 747 748/// If the input condition is a vector that needs to be scalarized, it must be 749/// <1 x i1>, so just convert to a normal ISD::SELECT 750/// (still with vector output type since that was acceptable if we got here). 751SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { 752 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0)); 753 EVT VT = N->getValueType(0); 754 755 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1), 756 N->getOperand(2)); 757} 758 759/// If the operand is a vector that needs to be scalarized then the 760/// result must be v1i1, so just convert to a scalar SETCC and wrap 761/// with a scalar_to_vector since the res type is legal if we got here 762SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { 763 assert(N->getValueType(0).isVector() && 764 N->getOperand(0).getValueType().isVector() && 765 "Operand types must be vectors"); 766 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type"); 767 768 EVT VT = N->getValueType(0); 769 SDValue LHS = GetScalarizedVector(N->getOperand(0)); 770 SDValue RHS = GetScalarizedVector(N->getOperand(1)); 771 772 EVT OpVT = N->getOperand(0).getValueType(); 773 EVT NVT = VT.getVectorElementType(); 774 SDLoc DL(N); 775 // Turn it into a scalar SETCC. 776 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, 777 N->getOperand(2)); 778 779 // Vectors may have a different boolean contents to scalars. Promote the 780 // value appropriately. 781 ISD::NodeType ExtendCode = 782 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 783 784 Res = DAG.getNode(ExtendCode, DL, NVT, Res); 785 786 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res); 787} 788 789/// If the value to store is a vector that needs to be scalarized, it must be 790/// <1 x ty>. Just store the element. 791SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ 792 assert(N->isUnindexed() && "Indexed store of one-element vector?"); 793 assert(OpNo == 1 && "Do not know how to scalarize this operand!"); 794 SDLoc dl(N); 795 796 if (N->isTruncatingStore()) 797 return DAG.getTruncStore( 798 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 799 N->getBasePtr(), N->getPointerInfo(), 800 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(), 801 N->getMemOperand()->getFlags(), N->getAAInfo()); 802 803 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), 804 N->getBasePtr(), N->getPointerInfo(), 805 N->getOriginalAlign(), N->getMemOperand()->getFlags(), 806 N->getAAInfo()); 807} 808 809/// If the value to round is a vector that needs to be scalarized, it must be 810/// <1 x ty>. Convert the element instead. 811SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { 812 assert(OpNo == 0 && "Wrong operand for scalarization!"); 813 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 814 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N), 815 N->getValueType(0).getVectorElementType(), Elt, 816 N->getOperand(1)); 817 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 818} 819 820SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, 821 unsigned OpNo) { 822 assert(OpNo == 1 && "Wrong operand for scalarization!"); 823 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 824 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 825 { N->getValueType(0).getVectorElementType(), 826 MVT::Other }, 827 { N->getOperand(0), Elt, N->getOperand(2) }); 828 // Legalize the chain result - switch anything that used the old chain to 829 // use the new one. 830 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 831 832 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 833 834 // Do our own replacement and return SDValue() to tell the caller that we 835 // handled all replacements since caller can only handle a single result. 836 ReplaceValueWith(SDValue(N, 0), Res); 837 return SDValue(); 838} 839 840/// If the value to extend is a vector that needs to be scalarized, it must be 841/// <1 x ty>. Convert the element instead. 842SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) { 843 SDValue Elt = GetScalarizedVector(N->getOperand(0)); 844 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), 845 N->getValueType(0).getVectorElementType(), Elt); 846 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 847} 848 849/// If the value to extend is a vector that needs to be scalarized, it must be 850/// <1 x ty>. Convert the element instead. 851SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) { 852 SDValue Elt = GetScalarizedVector(N->getOperand(1)); 853 SDValue Res = 854 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 855 {N->getValueType(0).getVectorElementType(), MVT::Other}, 856 {N->getOperand(0), Elt}); 857 // Legalize the chain result - switch anything that used the old chain to 858 // use the new one. 859 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 860 861 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); 862 863 // Do our own replacement and return SDValue() to tell the caller that we 864 // handled all replacements since caller can only handle a single result. 865 ReplaceValueWith(SDValue(N, 0), Res); 866 return SDValue(); 867} 868 869SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { 870 SDValue Res = GetScalarizedVector(N->getOperand(0)); 871 // Result type may be wider than element type. 872 if (Res.getValueType() != N->getValueType(0)) 873 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res); 874 return Res; 875} 876 877SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) { 878 SDValue AccOp = N->getOperand(0); 879 SDValue VecOp = N->getOperand(1); 880 881 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 882 883 SDValue Op = GetScalarizedVector(VecOp); 884 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0), 885 AccOp, Op, N->getFlags()); 886} 887 888//===----------------------------------------------------------------------===// 889// Result Vector Splitting 890//===----------------------------------------------------------------------===// 891 892/// This method is called when the specified result of the specified node is 893/// found to need vector splitting. At this point, the node may also have 894/// invalid operands or may have other results that need legalization, we just 895/// know that (at least) one result needs vector splitting. 896void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { 897 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n"); 898 SDValue Lo, Hi; 899 900 // See if the target wants to custom expand this node. 901 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 902 return; 903 904 switch (N->getOpcode()) { 905 default: 906#ifndef NDEBUG 907 dbgs() << "SplitVectorResult #" << ResNo << ": "; 908 N->dump(&DAG); 909 dbgs() << "\n"; 910#endif 911 report_fatal_error("Do not know how to split the result of this " 912 "operator!\n"); 913 914 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 915 case ISD::VSELECT: 916 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; 917 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 918 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 919 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break; 920 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break; 921 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break; 922 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break; 923 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break; 924 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break; 925 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break; 926 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break; 927 case ISD::SPLAT_VECTOR: 928 case ISD::SCALAR_TO_VECTOR: 929 SplitVecRes_ScalarOp(N, Lo, Hi); 930 break; 931 case ISD::STEP_VECTOR: 932 SplitVecRes_STEP_VECTOR(N, Lo, Hi); 933 break; 934 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break; 935 case ISD::LOAD: 936 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); 937 break; 938 case ISD::MLOAD: 939 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi); 940 break; 941 case ISD::MGATHER: 942 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi); 943 break; 944 case ISD::SETCC: 945 SplitVecRes_SETCC(N, Lo, Hi); 946 break; 947 case ISD::VECTOR_REVERSE: 948 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi); 949 break; 950 case ISD::VECTOR_SHUFFLE: 951 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); 952 break; 953 case ISD::VECTOR_SPLICE: 954 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi); 955 break; 956 case ISD::VAARG: 957 SplitVecRes_VAARG(N, Lo, Hi); 958 break; 959 960 case ISD::ANY_EXTEND_VECTOR_INREG: 961 case ISD::SIGN_EXTEND_VECTOR_INREG: 962 case ISD::ZERO_EXTEND_VECTOR_INREG: 963 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 964 break; 965 966 case ISD::ABS: 967 case ISD::BITREVERSE: 968 case ISD::BSWAP: 969 case ISD::CTLZ: 970 case ISD::CTTZ: 971 case ISD::CTLZ_ZERO_UNDEF: 972 case ISD::CTTZ_ZERO_UNDEF: 973 case ISD::CTPOP: 974 case ISD::FABS: 975 case ISD::FCEIL: 976 case ISD::FCOS: 977 case ISD::FEXP: 978 case ISD::FEXP2: 979 case ISD::FFLOOR: 980 case ISD::FLOG: 981 case ISD::FLOG10: 982 case ISD::FLOG2: 983 case ISD::FNEARBYINT: 984 case ISD::FNEG: 985 case ISD::FREEZE: 986 case ISD::FP_EXTEND: 987 case ISD::FP_ROUND: 988 case ISD::FP_TO_SINT: 989 case ISD::FP_TO_UINT: 990 case ISD::FRINT: 991 case ISD::FROUND: 992 case ISD::FROUNDEVEN: 993 case ISD::FSIN: 994 case ISD::FSQRT: 995 case ISD::FTRUNC: 996 case ISD::SINT_TO_FP: 997 case ISD::TRUNCATE: 998 case ISD::UINT_TO_FP: 999 case ISD::FCANONICALIZE: 1000 SplitVecRes_UnaryOp(N, Lo, Hi); 1001 break; 1002 1003 case ISD::ANY_EXTEND: 1004 case ISD::SIGN_EXTEND: 1005 case ISD::ZERO_EXTEND: 1006 SplitVecRes_ExtendOp(N, Lo, Hi); 1007 break; 1008 1009 case ISD::ADD: 1010 case ISD::SUB: 1011 case ISD::MUL: 1012 case ISD::MULHS: 1013 case ISD::MULHU: 1014 case ISD::FADD: 1015 case ISD::FSUB: 1016 case ISD::FMUL: 1017 case ISD::FMINNUM: 1018 case ISD::FMAXNUM: 1019 case ISD::FMINIMUM: 1020 case ISD::FMAXIMUM: 1021 case ISD::SDIV: 1022 case ISD::UDIV: 1023 case ISD::FDIV: 1024 case ISD::FPOW: 1025 case ISD::AND: 1026 case ISD::OR: 1027 case ISD::XOR: 1028 case ISD::SHL: 1029 case ISD::SRA: 1030 case ISD::SRL: 1031 case ISD::UREM: 1032 case ISD::SREM: 1033 case ISD::FREM: 1034 case ISD::SMIN: 1035 case ISD::SMAX: 1036 case ISD::UMIN: 1037 case ISD::UMAX: 1038 case ISD::SADDSAT: 1039 case ISD::UADDSAT: 1040 case ISD::SSUBSAT: 1041 case ISD::USUBSAT: 1042 case ISD::SSHLSAT: 1043 case ISD::USHLSAT: 1044 case ISD::ROTL: 1045 case ISD::ROTR: 1046 SplitVecRes_BinOp(N, Lo, Hi); 1047 break; 1048 case ISD::FMA: 1049 case ISD::FSHL: 1050 case ISD::FSHR: 1051 SplitVecRes_TernaryOp(N, Lo, Hi); 1052 break; 1053 1054#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 1055 case ISD::STRICT_##DAGN: 1056#include "llvm/IR/ConstrainedOps.def" 1057 SplitVecRes_StrictFPOp(N, Lo, Hi); 1058 break; 1059 1060 case ISD::FP_TO_UINT_SAT: 1061 case ISD::FP_TO_SINT_SAT: 1062 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi); 1063 break; 1064 1065 case ISD::UADDO: 1066 case ISD::SADDO: 1067 case ISD::USUBO: 1068 case ISD::SSUBO: 1069 case ISD::UMULO: 1070 case ISD::SMULO: 1071 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); 1072 break; 1073 case ISD::SMULFIX: 1074 case ISD::SMULFIXSAT: 1075 case ISD::UMULFIX: 1076 case ISD::UMULFIXSAT: 1077 case ISD::SDIVFIX: 1078 case ISD::SDIVFIXSAT: 1079 case ISD::UDIVFIX: 1080 case ISD::UDIVFIXSAT: 1081 SplitVecRes_FIX(N, Lo, Hi); 1082 break; 1083 } 1084 1085 // If Lo/Hi is null, the sub-method took care of registering results etc. 1086 if (Lo.getNode()) 1087 SetSplitVector(SDValue(N, ResNo), Lo, Hi); 1088} 1089 1090void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT, 1091 MachinePointerInfo &MPI, SDValue &Ptr, 1092 uint64_t *ScaledOffset) { 1093 SDLoc DL(N); 1094 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8; 1095 1096 if (MemVT.isScalableVector()) { 1097 SDNodeFlags Flags; 1098 SDValue BytesIncrement = DAG.getVScale( 1099 DL, Ptr.getValueType(), 1100 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize)); 1101 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 1102 Flags.setNoUnsignedWrap(true); 1103 if (ScaledOffset) 1104 *ScaledOffset += IncrementSize; 1105 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement, 1106 Flags); 1107 } else { 1108 MPI = N->getPointerInfo().getWithOffset(IncrementSize); 1109 // Increment the pointer to the other half. 1110 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize)); 1111 } 1112} 1113 1114void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, 1115 SDValue &Hi) { 1116 SDValue LHSLo, LHSHi; 1117 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1118 SDValue RHSLo, RHSHi; 1119 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1120 SDLoc dl(N); 1121 1122 const SDNodeFlags Flags = N->getFlags(); 1123 unsigned Opcode = N->getOpcode(); 1124 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags); 1125 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags); 1126} 1127 1128void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, 1129 SDValue &Hi) { 1130 SDValue Op0Lo, Op0Hi; 1131 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); 1132 SDValue Op1Lo, Op1Hi; 1133 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); 1134 SDValue Op2Lo, Op2Hi; 1135 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi); 1136 SDLoc dl(N); 1137 1138 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo, 1139 Op2Lo, N->getFlags()); 1140 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi, 1141 Op2Hi, N->getFlags()); 1142} 1143 1144void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { 1145 SDValue LHSLo, LHSHi; 1146 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1147 SDValue RHSLo, RHSHi; 1148 GetSplitVector(N->getOperand(1), RHSLo, RHSHi); 1149 SDLoc dl(N); 1150 SDValue Op2 = N->getOperand(2); 1151 1152 unsigned Opcode = N->getOpcode(); 1153 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2, 1154 N->getFlags()); 1155 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2, 1156 N->getFlags()); 1157} 1158 1159void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, 1160 SDValue &Hi) { 1161 // We know the result is a vector. The input may be either a vector or a 1162 // scalar value. 1163 EVT LoVT, HiVT; 1164 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1165 SDLoc dl(N); 1166 1167 SDValue InOp = N->getOperand(0); 1168 EVT InVT = InOp.getValueType(); 1169 1170 // Handle some special cases efficiently. 1171 switch (getTypeAction(InVT)) { 1172 case TargetLowering::TypeLegal: 1173 case TargetLowering::TypePromoteInteger: 1174 case TargetLowering::TypePromoteFloat: 1175 case TargetLowering::TypeSoftPromoteHalf: 1176 case TargetLowering::TypeSoftenFloat: 1177 case TargetLowering::TypeScalarizeVector: 1178 case TargetLowering::TypeWidenVector: 1179 break; 1180 case TargetLowering::TypeExpandInteger: 1181 case TargetLowering::TypeExpandFloat: 1182 // A scalar to vector conversion, where the scalar needs expansion. 1183 // If the vector is being split in two then we can just convert the 1184 // expanded pieces. 1185 if (LoVT == HiVT) { 1186 GetExpandedOp(InOp, Lo, Hi); 1187 if (DAG.getDataLayout().isBigEndian()) 1188 std::swap(Lo, Hi); 1189 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1190 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1191 return; 1192 } 1193 break; 1194 case TargetLowering::TypeSplitVector: 1195 // If the input is a vector that needs to be split, convert each split 1196 // piece of the input now. 1197 GetSplitVector(InOp, Lo, Hi); 1198 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1199 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1200 return; 1201 case TargetLowering::TypeScalarizeScalableVector: 1202 report_fatal_error("Scalarization of scalable vectors is not supported."); 1203 } 1204 1205 // In the general case, convert the input to an integer and split it by hand. 1206 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits()); 1207 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits()); 1208 if (DAG.getDataLayout().isBigEndian()) 1209 std::swap(LoIntVT, HiIntVT); 1210 1211 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi); 1212 1213 if (DAG.getDataLayout().isBigEndian()) 1214 std::swap(Lo, Hi); 1215 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo); 1216 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi); 1217} 1218 1219void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, 1220 SDValue &Hi) { 1221 EVT LoVT, HiVT; 1222 SDLoc dl(N); 1223 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1224 unsigned LoNumElts = LoVT.getVectorNumElements(); 1225 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts); 1226 Lo = DAG.getBuildVector(LoVT, dl, LoOps); 1227 1228 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end()); 1229 Hi = DAG.getBuildVector(HiVT, dl, HiOps); 1230} 1231 1232void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, 1233 SDValue &Hi) { 1234 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS"); 1235 SDLoc dl(N); 1236 unsigned NumSubvectors = N->getNumOperands() / 2; 1237 if (NumSubvectors == 1) { 1238 Lo = N->getOperand(0); 1239 Hi = N->getOperand(1); 1240 return; 1241 } 1242 1243 EVT LoVT, HiVT; 1244 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1245 1246 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors); 1247 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps); 1248 1249 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end()); 1250 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps); 1251} 1252 1253void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, 1254 SDValue &Hi) { 1255 SDValue Vec = N->getOperand(0); 1256 SDValue Idx = N->getOperand(1); 1257 SDLoc dl(N); 1258 1259 EVT LoVT, HiVT; 1260 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1261 1262 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); 1263 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1264 Hi = DAG.getNode( 1265 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, 1266 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl)); 1267} 1268 1269void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, 1270 SDValue &Hi) { 1271 SDValue Vec = N->getOperand(0); 1272 SDValue SubVec = N->getOperand(1); 1273 SDValue Idx = N->getOperand(2); 1274 SDLoc dl(N); 1275 GetSplitVector(Vec, Lo, Hi); 1276 1277 EVT VecVT = Vec.getValueType(); 1278 EVT LoVT = Lo.getValueType(); 1279 EVT SubVecVT = SubVec.getValueType(); 1280 unsigned VecElems = VecVT.getVectorMinNumElements(); 1281 unsigned SubElems = SubVecVT.getVectorMinNumElements(); 1282 unsigned LoElems = LoVT.getVectorMinNumElements(); 1283 1284 // If we know the index is in the first half, and we know the subvector 1285 // doesn't cross the boundary between the halves, we can avoid spilling the 1286 // vector, and insert into the lower half of the split vector directly. 1287 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 1288 if (IdxVal + SubElems <= LoElems) { 1289 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); 1290 return; 1291 } 1292 // Similarly if the subvector is fully in the high half, but mind that we 1293 // can't tell whether a fixed-length subvector is fully within the high half 1294 // of a scalable vector. 1295 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() && 1296 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) { 1297 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec, 1298 DAG.getVectorIdxConstant(IdxVal - LoElems, dl)); 1299 return; 1300 } 1301 1302 // Spill the vector to the stack. 1303 // In cases where the vector is illegal it will be broken down into parts 1304 // and stored in parts - we should use the alignment for the smallest part. 1305 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1306 SDValue StackPtr = 1307 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1308 auto &MF = DAG.getMachineFunction(); 1309 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1310 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1311 1312 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1313 SmallestAlign); 1314 1315 // Store the new subvector into the specified index. 1316 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1317 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, 1318 MachinePointerInfo::getUnknownStack(MF)); 1319 1320 // Load the Lo part from the stack slot. 1321 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo, 1322 SmallestAlign); 1323 1324 // Increment the pointer to the other part. 1325 auto *Load = cast<LoadSDNode>(Lo); 1326 MachinePointerInfo MPI = Load->getPointerInfo(); 1327 IncrementPointer(Load, LoVT, MPI, StackPtr); 1328 1329 // Load the Hi part from the stack slot. 1330 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign); 1331} 1332 1333void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, 1334 SDValue &Hi) { 1335 SDLoc dl(N); 1336 GetSplitVector(N->getOperand(0), Lo, Hi); 1337 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1)); 1338 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1)); 1339} 1340 1341void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, 1342 SDValue &Hi) { 1343 SDValue LHSLo, LHSHi; 1344 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1345 SDLoc DL(N); 1346 1347 SDValue RHSLo, RHSHi; 1348 SDValue RHS = N->getOperand(1); 1349 EVT RHSVT = RHS.getValueType(); 1350 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector) 1351 GetSplitVector(RHS, RHSLo, RHSHi); 1352 else 1353 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS)); 1354 1355 1356 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo); 1357 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi); 1358} 1359 1360void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, 1361 SDValue &Hi) { 1362 SDValue LHSLo, LHSHi; 1363 GetSplitVector(N->getOperand(0), LHSLo, LHSHi); 1364 SDLoc dl(N); 1365 1366 EVT LoVT, HiVT; 1367 std::tie(LoVT, HiVT) = 1368 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT()); 1369 1370 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, 1371 DAG.getValueType(LoVT)); 1372 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, 1373 DAG.getValueType(HiVT)); 1374} 1375 1376void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, 1377 SDValue &Hi) { 1378 unsigned Opcode = N->getOpcode(); 1379 SDValue N0 = N->getOperand(0); 1380 1381 SDLoc dl(N); 1382 SDValue InLo, InHi; 1383 1384 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector) 1385 GetSplitVector(N0, InLo, InHi); 1386 else 1387 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0); 1388 1389 EVT InLoVT = InLo.getValueType(); 1390 unsigned InNumElements = InLoVT.getVectorNumElements(); 1391 1392 EVT OutLoVT, OutHiVT; 1393 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1394 unsigned OutNumElements = OutLoVT.getVectorNumElements(); 1395 assert((2 * OutNumElements) <= InNumElements && 1396 "Illegal extend vector in reg split"); 1397 1398 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the 1399 // input vector (i.e. we only use InLo): 1400 // OutLo will extend the first OutNumElements from InLo. 1401 // OutHi will extend the next OutNumElements from InLo. 1402 1403 // Shuffle the elements from InLo for OutHi into the bottom elements to 1404 // create a 'fake' InHi. 1405 SmallVector<int, 8> SplitHi(InNumElements, -1); 1406 for (unsigned i = 0; i != OutNumElements; ++i) 1407 SplitHi[i] = i + OutNumElements; 1408 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi); 1409 1410 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo); 1411 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi); 1412} 1413 1414void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, 1415 SDValue &Hi) { 1416 unsigned NumOps = N->getNumOperands(); 1417 SDValue Chain = N->getOperand(0); 1418 EVT LoVT, HiVT; 1419 SDLoc dl(N); 1420 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1421 1422 SmallVector<SDValue, 4> OpsLo(NumOps); 1423 SmallVector<SDValue, 4> OpsHi(NumOps); 1424 1425 // The Chain is the first operand. 1426 OpsLo[0] = Chain; 1427 OpsHi[0] = Chain; 1428 1429 // Now process the remaining operands. 1430 for (unsigned i = 1; i < NumOps; ++i) { 1431 SDValue Op = N->getOperand(i); 1432 SDValue OpLo = Op; 1433 SDValue OpHi = Op; 1434 1435 EVT InVT = Op.getValueType(); 1436 if (InVT.isVector()) { 1437 // If the input also splits, handle it directly for a 1438 // compile time speedup. Otherwise split it by hand. 1439 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1440 GetSplitVector(Op, OpLo, OpHi); 1441 else 1442 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i); 1443 } 1444 1445 OpsLo[i] = OpLo; 1446 OpsHi[i] = OpHi; 1447 } 1448 1449 EVT LoValueVTs[] = {LoVT, MVT::Other}; 1450 EVT HiValueVTs[] = {HiVT, MVT::Other}; 1451 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo, 1452 N->getFlags()); 1453 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi, 1454 N->getFlags()); 1455 1456 // Build a factor node to remember that this Op is independent of the 1457 // other one. 1458 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 1459 Lo.getValue(1), Hi.getValue(1)); 1460 1461 // Legalize the chain result - switch anything that used the old chain to 1462 // use the new one. 1463 ReplaceValueWith(SDValue(N, 1), Chain); 1464} 1465 1466SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { 1467 SDValue Chain = N->getOperand(0); 1468 EVT VT = N->getValueType(0); 1469 unsigned NE = VT.getVectorNumElements(); 1470 EVT EltVT = VT.getVectorElementType(); 1471 SDLoc dl(N); 1472 1473 SmallVector<SDValue, 8> Scalars; 1474 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 1475 1476 // If ResNE is 0, fully unroll the vector op. 1477 if (ResNE == 0) 1478 ResNE = NE; 1479 else if (NE > ResNE) 1480 NE = ResNE; 1481 1482 //The results of each unrolled operation, including the chain. 1483 EVT ChainVTs[] = {EltVT, MVT::Other}; 1484 SmallVector<SDValue, 8> Chains; 1485 1486 unsigned i; 1487 for (i = 0; i != NE; ++i) { 1488 Operands[0] = Chain; 1489 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { 1490 SDValue Operand = N->getOperand(j); 1491 EVT OperandVT = Operand.getValueType(); 1492 if (OperandVT.isVector()) { 1493 EVT OperandEltVT = OperandVT.getVectorElementType(); 1494 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, 1495 Operand, DAG.getVectorIdxConstant(i, dl)); 1496 } else { 1497 Operands[j] = Operand; 1498 } 1499 } 1500 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); 1501 Scalar.getNode()->setFlags(N->getFlags()); 1502 1503 //Add in the scalar as well as its chain value to the 1504 //result vectors. 1505 Scalars.push_back(Scalar); 1506 Chains.push_back(Scalar.getValue(1)); 1507 } 1508 1509 for (; i < ResNE; ++i) 1510 Scalars.push_back(DAG.getUNDEF(EltVT)); 1511 1512 // Build a new factor node to connect the chain back together. 1513 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 1514 ReplaceValueWith(SDValue(N, 1), Chain); 1515 1516 // Create a new BUILD_VECTOR node 1517 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE); 1518 return DAG.getBuildVector(VecVT, dl, Scalars); 1519} 1520 1521void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, 1522 SDValue &Lo, SDValue &Hi) { 1523 SDLoc dl(N); 1524 EVT ResVT = N->getValueType(0); 1525 EVT OvVT = N->getValueType(1); 1526 EVT LoResVT, HiResVT, LoOvVT, HiOvVT; 1527 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT); 1528 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT); 1529 1530 SDValue LoLHS, HiLHS, LoRHS, HiRHS; 1531 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) { 1532 GetSplitVector(N->getOperand(0), LoLHS, HiLHS); 1533 GetSplitVector(N->getOperand(1), LoRHS, HiRHS); 1534 } else { 1535 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0); 1536 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1); 1537 } 1538 1539 unsigned Opcode = N->getOpcode(); 1540 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); 1541 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); 1542 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); 1543 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); 1544 LoNode->setFlags(N->getFlags()); 1545 HiNode->setFlags(N->getFlags()); 1546 1547 Lo = SDValue(LoNode, ResNo); 1548 Hi = SDValue(HiNode, ResNo); 1549 1550 // Replace the other vector result not being explicitly split here. 1551 unsigned OtherNo = 1 - ResNo; 1552 EVT OtherVT = N->getValueType(OtherNo); 1553 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { 1554 SetSplitVector(SDValue(N, OtherNo), 1555 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1556 } else { 1557 SDValue OtherVal = DAG.getNode( 1558 ISD::CONCAT_VECTORS, dl, OtherVT, 1559 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); 1560 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 1561 } 1562} 1563 1564void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, 1565 SDValue &Hi) { 1566 SDValue Vec = N->getOperand(0); 1567 SDValue Elt = N->getOperand(1); 1568 SDValue Idx = N->getOperand(2); 1569 SDLoc dl(N); 1570 GetSplitVector(Vec, Lo, Hi); 1571 1572 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) { 1573 unsigned IdxVal = CIdx->getZExtValue(); 1574 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements(); 1575 if (IdxVal < LoNumElts) { 1576 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, 1577 Lo.getValueType(), Lo, Elt, Idx); 1578 return; 1579 } else if (!Vec.getValueType().isScalableVector()) { 1580 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt, 1581 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl)); 1582 return; 1583 } 1584 } 1585 1586 // See if the target wants to custom expand this node. 1587 if (CustomLowerNode(N, N->getValueType(0), true)) 1588 return; 1589 1590 // Make the vector elements byte-addressable if they aren't already. 1591 EVT VecVT = Vec.getValueType(); 1592 EVT EltVT = VecVT.getVectorElementType(); 1593 if (VecVT.getScalarSizeInBits() < 8) { 1594 EltVT = MVT::i8; 1595 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 1596 VecVT.getVectorElementCount()); 1597 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 1598 // Extend the element type to match if needed. 1599 if (EltVT.bitsGT(Elt.getValueType())) 1600 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt); 1601 } 1602 1603 // Spill the vector to the stack. 1604 // In cases where the vector is illegal it will be broken down into parts 1605 // and stored in parts - we should use the alignment for the smallest part. 1606 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 1607 SDValue StackPtr = 1608 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 1609 auto &MF = DAG.getMachineFunction(); 1610 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 1611 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 1612 1613 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 1614 SmallestAlign); 1615 1616 // Store the new element. This may be larger than the vector element type, 1617 // so use a truncating store. 1618 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 1619 Store = DAG.getTruncStore( 1620 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT, 1621 commonAlignment(SmallestAlign, 1622 EltVT.getFixedSizeInBits() / 8)); 1623 1624 EVT LoVT, HiVT; 1625 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT); 1626 1627 // Load the Lo part from the stack slot. 1628 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign); 1629 1630 // Increment the pointer to the other part. 1631 auto Load = cast<LoadSDNode>(Lo); 1632 MachinePointerInfo MPI = Load->getPointerInfo(); 1633 IncrementPointer(Load, LoVT, MPI, StackPtr); 1634 1635 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign); 1636 1637 // If we adjusted the original type, we need to truncate the results. 1638 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1639 if (LoVT != Lo.getValueType()) 1640 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo); 1641 if (HiVT != Hi.getValueType()) 1642 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi); 1643} 1644 1645void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, 1646 SDValue &Hi) { 1647 EVT LoVT, HiVT; 1648 SDLoc dl(N); 1649 assert(N->getValueType(0).isScalableVector() && 1650 "Only scalable vectors are supported for STEP_VECTOR"); 1651 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1652 SDValue Step = N->getOperand(0); 1653 1654 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step); 1655 1656 // Hi = Lo + (EltCnt * Step) 1657 EVT EltVT = Step.getValueType(); 1658 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue(); 1659 SDValue StartOfHi = 1660 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements()); 1661 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType()); 1662 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi); 1663 1664 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step); 1665 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi); 1666} 1667 1668void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, 1669 SDValue &Hi) { 1670 EVT LoVT, HiVT; 1671 SDLoc dl(N); 1672 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1673 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0)); 1674 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) { 1675 Hi = DAG.getUNDEF(HiVT); 1676 } else { 1677 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode"); 1678 Hi = Lo; 1679 } 1680} 1681 1682void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, 1683 SDValue &Hi) { 1684 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!"); 1685 EVT LoVT, HiVT; 1686 SDLoc dl(LD); 1687 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0)); 1688 1689 ISD::LoadExtType ExtType = LD->getExtensionType(); 1690 SDValue Ch = LD->getChain(); 1691 SDValue Ptr = LD->getBasePtr(); 1692 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 1693 EVT MemoryVT = LD->getMemoryVT(); 1694 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 1695 AAMDNodes AAInfo = LD->getAAInfo(); 1696 1697 EVT LoMemVT, HiMemVT; 1698 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1699 1700 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) { 1701 SDValue Value, NewChain; 1702 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 1703 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl); 1704 ReplaceValueWith(SDValue(LD, 1), NewChain); 1705 return; 1706 } 1707 1708 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset, 1709 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(), 1710 MMOFlags, AAInfo); 1711 1712 MachinePointerInfo MPI; 1713 IncrementPointer(LD, LoMemVT, MPI, Ptr); 1714 1715 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI, 1716 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 1717 1718 // Build a factor node to remember that this load is independent of the 1719 // other one. 1720 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1721 Hi.getValue(1)); 1722 1723 // Legalize the chain result - switch anything that used the old chain to 1724 // use the new one. 1725 ReplaceValueWith(SDValue(LD, 1), Ch); 1726} 1727 1728void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, 1729 SDValue &Lo, SDValue &Hi) { 1730 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!"); 1731 EVT LoVT, HiVT; 1732 SDLoc dl(MLD); 1733 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0)); 1734 1735 SDValue Ch = MLD->getChain(); 1736 SDValue Ptr = MLD->getBasePtr(); 1737 SDValue Offset = MLD->getOffset(); 1738 assert(Offset.isUndef() && "Unexpected indexed masked load offset"); 1739 SDValue Mask = MLD->getMask(); 1740 SDValue PassThru = MLD->getPassThru(); 1741 Align Alignment = MLD->getOriginalAlign(); 1742 ISD::LoadExtType ExtType = MLD->getExtensionType(); 1743 1744 // Split Mask operand 1745 SDValue MaskLo, MaskHi; 1746 if (Mask.getOpcode() == ISD::SETCC) { 1747 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1748 } else { 1749 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1750 GetSplitVector(Mask, MaskLo, MaskHi); 1751 else 1752 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1753 } 1754 1755 EVT MemoryVT = MLD->getMemoryVT(); 1756 EVT LoMemVT, HiMemVT; 1757 bool HiIsEmpty = false; 1758 std::tie(LoMemVT, HiMemVT) = 1759 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty); 1760 1761 SDValue PassThruLo, PassThruHi; 1762 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 1763 GetSplitVector(PassThru, PassThruLo, PassThruHi); 1764 else 1765 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 1766 1767 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize()); 1768 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 1769 MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment, 1770 MLD->getAAInfo(), MLD->getRanges()); 1771 1772 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT, 1773 MMO, MLD->getAddressingMode(), ExtType, 1774 MLD->isExpandingLoad()); 1775 1776 if (HiIsEmpty) { 1777 // The hi masked load has zero storage size. We therefore simply set it to 1778 // the low masked load and rely on subsequent removal from the chain. 1779 Hi = Lo; 1780 } else { 1781 // Generate hi masked load. 1782 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG, 1783 MLD->isExpandingLoad()); 1784 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize()); 1785 1786 MachinePointerInfo MPI; 1787 if (LoMemVT.isScalableVector()) 1788 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace()); 1789 else 1790 MPI = MLD->getPointerInfo().getWithOffset( 1791 LoMemVT.getStoreSize().getFixedSize()); 1792 1793 MMO = DAG.getMachineFunction().getMachineMemOperand( 1794 MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(), 1795 MLD->getRanges()); 1796 1797 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi, 1798 HiMemVT, MMO, MLD->getAddressingMode(), ExtType, 1799 MLD->isExpandingLoad()); 1800 } 1801 1802 // Build a factor node to remember that this load is independent of the 1803 // other one. 1804 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1805 Hi.getValue(1)); 1806 1807 // Legalize the chain result - switch anything that used the old chain to 1808 // use the new one. 1809 ReplaceValueWith(SDValue(MLD, 1), Ch); 1810 1811} 1812 1813void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, 1814 SDValue &Lo, SDValue &Hi) { 1815 EVT LoVT, HiVT; 1816 SDLoc dl(MGT); 1817 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 1818 1819 SDValue Ch = MGT->getChain(); 1820 SDValue Ptr = MGT->getBasePtr(); 1821 SDValue Mask = MGT->getMask(); 1822 SDValue PassThru = MGT->getPassThru(); 1823 SDValue Index = MGT->getIndex(); 1824 SDValue Scale = MGT->getScale(); 1825 EVT MemoryVT = MGT->getMemoryVT(); 1826 Align Alignment = MGT->getOriginalAlign(); 1827 ISD::LoadExtType ExtType = MGT->getExtensionType(); 1828 1829 // Split Mask operand 1830 SDValue MaskLo, MaskHi; 1831 if (Mask.getOpcode() == ISD::SETCC) { 1832 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 1833 } else { 1834 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 1835 GetSplitVector(Mask, MaskLo, MaskHi); 1836 else 1837 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 1838 } 1839 1840 EVT LoMemVT, HiMemVT; 1841 // Split MemoryVT 1842 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 1843 1844 SDValue PassThruLo, PassThruHi; 1845 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 1846 GetSplitVector(PassThru, PassThruLo, PassThruHi); 1847 else 1848 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 1849 1850 SDValue IndexHi, IndexLo; 1851 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 1852 GetSplitVector(Index, IndexLo, IndexHi); 1853 else 1854 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 1855 1856 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 1857 MGT->getPointerInfo(), MachineMemOperand::MOLoad, 1858 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(), 1859 MGT->getRanges()); 1860 1861 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale}; 1862 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo, 1863 MMO, MGT->getIndexType(), ExtType); 1864 1865 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale}; 1866 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi, 1867 MMO, MGT->getIndexType(), ExtType); 1868 1869 // Build a factor node to remember that this load is independent of the 1870 // other one. 1871 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 1872 Hi.getValue(1)); 1873 1874 // Legalize the chain result - switch anything that used the old chain to 1875 // use the new one. 1876 ReplaceValueWith(SDValue(MGT, 1), Ch); 1877} 1878 1879 1880void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { 1881 assert(N->getValueType(0).isVector() && 1882 N->getOperand(0).getValueType().isVector() && 1883 "Operand types must be vectors"); 1884 1885 EVT LoVT, HiVT; 1886 SDLoc DL(N); 1887 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1888 1889 // If the input also splits, handle it directly. Otherwise split it by hand. 1890 SDValue LL, LH, RL, RH; 1891 if (getTypeAction(N->getOperand(0).getValueType()) == 1892 TargetLowering::TypeSplitVector) 1893 GetSplitVector(N->getOperand(0), LL, LH); 1894 else 1895 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0); 1896 1897 if (getTypeAction(N->getOperand(1).getValueType()) == 1898 TargetLowering::TypeSplitVector) 1899 GetSplitVector(N->getOperand(1), RL, RH); 1900 else 1901 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1); 1902 1903 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2)); 1904 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2)); 1905} 1906 1907void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, 1908 SDValue &Hi) { 1909 // Get the dest types - they may not match the input types, e.g. int_to_fp. 1910 EVT LoVT, HiVT; 1911 SDLoc dl(N); 1912 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); 1913 1914 // If the input also splits, handle it directly for a compile time speedup. 1915 // Otherwise split it by hand. 1916 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 1917 EVT InVT = N->getOperand(OpNo).getValueType(); 1918 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) 1919 GetSplitVector(N->getOperand(OpNo), Lo, Hi); 1920 else 1921 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo); 1922 1923 if (N->getOpcode() == ISD::FP_ROUND) { 1924 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1), 1925 N->getFlags()); 1926 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1), 1927 N->getFlags()); 1928 } else { 1929 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags()); 1930 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags()); 1931 } 1932} 1933 1934void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, 1935 SDValue &Hi) { 1936 SDLoc dl(N); 1937 EVT SrcVT = N->getOperand(0).getValueType(); 1938 EVT DestVT = N->getValueType(0); 1939 EVT LoVT, HiVT; 1940 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT); 1941 1942 // We can do better than a generic split operation if the extend is doing 1943 // more than just doubling the width of the elements and the following are 1944 // true: 1945 // - The number of vector elements is even, 1946 // - the source type is legal, 1947 // - the type of a split source is illegal, 1948 // - the type of an extended (by doubling element size) source is legal, and 1949 // - the type of that extended source when split is legal. 1950 // 1951 // This won't necessarily completely legalize the operation, but it will 1952 // more effectively move in the right direction and prevent falling down 1953 // to scalarization in many cases due to the input vector being split too 1954 // far. 1955 if (SrcVT.getVectorElementCount().isKnownEven() && 1956 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) { 1957 LLVMContext &Ctx = *DAG.getContext(); 1958 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx); 1959 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx); 1960 1961 EVT SplitLoVT, SplitHiVT; 1962 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT); 1963 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) && 1964 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) { 1965 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:"; 1966 N->dump(&DAG); dbgs() << "\n"); 1967 // Extend the source vector by one step. 1968 SDValue NewSrc = 1969 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0)); 1970 // Get the low and high halves of the new, extended one step, vector. 1971 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl); 1972 // Extend those vector halves the rest of the way. 1973 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); 1974 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); 1975 return; 1976 } 1977 } 1978 // Fall back to the generic unary operator splitting otherwise. 1979 SplitVecRes_UnaryOp(N, Lo, Hi); 1980} 1981 1982void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, 1983 SDValue &Lo, SDValue &Hi) { 1984 // The low and high parts of the original input give four input vectors. 1985 SDValue Inputs[4]; 1986 SDLoc dl(N); 1987 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]); 1988 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]); 1989 EVT NewVT = Inputs[0].getValueType(); 1990 unsigned NewElts = NewVT.getVectorNumElements(); 1991 1992 // If Lo or Hi uses elements from at most two of the four input vectors, then 1993 // express it as a vector shuffle of those two inputs. Otherwise extract the 1994 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR. 1995 SmallVector<int, 16> Ops; 1996 for (unsigned High = 0; High < 2; ++High) { 1997 SDValue &Output = High ? Hi : Lo; 1998 1999 // Build a shuffle mask for the output, discovering on the fly which 2000 // input vectors to use as shuffle operands (recorded in InputUsed). 2001 // If building a suitable shuffle vector proves too hard, then bail 2002 // out with useBuildVector set. 2003 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered. 2004 unsigned FirstMaskIdx = High * NewElts; 2005 bool useBuildVector = false; 2006 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 2007 // The mask element. This indexes into the input. 2008 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 2009 2010 // The input vector this mask element indexes into. 2011 unsigned Input = (unsigned)Idx / NewElts; 2012 2013 if (Input >= array_lengthof(Inputs)) { 2014 // The mask element does not index into any input vector. 2015 Ops.push_back(-1); 2016 continue; 2017 } 2018 2019 // Turn the index into an offset from the start of the input vector. 2020 Idx -= Input * NewElts; 2021 2022 // Find or create a shuffle vector operand to hold this input. 2023 unsigned OpNo; 2024 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) { 2025 if (InputUsed[OpNo] == Input) { 2026 // This input vector is already an operand. 2027 break; 2028 } else if (InputUsed[OpNo] == -1U) { 2029 // Create a new operand for this input vector. 2030 InputUsed[OpNo] = Input; 2031 break; 2032 } 2033 } 2034 2035 if (OpNo >= array_lengthof(InputUsed)) { 2036 // More than two input vectors used! Give up on trying to create a 2037 // shuffle vector. Insert all elements into a BUILD_VECTOR instead. 2038 useBuildVector = true; 2039 break; 2040 } 2041 2042 // Add the mask index for the new shuffle vector. 2043 Ops.push_back(Idx + OpNo * NewElts); 2044 } 2045 2046 if (useBuildVector) { 2047 EVT EltVT = NewVT.getVectorElementType(); 2048 SmallVector<SDValue, 16> SVOps; 2049 2050 // Extract the input elements by hand. 2051 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) { 2052 // The mask element. This indexes into the input. 2053 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset); 2054 2055 // The input vector this mask element indexes into. 2056 unsigned Input = (unsigned)Idx / NewElts; 2057 2058 if (Input >= array_lengthof(Inputs)) { 2059 // The mask element is "undef" or indexes off the end of the input. 2060 SVOps.push_back(DAG.getUNDEF(EltVT)); 2061 continue; 2062 } 2063 2064 // Turn the index into an offset from the start of the input vector. 2065 Idx -= Input * NewElts; 2066 2067 // Extract the vector element by hand. 2068 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, 2069 Inputs[Input], 2070 DAG.getVectorIdxConstant(Idx, dl))); 2071 } 2072 2073 // Construct the Lo/Hi output using a BUILD_VECTOR. 2074 Output = DAG.getBuildVector(NewVT, dl, SVOps); 2075 } else if (InputUsed[0] == -1U) { 2076 // No input vectors were used! The result is undefined. 2077 Output = DAG.getUNDEF(NewVT); 2078 } else { 2079 SDValue Op0 = Inputs[InputUsed[0]]; 2080 // If only one input was used, use an undefined vector for the other. 2081 SDValue Op1 = InputUsed[1] == -1U ? 2082 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]]; 2083 // At least one input vector was used. Create a new shuffle vector. 2084 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops); 2085 } 2086 2087 Ops.clear(); 2088 } 2089} 2090 2091void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { 2092 EVT OVT = N->getValueType(0); 2093 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext()); 2094 SDValue Chain = N->getOperand(0); 2095 SDValue Ptr = N->getOperand(1); 2096 SDValue SV = N->getOperand(2); 2097 SDLoc dl(N); 2098 2099 const Align Alignment = 2100 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext())); 2101 2102 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value()); 2103 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value()); 2104 Chain = Hi.getValue(1); 2105 2106 // Modified the chain - switch anything that used the old chain to use 2107 // the new one. 2108 ReplaceValueWith(SDValue(N, 1), Chain); 2109} 2110 2111void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, 2112 SDValue &Hi) { 2113 EVT DstVTLo, DstVTHi; 2114 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0)); 2115 SDLoc dl(N); 2116 2117 SDValue SrcLo, SrcHi; 2118 EVT SrcVT = N->getOperand(0).getValueType(); 2119 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector) 2120 GetSplitVector(N->getOperand(0), SrcLo, SrcHi); 2121 else 2122 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0); 2123 2124 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1)); 2125 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1)); 2126} 2127 2128//===----------------------------------------------------------------------===// 2129// Operand Vector Splitting 2130//===----------------------------------------------------------------------===// 2131 2132/// This method is called when the specified operand of the specified node is 2133/// found to need vector splitting. At this point, all of the result types of 2134/// the node are known to be legal, but other operands of the node may need 2135/// legalization as well as the specified one. 2136bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { 2137 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n"); 2138 SDValue Res = SDValue(); 2139 2140 // See if the target wants to custom split this node. 2141 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2142 return false; 2143 2144 switch (N->getOpcode()) { 2145 default: 2146#ifndef NDEBUG 2147 dbgs() << "SplitVectorOperand Op #" << OpNo << ": "; 2148 N->dump(&DAG); 2149 dbgs() << "\n"; 2150#endif 2151 report_fatal_error("Do not know how to split this operator's " 2152 "operand!\n"); 2153 2154 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break; 2155 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break; 2156 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break; 2157 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break; 2158 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break; 2159 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break; 2160 case ISD::TRUNCATE: 2161 Res = SplitVecOp_TruncateHelper(N); 2162 break; 2163 case ISD::STRICT_FP_ROUND: 2164 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; 2165 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break; 2166 case ISD::STORE: 2167 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo); 2168 break; 2169 case ISD::MSTORE: 2170 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo); 2171 break; 2172 case ISD::MSCATTER: 2173 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo); 2174 break; 2175 case ISD::MGATHER: 2176 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo); 2177 break; 2178 case ISD::VSELECT: 2179 Res = SplitVecOp_VSELECT(N, OpNo); 2180 break; 2181 case ISD::STRICT_SINT_TO_FP: 2182 case ISD::STRICT_UINT_TO_FP: 2183 case ISD::SINT_TO_FP: 2184 case ISD::UINT_TO_FP: 2185 if (N->getValueType(0).bitsLT( 2186 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType())) 2187 Res = SplitVecOp_TruncateHelper(N); 2188 else 2189 Res = SplitVecOp_UnaryOp(N); 2190 break; 2191 case ISD::FP_TO_SINT_SAT: 2192 case ISD::FP_TO_UINT_SAT: 2193 Res = SplitVecOp_FP_TO_XINT_SAT(N); 2194 break; 2195 case ISD::FP_TO_SINT: 2196 case ISD::FP_TO_UINT: 2197 case ISD::STRICT_FP_TO_SINT: 2198 case ISD::STRICT_FP_TO_UINT: 2199 case ISD::STRICT_FP_EXTEND: 2200 case ISD::FP_EXTEND: 2201 case ISD::SIGN_EXTEND: 2202 case ISD::ZERO_EXTEND: 2203 case ISD::ANY_EXTEND: 2204 case ISD::FTRUNC: 2205 Res = SplitVecOp_UnaryOp(N); 2206 break; 2207 2208 case ISD::ANY_EXTEND_VECTOR_INREG: 2209 case ISD::SIGN_EXTEND_VECTOR_INREG: 2210 case ISD::ZERO_EXTEND_VECTOR_INREG: 2211 Res = SplitVecOp_ExtVecInRegOp(N); 2212 break; 2213 2214 case ISD::VECREDUCE_FADD: 2215 case ISD::VECREDUCE_FMUL: 2216 case ISD::VECREDUCE_ADD: 2217 case ISD::VECREDUCE_MUL: 2218 case ISD::VECREDUCE_AND: 2219 case ISD::VECREDUCE_OR: 2220 case ISD::VECREDUCE_XOR: 2221 case ISD::VECREDUCE_SMAX: 2222 case ISD::VECREDUCE_SMIN: 2223 case ISD::VECREDUCE_UMAX: 2224 case ISD::VECREDUCE_UMIN: 2225 case ISD::VECREDUCE_FMAX: 2226 case ISD::VECREDUCE_FMIN: 2227 Res = SplitVecOp_VECREDUCE(N, OpNo); 2228 break; 2229 case ISD::VECREDUCE_SEQ_FADD: 2230 case ISD::VECREDUCE_SEQ_FMUL: 2231 Res = SplitVecOp_VECREDUCE_SEQ(N); 2232 break; 2233 } 2234 2235 // If the result is null, the sub-method took care of registering results etc. 2236 if (!Res.getNode()) return false; 2237 2238 // If the result is N, the sub-method updated N in place. Tell the legalizer 2239 // core about this. 2240 if (Res.getNode() == N) 2241 return true; 2242 2243 if (N->isStrictFPOpcode()) 2244 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 2245 "Invalid operand expansion"); 2246 else 2247 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2248 "Invalid operand expansion"); 2249 2250 ReplaceValueWith(SDValue(N, 0), Res); 2251 return false; 2252} 2253 2254SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { 2255 // The only possibility for an illegal operand is the mask, since result type 2256 // legalization would have handled this node already otherwise. 2257 assert(OpNo == 0 && "Illegal operand must be mask"); 2258 2259 SDValue Mask = N->getOperand(0); 2260 SDValue Src0 = N->getOperand(1); 2261 SDValue Src1 = N->getOperand(2); 2262 EVT Src0VT = Src0.getValueType(); 2263 SDLoc DL(N); 2264 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?"); 2265 2266 SDValue Lo, Hi; 2267 GetSplitVector(N->getOperand(0), Lo, Hi); 2268 assert(Lo.getValueType() == Hi.getValueType() && 2269 "Lo and Hi have differing types"); 2270 2271 EVT LoOpVT, HiOpVT; 2272 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT); 2273 assert(LoOpVT == HiOpVT && "Asymmetric vector split?"); 2274 2275 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask; 2276 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL); 2277 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL); 2278 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL); 2279 2280 SDValue LoSelect = 2281 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1); 2282 SDValue HiSelect = 2283 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1); 2284 2285 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect); 2286} 2287 2288SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { 2289 EVT ResVT = N->getValueType(0); 2290 SDValue Lo, Hi; 2291 SDLoc dl(N); 2292 2293 SDValue VecOp = N->getOperand(OpNo); 2294 EVT VecVT = VecOp.getValueType(); 2295 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2296 GetSplitVector(VecOp, Lo, Hi); 2297 EVT LoOpVT, HiOpVT; 2298 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 2299 2300 // Use the appropriate scalar instruction on the split subvectors before 2301 // reducing the now partially reduced smaller vector. 2302 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode()); 2303 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags()); 2304 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags()); 2305} 2306 2307SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) { 2308 EVT ResVT = N->getValueType(0); 2309 SDValue Lo, Hi; 2310 SDLoc dl(N); 2311 2312 SDValue AccOp = N->getOperand(0); 2313 SDValue VecOp = N->getOperand(1); 2314 SDNodeFlags Flags = N->getFlags(); 2315 2316 EVT VecVT = VecOp.getValueType(); 2317 assert(VecVT.isVector() && "Can only split reduce vector operand"); 2318 GetSplitVector(VecOp, Lo, Hi); 2319 EVT LoOpVT, HiOpVT; 2320 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT); 2321 2322 // Reduce low half. 2323 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags); 2324 2325 // Reduce high half, using low half result as initial value. 2326 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags); 2327} 2328 2329SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { 2330 // The result has a legal vector type, but the input needs splitting. 2331 EVT ResVT = N->getValueType(0); 2332 SDValue Lo, Hi; 2333 SDLoc dl(N); 2334 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 2335 EVT InVT = Lo.getValueType(); 2336 2337 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2338 InVT.getVectorElementCount()); 2339 2340 if (N->isStrictFPOpcode()) { 2341 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2342 { N->getOperand(0), Lo }); 2343 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 2344 { N->getOperand(0), Hi }); 2345 2346 // Build a factor node to remember that this operation is independent 2347 // of the other one. 2348 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2349 Hi.getValue(1)); 2350 2351 // Legalize the chain result - switch anything that used the old chain to 2352 // use the new one. 2353 ReplaceValueWith(SDValue(N, 1), Ch); 2354 } else { 2355 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); 2356 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); 2357 } 2358 2359 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 2360} 2361 2362SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { 2363 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will 2364 // end up being split all the way down to individual components. Convert the 2365 // split pieces into integers and reassemble. 2366 SDValue Lo, Hi; 2367 GetSplitVector(N->getOperand(0), Lo, Hi); 2368 Lo = BitConvertToInteger(Lo); 2369 Hi = BitConvertToInteger(Hi); 2370 2371 if (DAG.getDataLayout().isBigEndian()) 2372 std::swap(Lo, Hi); 2373 2374 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), 2375 JoinIntegers(Lo, Hi)); 2376} 2377 2378SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, 2379 unsigned OpNo) { 2380 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec."); 2381 // We know that the result type is legal. 2382 EVT ResVT = N->getValueType(0); 2383 2384 SDValue Vec = N->getOperand(0); 2385 SDValue SubVec = N->getOperand(1); 2386 SDValue Idx = N->getOperand(2); 2387 SDLoc dl(N); 2388 2389 SDValue Lo, Hi; 2390 GetSplitVector(SubVec, Lo, Hi); 2391 2392 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2393 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 2394 2395 SDValue FirstInsertion = 2396 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx); 2397 SDValue SecondInsertion = 2398 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi, 2399 DAG.getVectorIdxConstant(IdxVal + LoElts, dl)); 2400 2401 return SecondInsertion; 2402} 2403 2404SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 2405 // We know that the extracted result type is legal. 2406 EVT SubVT = N->getValueType(0); 2407 2408 SDValue Idx = N->getOperand(1); 2409 SDLoc dl(N); 2410 SDValue Lo, Hi; 2411 2412 if (SubVT.isScalableVector() != 2413 N->getOperand(0).getValueType().isScalableVector()) 2414 report_fatal_error("Extracting a fixed-length vector from an illegal " 2415 "scalable vector is not yet supported"); 2416 2417 GetSplitVector(N->getOperand(0), Lo, Hi); 2418 2419 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 2420 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2421 2422 if (IdxVal < LoElts) { 2423 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts && 2424 "Extracted subvector crosses vector split!"); 2425 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx); 2426 } else { 2427 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi, 2428 DAG.getVectorIdxConstant(IdxVal - LoElts, dl)); 2429 } 2430} 2431 2432SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 2433 SDValue Vec = N->getOperand(0); 2434 SDValue Idx = N->getOperand(1); 2435 EVT VecVT = Vec.getValueType(); 2436 2437 if (isa<ConstantSDNode>(Idx)) { 2438 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 2439 2440 SDValue Lo, Hi; 2441 GetSplitVector(Vec, Lo, Hi); 2442 2443 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); 2444 2445 if (IdxVal < LoElts) 2446 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); 2447 else if (!Vec.getValueType().isScalableVector()) 2448 return SDValue(DAG.UpdateNodeOperands(N, Hi, 2449 DAG.getConstant(IdxVal - LoElts, SDLoc(N), 2450 Idx.getValueType())), 0); 2451 } 2452 2453 // See if the target wants to custom expand this node. 2454 if (CustomLowerNode(N, N->getValueType(0), true)) 2455 return SDValue(); 2456 2457 // Make the vector elements byte-addressable if they aren't already. 2458 SDLoc dl(N); 2459 EVT EltVT = VecVT.getVectorElementType(); 2460 if (VecVT.getScalarSizeInBits() < 8) { 2461 EltVT = MVT::i8; 2462 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 2463 VecVT.getVectorElementCount()); 2464 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); 2465 } 2466 2467 // Store the vector to the stack. 2468 // In cases where the vector is illegal it will be broken down into parts 2469 // and stored in parts - we should use the alignment for the smallest part. 2470 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false); 2471 SDValue StackPtr = 2472 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign); 2473 auto &MF = DAG.getMachineFunction(); 2474 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); 2475 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex); 2476 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo, 2477 SmallestAlign); 2478 2479 // Load back the required element. 2480 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); 2481 2482 // FIXME: This is to handle i1 vectors with elements promoted to i8. 2483 // i1 vector handling needs general improvement. 2484 if (N->getValueType(0).bitsLT(EltVT)) { 2485 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr, 2486 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); 2487 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0)); 2488 } 2489 2490 return DAG.getExtLoad( 2491 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, 2492 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT, 2493 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8)); 2494} 2495 2496SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { 2497 SDValue Lo, Hi; 2498 2499 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so 2500 // splitting the result has the same effect as splitting the input operand. 2501 SplitVecRes_ExtVecInRegOp(N, Lo, Hi); 2502 2503 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi); 2504} 2505 2506SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, 2507 unsigned OpNo) { 2508 EVT LoVT, HiVT; 2509 SDLoc dl(MGT); 2510 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0)); 2511 2512 SDValue Ch = MGT->getChain(); 2513 SDValue Ptr = MGT->getBasePtr(); 2514 SDValue Index = MGT->getIndex(); 2515 SDValue Scale = MGT->getScale(); 2516 SDValue Mask = MGT->getMask(); 2517 SDValue PassThru = MGT->getPassThru(); 2518 Align Alignment = MGT->getOriginalAlign(); 2519 ISD::LoadExtType ExtType = MGT->getExtensionType(); 2520 2521 SDValue MaskLo, MaskHi; 2522 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2523 // Split Mask operand 2524 GetSplitVector(Mask, MaskLo, MaskHi); 2525 else 2526 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl); 2527 2528 EVT MemoryVT = MGT->getMemoryVT(); 2529 EVT LoMemVT, HiMemVT; 2530 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2531 2532 SDValue PassThruLo, PassThruHi; 2533 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector) 2534 GetSplitVector(PassThru, PassThruLo, PassThruHi); 2535 else 2536 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl); 2537 2538 SDValue IndexHi, IndexLo; 2539 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 2540 GetSplitVector(Index, IndexLo, IndexHi); 2541 else 2542 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl); 2543 2544 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2545 MGT->getPointerInfo(), MachineMemOperand::MOLoad, 2546 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(), 2547 MGT->getRanges()); 2548 2549 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale}; 2550 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, 2551 OpsLo, MMO, MGT->getIndexType(), ExtType); 2552 2553 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale}; 2554 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, 2555 OpsHi, MMO, MGT->getIndexType(), ExtType); 2556 2557 // Build a factor node to remember that this load is independent of the 2558 // other one. 2559 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), 2560 Hi.getValue(1)); 2561 2562 // Legalize the chain result - switch anything that used the old chain to 2563 // use the new one. 2564 ReplaceValueWith(SDValue(MGT, 1), Ch); 2565 2566 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo, 2567 Hi); 2568 ReplaceValueWith(SDValue(MGT, 0), Res); 2569 return SDValue(); 2570} 2571 2572SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, 2573 unsigned OpNo) { 2574 assert(N->isUnindexed() && "Indexed masked store of vector?"); 2575 SDValue Ch = N->getChain(); 2576 SDValue Ptr = N->getBasePtr(); 2577 SDValue Offset = N->getOffset(); 2578 assert(Offset.isUndef() && "Unexpected indexed masked store offset"); 2579 SDValue Mask = N->getMask(); 2580 SDValue Data = N->getValue(); 2581 Align Alignment = N->getOriginalAlign(); 2582 SDLoc DL(N); 2583 2584 SDValue DataLo, DataHi; 2585 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 2586 // Split Data operand 2587 GetSplitVector(Data, DataLo, DataHi); 2588 else 2589 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 2590 2591 // Split Mask operand 2592 SDValue MaskLo, MaskHi; 2593 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 2594 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 2595 } else { 2596 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2597 GetSplitVector(Mask, MaskLo, MaskHi); 2598 else 2599 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 2600 } 2601 2602 EVT MemoryVT = N->getMemoryVT(); 2603 EVT LoMemVT, HiMemVT; 2604 bool HiIsEmpty = false; 2605 std::tie(LoMemVT, HiMemVT) = 2606 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty); 2607 2608 SDValue Lo, Hi, Res; 2609 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize()); 2610 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2611 N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment, 2612 N->getAAInfo(), N->getRanges()); 2613 2614 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO, 2615 N->getAddressingMode(), N->isTruncatingStore(), 2616 N->isCompressingStore()); 2617 2618 if (HiIsEmpty) { 2619 // The hi masked store has zero storage size. 2620 // Only the lo masked store is needed. 2621 Res = Lo; 2622 } else { 2623 2624 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG, 2625 N->isCompressingStore()); 2626 2627 MachinePointerInfo MPI; 2628 if (LoMemVT.isScalableVector()) { 2629 Alignment = commonAlignment( 2630 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8); 2631 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace()); 2632 } else 2633 MPI = N->getPointerInfo().getWithOffset( 2634 LoMemVT.getStoreSize().getFixedSize()); 2635 2636 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize()); 2637 MMO = DAG.getMachineFunction().getMachineMemOperand( 2638 MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(), 2639 N->getRanges()); 2640 2641 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO, 2642 N->getAddressingMode(), N->isTruncatingStore(), 2643 N->isCompressingStore()); 2644 2645 // Build a factor node to remember that this store is independent of the 2646 // other one. 2647 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 2648 } 2649 2650 return Res; 2651} 2652 2653SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N, 2654 unsigned OpNo) { 2655 SDValue Ch = N->getChain(); 2656 SDValue Ptr = N->getBasePtr(); 2657 SDValue Mask = N->getMask(); 2658 SDValue Index = N->getIndex(); 2659 SDValue Scale = N->getScale(); 2660 SDValue Data = N->getValue(); 2661 EVT MemoryVT = N->getMemoryVT(); 2662 Align Alignment = N->getOriginalAlign(); 2663 SDLoc DL(N); 2664 2665 // Split all operands 2666 2667 EVT LoMemVT, HiMemVT; 2668 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2669 2670 SDValue DataLo, DataHi; 2671 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector) 2672 // Split Data operand 2673 GetSplitVector(Data, DataLo, DataHi); 2674 else 2675 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL); 2676 2677 // Split Mask operand 2678 SDValue MaskLo, MaskHi; 2679 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) { 2680 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi); 2681 } else { 2682 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) 2683 GetSplitVector(Mask, MaskLo, MaskHi); 2684 else 2685 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); 2686 } 2687 2688 SDValue IndexHi, IndexLo; 2689 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector) 2690 GetSplitVector(Index, IndexLo, IndexHi); 2691 else 2692 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL); 2693 2694 SDValue Lo; 2695 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 2696 N->getPointerInfo(), MachineMemOperand::MOStore, 2697 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges()); 2698 2699 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale}; 2700 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT, 2701 DL, OpsLo, MMO, N->getIndexType(), 2702 N->isTruncatingStore()); 2703 2704 // The order of the Scatter operation after split is well defined. The "Hi" 2705 // part comes after the "Lo". So these two operations should be chained one 2706 // after another. 2707 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale}; 2708 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT, 2709 DL, OpsHi, MMO, N->getIndexType(), 2710 N->isTruncatingStore()); 2711} 2712 2713SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { 2714 assert(N->isUnindexed() && "Indexed store of vector?"); 2715 assert(OpNo == 1 && "Can only split the stored value"); 2716 SDLoc DL(N); 2717 2718 bool isTruncating = N->isTruncatingStore(); 2719 SDValue Ch = N->getChain(); 2720 SDValue Ptr = N->getBasePtr(); 2721 EVT MemoryVT = N->getMemoryVT(); 2722 Align Alignment = N->getOriginalAlign(); 2723 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); 2724 AAMDNodes AAInfo = N->getAAInfo(); 2725 SDValue Lo, Hi; 2726 GetSplitVector(N->getOperand(1), Lo, Hi); 2727 2728 EVT LoMemVT, HiMemVT; 2729 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); 2730 2731 // Scalarize if the split halves are not byte-sized. 2732 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) 2733 return TLI.scalarizeVectorStore(N, DAG); 2734 2735 if (isTruncating) 2736 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT, 2737 Alignment, MMOFlags, AAInfo); 2738 else 2739 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, 2740 AAInfo); 2741 2742 MachinePointerInfo MPI; 2743 IncrementPointer(N, LoMemVT, MPI, Ptr); 2744 2745 if (isTruncating) 2746 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI, 2747 HiMemVT, Alignment, MMOFlags, AAInfo); 2748 else 2749 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo); 2750 2751 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); 2752} 2753 2754SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { 2755 SDLoc DL(N); 2756 2757 // The input operands all must have the same type, and we know the result 2758 // type is valid. Convert this to a buildvector which extracts all the 2759 // input elements. 2760 // TODO: If the input elements are power-two vectors, we could convert this to 2761 // a new CONCAT_VECTORS node with elements that are half-wide. 2762 SmallVector<SDValue, 32> Elts; 2763 EVT EltVT = N->getValueType(0).getVectorElementType(); 2764 for (const SDValue &Op : N->op_values()) { 2765 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements(); 2766 i != e; ++i) { 2767 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op, 2768 DAG.getVectorIdxConstant(i, DL))); 2769 } 2770 } 2771 2772 return DAG.getBuildVector(N->getValueType(0), DL, Elts); 2773} 2774 2775SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { 2776 // The result type is legal, but the input type is illegal. If splitting 2777 // ends up with the result type of each half still being legal, just 2778 // do that. If, however, that would result in an illegal result type, 2779 // we can try to get more clever with power-two vectors. Specifically, 2780 // split the input type, but also widen the result element size, then 2781 // concatenate the halves and truncate again. For example, consider a target 2782 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit 2783 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do: 2784 // %inlo = v4i32 extract_subvector %in, 0 2785 // %inhi = v4i32 extract_subvector %in, 4 2786 // %lo16 = v4i16 trunc v4i32 %inlo 2787 // %hi16 = v4i16 trunc v4i32 %inhi 2788 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16 2789 // %res = v8i8 trunc v8i16 %in16 2790 // 2791 // Without this transform, the original truncate would end up being 2792 // scalarized, which is pretty much always a last resort. 2793 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 2794 SDValue InVec = N->getOperand(OpNo); 2795 EVT InVT = InVec->getValueType(0); 2796 EVT OutVT = N->getValueType(0); 2797 ElementCount NumElements = OutVT.getVectorElementCount(); 2798 bool IsFloat = OutVT.isFloatingPoint(); 2799 2800 unsigned InElementSize = InVT.getScalarSizeInBits(); 2801 unsigned OutElementSize = OutVT.getScalarSizeInBits(); 2802 2803 // Determine the split output VT. If its legal we can just split dirctly. 2804 EVT LoOutVT, HiOutVT; 2805 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT); 2806 assert(LoOutVT == HiOutVT && "Unequal split?"); 2807 2808 // If the input elements are only 1/2 the width of the result elements, 2809 // just use the normal splitting. Our trick only work if there's room 2810 // to split more than once. 2811 if (isTypeLegal(LoOutVT) || 2812 InElementSize <= OutElementSize * 2) 2813 return SplitVecOp_UnaryOp(N); 2814 SDLoc DL(N); 2815 2816 // Don't touch if this will be scalarized. 2817 EVT FinalVT = InVT; 2818 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 2819 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext()); 2820 2821 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector) 2822 return SplitVecOp_UnaryOp(N); 2823 2824 // Get the split input vector. 2825 SDValue InLoVec, InHiVec; 2826 GetSplitVector(InVec, InLoVec, InHiVec); 2827 2828 // Truncate them to 1/2 the element size. 2829 // 2830 // This assumes the number of elements is a power of two; any vector that 2831 // isn't should be widened, not split. 2832 EVT HalfElementVT = IsFloat ? 2833 EVT::getFloatingPointVT(InElementSize/2) : 2834 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2); 2835 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, 2836 NumElements.divideCoefficientBy(2)); 2837 2838 SDValue HalfLo; 2839 SDValue HalfHi; 2840 SDValue Chain; 2841 if (N->isStrictFPOpcode()) { 2842 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 2843 {N->getOperand(0), InLoVec}); 2844 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other}, 2845 {N->getOperand(0), InHiVec}); 2846 // Legalize the chain result - switch anything that used the old chain to 2847 // use the new one. 2848 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1), 2849 HalfHi.getValue(1)); 2850 } else { 2851 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec); 2852 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec); 2853 } 2854 // Concatenate them to get the full intermediate truncation result. 2855 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements); 2856 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo, 2857 HalfHi); 2858 // Now finish up by truncating all the way down to the original result 2859 // type. This should normally be something that ends up being legal directly, 2860 // but in theory if a target has very wide vectors and an annoyingly 2861 // restricted set of legal types, this split can chain to build things up. 2862 2863 if (N->isStrictFPOpcode()) { 2864 SDValue Res = DAG.getNode( 2865 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other}, 2866 {Chain, InterVec, 2867 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))}); 2868 // Relink the chain 2869 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1)); 2870 return Res; 2871 } 2872 2873 return IsFloat 2874 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec, 2875 DAG.getTargetConstant( 2876 0, DL, TLI.getPointerTy(DAG.getDataLayout()))) 2877 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec); 2878} 2879 2880SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { 2881 assert(N->getValueType(0).isVector() && 2882 N->getOperand(0).getValueType().isVector() && 2883 "Operand types must be vectors"); 2884 // The result has a legal vector type, but the input needs splitting. 2885 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes; 2886 SDLoc DL(N); 2887 GetSplitVector(N->getOperand(0), Lo0, Hi0); 2888 GetSplitVector(N->getOperand(1), Lo1, Hi1); 2889 auto PartEltCnt = Lo0.getValueType().getVectorElementCount(); 2890 2891 LLVMContext &Context = *DAG.getContext(); 2892 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt); 2893 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2); 2894 2895 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2)); 2896 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2)); 2897 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes); 2898 2899 EVT OpVT = N->getOperand(0).getValueType(); 2900 ISD::NodeType ExtendCode = 2901 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 2902 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con); 2903} 2904 2905 2906SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { 2907 // The result has a legal vector type, but the input needs splitting. 2908 EVT ResVT = N->getValueType(0); 2909 SDValue Lo, Hi; 2910 SDLoc DL(N); 2911 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); 2912 EVT InVT = Lo.getValueType(); 2913 2914 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2915 InVT.getVectorElementCount()); 2916 2917 if (N->isStrictFPOpcode()) { 2918 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 2919 { N->getOperand(0), Lo, N->getOperand(2) }); 2920 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 2921 { N->getOperand(0), Hi, N->getOperand(2) }); 2922 // Legalize the chain result - switch anything that used the old chain to 2923 // use the new one. 2924 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 2925 Lo.getValue(1), Hi.getValue(1)); 2926 ReplaceValueWith(SDValue(N, 1), NewChain); 2927 } else { 2928 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); 2929 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); 2930 } 2931 2932 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); 2933} 2934 2935SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) { 2936 // The result (and the first input) has a legal vector type, but the second 2937 // input needs splitting. 2938 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements()); 2939} 2940 2941SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { 2942 EVT ResVT = N->getValueType(0); 2943 SDValue Lo, Hi; 2944 SDLoc dl(N); 2945 GetSplitVector(N->getOperand(0), Lo, Hi); 2946 EVT InVT = Lo.getValueType(); 2947 2948 EVT NewResVT = 2949 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), 2950 InVT.getVectorElementCount()); 2951 2952 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1)); 2953 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1)); 2954 2955 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); 2956} 2957 2958//===----------------------------------------------------------------------===// 2959// Result Vector Widening 2960//===----------------------------------------------------------------------===// 2961 2962void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { 2963 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG); 2964 dbgs() << "\n"); 2965 2966 // See if the target wants to custom widen this node. 2967 if (CustomWidenLowerNode(N, N->getValueType(ResNo))) 2968 return; 2969 2970 SDValue Res = SDValue(); 2971 switch (N->getOpcode()) { 2972 default: 2973#ifndef NDEBUG 2974 dbgs() << "WidenVectorResult #" << ResNo << ": "; 2975 N->dump(&DAG); 2976 dbgs() << "\n"; 2977#endif 2978 llvm_unreachable("Do not know how to widen the result of this operator!"); 2979 2980 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break; 2981 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break; 2982 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break; 2983 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break; 2984 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break; 2985 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break; 2986 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break; 2987 case ISD::SPLAT_VECTOR: 2988 case ISD::SCALAR_TO_VECTOR: 2989 Res = WidenVecRes_ScalarOp(N); 2990 break; 2991 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break; 2992 case ISD::VSELECT: 2993 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break; 2994 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break; 2995 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break; 2996 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break; 2997 case ISD::VECTOR_SHUFFLE: 2998 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); 2999 break; 3000 case ISD::MLOAD: 3001 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N)); 3002 break; 3003 case ISD::MGATHER: 3004 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N)); 3005 break; 3006 3007 case ISD::ADD: 3008 case ISD::AND: 3009 case ISD::MUL: 3010 case ISD::MULHS: 3011 case ISD::MULHU: 3012 case ISD::OR: 3013 case ISD::SUB: 3014 case ISD::XOR: 3015 case ISD::SHL: 3016 case ISD::SRA: 3017 case ISD::SRL: 3018 case ISD::FMINNUM: 3019 case ISD::FMAXNUM: 3020 case ISD::FMINIMUM: 3021 case ISD::FMAXIMUM: 3022 case ISD::SMIN: 3023 case ISD::SMAX: 3024 case ISD::UMIN: 3025 case ISD::UMAX: 3026 case ISD::UADDSAT: 3027 case ISD::SADDSAT: 3028 case ISD::USUBSAT: 3029 case ISD::SSUBSAT: 3030 case ISD::SSHLSAT: 3031 case ISD::USHLSAT: 3032 case ISD::ROTL: 3033 case ISD::ROTR: 3034 Res = WidenVecRes_Binary(N); 3035 break; 3036 3037 case ISD::FADD: 3038 case ISD::FMUL: 3039 case ISD::FPOW: 3040 case ISD::FSUB: 3041 case ISD::FDIV: 3042 case ISD::FREM: 3043 case ISD::SDIV: 3044 case ISD::UDIV: 3045 case ISD::SREM: 3046 case ISD::UREM: 3047 Res = WidenVecRes_BinaryCanTrap(N); 3048 break; 3049 3050 case ISD::SMULFIX: 3051 case ISD::SMULFIXSAT: 3052 case ISD::UMULFIX: 3053 case ISD::UMULFIXSAT: 3054 // These are binary operations, but with an extra operand that shouldn't 3055 // be widened (the scale). 3056 Res = WidenVecRes_BinaryWithExtraScalarOp(N); 3057 break; 3058 3059#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 3060 case ISD::STRICT_##DAGN: 3061#include "llvm/IR/ConstrainedOps.def" 3062 Res = WidenVecRes_StrictFP(N); 3063 break; 3064 3065 case ISD::UADDO: 3066 case ISD::SADDO: 3067 case ISD::USUBO: 3068 case ISD::SSUBO: 3069 case ISD::UMULO: 3070 case ISD::SMULO: 3071 Res = WidenVecRes_OverflowOp(N, ResNo); 3072 break; 3073 3074 case ISD::FCOPYSIGN: 3075 Res = WidenVecRes_FCOPYSIGN(N); 3076 break; 3077 3078 case ISD::FPOWI: 3079 Res = WidenVecRes_POWI(N); 3080 break; 3081 3082 case ISD::ANY_EXTEND_VECTOR_INREG: 3083 case ISD::SIGN_EXTEND_VECTOR_INREG: 3084 case ISD::ZERO_EXTEND_VECTOR_INREG: 3085 Res = WidenVecRes_EXTEND_VECTOR_INREG(N); 3086 break; 3087 3088 case ISD::ANY_EXTEND: 3089 case ISD::FP_EXTEND: 3090 case ISD::FP_ROUND: 3091 case ISD::FP_TO_SINT: 3092 case ISD::FP_TO_UINT: 3093 case ISD::SIGN_EXTEND: 3094 case ISD::SINT_TO_FP: 3095 case ISD::TRUNCATE: 3096 case ISD::UINT_TO_FP: 3097 case ISD::ZERO_EXTEND: 3098 Res = WidenVecRes_Convert(N); 3099 break; 3100 3101 case ISD::FP_TO_SINT_SAT: 3102 case ISD::FP_TO_UINT_SAT: 3103 Res = WidenVecRes_FP_TO_XINT_SAT(N); 3104 break; 3105 3106 case ISD::FABS: 3107 case ISD::FCEIL: 3108 case ISD::FCOS: 3109 case ISD::FEXP: 3110 case ISD::FEXP2: 3111 case ISD::FFLOOR: 3112 case ISD::FLOG: 3113 case ISD::FLOG10: 3114 case ISD::FLOG2: 3115 case ISD::FNEARBYINT: 3116 case ISD::FRINT: 3117 case ISD::FROUND: 3118 case ISD::FROUNDEVEN: 3119 case ISD::FSIN: 3120 case ISD::FSQRT: 3121 case ISD::FTRUNC: { 3122 // We're going to widen this vector op to a legal type by padding with undef 3123 // elements. If the wide vector op is eventually going to be expanded to 3124 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary 3125 // libcalls on the undef elements. 3126 EVT VT = N->getValueType(0); 3127 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3128 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && 3129 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { 3130 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements()); 3131 break; 3132 } 3133 } 3134 // If the target has custom/legal support for the scalar FP intrinsic ops 3135 // (they are probably not destined to become libcalls), then widen those like 3136 // any other unary ops. 3137 LLVM_FALLTHROUGH; 3138 3139 case ISD::ABS: 3140 case ISD::BITREVERSE: 3141 case ISD::BSWAP: 3142 case ISD::CTLZ: 3143 case ISD::CTLZ_ZERO_UNDEF: 3144 case ISD::CTPOP: 3145 case ISD::CTTZ: 3146 case ISD::CTTZ_ZERO_UNDEF: 3147 case ISD::FNEG: 3148 case ISD::FREEZE: 3149 case ISD::FCANONICALIZE: 3150 Res = WidenVecRes_Unary(N); 3151 break; 3152 case ISD::FMA: 3153 case ISD::FSHL: 3154 case ISD::FSHR: 3155 Res = WidenVecRes_Ternary(N); 3156 break; 3157 } 3158 3159 // If Res is null, the sub-method took care of registering the result. 3160 if (Res.getNode()) 3161 SetWidenedVector(SDValue(N, ResNo), Res); 3162} 3163 3164SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { 3165 // Ternary op widening. 3166 SDLoc dl(N); 3167 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3168 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3169 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3170 SDValue InOp3 = GetWidenedVector(N->getOperand(2)); 3171 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3); 3172} 3173 3174SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { 3175 // Binary op widening. 3176 SDLoc dl(N); 3177 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3178 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3179 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3180 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags()); 3181} 3182 3183SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) { 3184 // Binary op widening, but with an extra operand that shouldn't be widened. 3185 SDLoc dl(N); 3186 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3187 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3188 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3189 SDValue InOp3 = N->getOperand(2); 3190 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3, 3191 N->getFlags()); 3192} 3193 3194// Given a vector of operations that have been broken up to widen, see 3195// if we can collect them together into the next widest legal VT. This 3196// implementation is trap-safe. 3197static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, 3198 SmallVectorImpl<SDValue> &ConcatOps, 3199 unsigned ConcatEnd, EVT VT, EVT MaxVT, 3200 EVT WidenVT) { 3201 // Check to see if we have a single operation with the widen type. 3202 if (ConcatEnd == 1) { 3203 VT = ConcatOps[0].getValueType(); 3204 if (VT == WidenVT) 3205 return ConcatOps[0]; 3206 } 3207 3208 SDLoc dl(ConcatOps[0]); 3209 EVT WidenEltVT = WidenVT.getVectorElementType(); 3210 3211 // while (Some element of ConcatOps is not of type MaxVT) { 3212 // From the end of ConcatOps, collect elements of the same type and put 3213 // them into an op of the next larger supported type 3214 // } 3215 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { 3216 int Idx = ConcatEnd - 1; 3217 VT = ConcatOps[Idx--].getValueType(); 3218 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) 3219 Idx--; 3220 3221 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1; 3222 EVT NextVT; 3223 do { 3224 NextSize *= 2; 3225 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize); 3226 } while (!TLI.isTypeLegal(NextVT)); 3227 3228 if (!VT.isVector()) { 3229 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT 3230 SDValue VecOp = DAG.getUNDEF(NextVT); 3231 unsigned NumToInsert = ConcatEnd - Idx - 1; 3232 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) { 3233 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, 3234 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl)); 3235 } 3236 ConcatOps[Idx+1] = VecOp; 3237 ConcatEnd = Idx + 2; 3238 } else { 3239 // Vector type, create a CONCAT_VECTORS of type NextVT 3240 SDValue undefVec = DAG.getUNDEF(VT); 3241 unsigned OpsToConcat = NextSize/VT.getVectorNumElements(); 3242 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat); 3243 unsigned RealVals = ConcatEnd - Idx - 1; 3244 unsigned SubConcatEnd = 0; 3245 unsigned SubConcatIdx = Idx + 1; 3246 while (SubConcatEnd < RealVals) 3247 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx]; 3248 while (SubConcatEnd < OpsToConcat) 3249 SubConcatOps[SubConcatEnd++] = undefVec; 3250 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl, 3251 NextVT, SubConcatOps); 3252 ConcatEnd = SubConcatIdx + 1; 3253 } 3254 } 3255 3256 // Check to see if we have a single operation with the widen type. 3257 if (ConcatEnd == 1) { 3258 VT = ConcatOps[0].getValueType(); 3259 if (VT == WidenVT) 3260 return ConcatOps[0]; 3261 } 3262 3263 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT 3264 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements(); 3265 if (NumOps != ConcatEnd ) { 3266 SDValue UndefVal = DAG.getUNDEF(MaxVT); 3267 for (unsigned j = ConcatEnd; j < NumOps; ++j) 3268 ConcatOps[j] = UndefVal; 3269 } 3270 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 3271 makeArrayRef(ConcatOps.data(), NumOps)); 3272} 3273 3274SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { 3275 // Binary op widening for operations that can trap. 3276 unsigned Opcode = N->getOpcode(); 3277 SDLoc dl(N); 3278 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3279 EVT WidenEltVT = WidenVT.getVectorElementType(); 3280 EVT VT = WidenVT; 3281 unsigned NumElts = VT.getVectorNumElements(); 3282 const SDNodeFlags Flags = N->getFlags(); 3283 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 3284 NumElts = NumElts / 2; 3285 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3286 } 3287 3288 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) { 3289 // Operation doesn't trap so just widen as normal. 3290 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3291 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3292 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags); 3293 } 3294 3295 // No legal vector version so unroll the vector operation and then widen. 3296 if (NumElts == 1) 3297 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 3298 3299 // Since the operation can trap, apply operation on the original vector. 3300 EVT MaxVT = VT; 3301 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 3302 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 3303 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 3304 3305 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 3306 unsigned ConcatEnd = 0; // Current ConcatOps index. 3307 int Idx = 0; // Current Idx into input vectors. 3308 3309 // NumElts := greatest legal vector size (at most WidenVT) 3310 // while (orig. vector has unhandled elements) { 3311 // take munches of size NumElts from the beginning and add to ConcatOps 3312 // NumElts := next smaller supported vector size or 1 3313 // } 3314 while (CurNumElts != 0) { 3315 while (CurNumElts >= NumElts) { 3316 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1, 3317 DAG.getVectorIdxConstant(Idx, dl)); 3318 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2, 3319 DAG.getVectorIdxConstant(Idx, dl)); 3320 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags); 3321 Idx += NumElts; 3322 CurNumElts -= NumElts; 3323 } 3324 do { 3325 NumElts = NumElts / 2; 3326 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3327 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 3328 3329 if (NumElts == 1) { 3330 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 3331 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 3332 InOp1, DAG.getVectorIdxConstant(Idx, dl)); 3333 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, 3334 InOp2, DAG.getVectorIdxConstant(Idx, dl)); 3335 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT, 3336 EOp1, EOp2, Flags); 3337 } 3338 CurNumElts = 0; 3339 } 3340 } 3341 3342 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 3343} 3344 3345SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { 3346 switch (N->getOpcode()) { 3347 case ISD::STRICT_FSETCC: 3348 case ISD::STRICT_FSETCCS: 3349 return WidenVecRes_STRICT_FSETCC(N); 3350 case ISD::STRICT_FP_EXTEND: 3351 case ISD::STRICT_FP_ROUND: 3352 case ISD::STRICT_FP_TO_SINT: 3353 case ISD::STRICT_FP_TO_UINT: 3354 case ISD::STRICT_SINT_TO_FP: 3355 case ISD::STRICT_UINT_TO_FP: 3356 return WidenVecRes_Convert_StrictFP(N); 3357 default: 3358 break; 3359 } 3360 3361 // StrictFP op widening for operations that can trap. 3362 unsigned NumOpers = N->getNumOperands(); 3363 unsigned Opcode = N->getOpcode(); 3364 SDLoc dl(N); 3365 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3366 EVT WidenEltVT = WidenVT.getVectorElementType(); 3367 EVT VT = WidenVT; 3368 unsigned NumElts = VT.getVectorNumElements(); 3369 while (!TLI.isTypeLegal(VT) && NumElts != 1) { 3370 NumElts = NumElts / 2; 3371 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3372 } 3373 3374 // No legal vector version so unroll the vector operation and then widen. 3375 if (NumElts == 1) 3376 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements()); 3377 3378 // Since the operation can trap, apply operation on the original vector. 3379 EVT MaxVT = VT; 3380 SmallVector<SDValue, 4> InOps; 3381 unsigned CurNumElts = N->getValueType(0).getVectorNumElements(); 3382 3383 SmallVector<SDValue, 16> ConcatOps(CurNumElts); 3384 SmallVector<SDValue, 16> Chains; 3385 unsigned ConcatEnd = 0; // Current ConcatOps index. 3386 int Idx = 0; // Current Idx into input vectors. 3387 3388 // The Chain is the first operand. 3389 InOps.push_back(N->getOperand(0)); 3390 3391 // Now process the remaining operands. 3392 for (unsigned i = 1; i < NumOpers; ++i) { 3393 SDValue Oper = N->getOperand(i); 3394 3395 if (Oper.getValueType().isVector()) { 3396 assert(Oper.getValueType() == N->getValueType(0) && 3397 "Invalid operand type to widen!"); 3398 Oper = GetWidenedVector(Oper); 3399 } 3400 3401 InOps.push_back(Oper); 3402 } 3403 3404 // NumElts := greatest legal vector size (at most WidenVT) 3405 // while (orig. vector has unhandled elements) { 3406 // take munches of size NumElts from the beginning and add to ConcatOps 3407 // NumElts := next smaller supported vector size or 1 3408 // } 3409 while (CurNumElts != 0) { 3410 while (CurNumElts >= NumElts) { 3411 SmallVector<SDValue, 4> EOps; 3412 3413 for (unsigned i = 0; i < NumOpers; ++i) { 3414 SDValue Op = InOps[i]; 3415 3416 if (Op.getValueType().isVector()) 3417 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op, 3418 DAG.getVectorIdxConstant(Idx, dl)); 3419 3420 EOps.push_back(Op); 3421 } 3422 3423 EVT OperVT[] = {VT, MVT::Other}; 3424 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps); 3425 ConcatOps[ConcatEnd++] = Oper; 3426 Chains.push_back(Oper.getValue(1)); 3427 Idx += NumElts; 3428 CurNumElts -= NumElts; 3429 } 3430 do { 3431 NumElts = NumElts / 2; 3432 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts); 3433 } while (!TLI.isTypeLegal(VT) && NumElts != 1); 3434 3435 if (NumElts == 1) { 3436 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) { 3437 SmallVector<SDValue, 4> EOps; 3438 3439 for (unsigned i = 0; i < NumOpers; ++i) { 3440 SDValue Op = InOps[i]; 3441 3442 if (Op.getValueType().isVector()) 3443 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op, 3444 DAG.getVectorIdxConstant(Idx, dl)); 3445 3446 EOps.push_back(Op); 3447 } 3448 3449 EVT WidenVT[] = {WidenEltVT, MVT::Other}; 3450 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps); 3451 ConcatOps[ConcatEnd++] = Oper; 3452 Chains.push_back(Oper.getValue(1)); 3453 } 3454 CurNumElts = 0; 3455 } 3456 } 3457 3458 // Build a factor node to remember all the Ops that have been created. 3459 SDValue NewChain; 3460 if (Chains.size() == 1) 3461 NewChain = Chains[0]; 3462 else 3463 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 3464 ReplaceValueWith(SDValue(N, 1), NewChain); 3465 3466 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); 3467} 3468 3469SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { 3470 SDLoc DL(N); 3471 EVT ResVT = N->getValueType(0); 3472 EVT OvVT = N->getValueType(1); 3473 EVT WideResVT, WideOvVT; 3474 SDValue WideLHS, WideRHS; 3475 3476 // TODO: This might result in a widen/split loop. 3477 if (ResNo == 0) { 3478 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT); 3479 WideOvVT = EVT::getVectorVT( 3480 *DAG.getContext(), OvVT.getVectorElementType(), 3481 WideResVT.getVectorNumElements()); 3482 3483 WideLHS = GetWidenedVector(N->getOperand(0)); 3484 WideRHS = GetWidenedVector(N->getOperand(1)); 3485 } else { 3486 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT); 3487 WideResVT = EVT::getVectorVT( 3488 *DAG.getContext(), ResVT.getVectorElementType(), 3489 WideOvVT.getVectorNumElements()); 3490 3491 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 3492 WideLHS = DAG.getNode( 3493 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 3494 N->getOperand(0), Zero); 3495 WideRHS = DAG.getNode( 3496 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), 3497 N->getOperand(1), Zero); 3498 } 3499 3500 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT); 3501 SDNode *WideNode = DAG.getNode( 3502 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode(); 3503 3504 // Replace the other vector result not being explicitly widened here. 3505 unsigned OtherNo = 1 - ResNo; 3506 EVT OtherVT = N->getValueType(OtherNo); 3507 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) { 3508 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo)); 3509 } else { 3510 SDValue Zero = DAG.getVectorIdxConstant(0, DL); 3511 SDValue OtherVal = DAG.getNode( 3512 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero); 3513 ReplaceValueWith(SDValue(N, OtherNo), OtherVal); 3514 } 3515 3516 return SDValue(WideNode, ResNo); 3517} 3518 3519SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { 3520 LLVMContext &Ctx = *DAG.getContext(); 3521 SDValue InOp = N->getOperand(0); 3522 SDLoc DL(N); 3523 3524 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0)); 3525 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3526 3527 EVT InVT = InOp.getValueType(); 3528 3529 unsigned Opcode = N->getOpcode(); 3530 const SDNodeFlags Flags = N->getFlags(); 3531 3532 // Handle the case of ZERO_EXTEND where the promoted InVT element size does 3533 // not equal that of WidenVT. 3534 if (N->getOpcode() == ISD::ZERO_EXTEND && 3535 getTypeAction(InVT) == TargetLowering::TypePromoteInteger && 3536 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() != 3537 WidenVT.getScalarSizeInBits()) { 3538 InOp = ZExtPromotedInteger(InOp); 3539 InVT = InOp.getValueType(); 3540 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits()) 3541 Opcode = ISD::TRUNCATE; 3542 } 3543 3544 EVT InEltVT = InVT.getVectorElementType(); 3545 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenNumElts); 3546 unsigned InVTNumElts = InVT.getVectorNumElements(); 3547 3548 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 3549 InOp = GetWidenedVector(N->getOperand(0)); 3550 InVT = InOp.getValueType(); 3551 InVTNumElts = InVT.getVectorNumElements(); 3552 if (InVTNumElts == WidenNumElts) { 3553 if (N->getNumOperands() == 1) 3554 return DAG.getNode(Opcode, DL, WidenVT, InOp); 3555 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags); 3556 } 3557 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) { 3558 // If both input and result vector types are of same width, extend 3559 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which 3560 // accepts fewer elements in the result than in the input. 3561 if (Opcode == ISD::ANY_EXTEND) 3562 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3563 if (Opcode == ISD::SIGN_EXTEND) 3564 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3565 if (Opcode == ISD::ZERO_EXTEND) 3566 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp); 3567 } 3568 } 3569 3570 if (TLI.isTypeLegal(InWidenVT)) { 3571 // Because the result and the input are different vector types, widening 3572 // the result could create a legal type but widening the input might make 3573 // it an illegal type that might lead to repeatedly splitting the input 3574 // and then widening it. To avoid this, we widen the input only if 3575 // it results in a legal type. 3576 if (WidenNumElts % InVTNumElts == 0) { 3577 // Widen the input and call convert on the widened input vector. 3578 unsigned NumConcat = WidenNumElts/InVTNumElts; 3579 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT)); 3580 Ops[0] = InOp; 3581 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops); 3582 if (N->getNumOperands() == 1) 3583 return DAG.getNode(Opcode, DL, WidenVT, InVec); 3584 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags); 3585 } 3586 3587 if (InVTNumElts % WidenNumElts == 0) { 3588 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp, 3589 DAG.getVectorIdxConstant(0, DL)); 3590 // Extract the input and convert the shorten input vector. 3591 if (N->getNumOperands() == 1) 3592 return DAG.getNode(Opcode, DL, WidenVT, InVal); 3593 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags); 3594 } 3595 } 3596 3597 // Otherwise unroll into some nasty scalar code and rebuild the vector. 3598 EVT EltVT = WidenVT.getVectorElementType(); 3599 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 3600 // Use the original element count so we don't do more scalar opts than 3601 // necessary. 3602 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 3603 for (unsigned i=0; i < MinElts; ++i) { 3604 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 3605 DAG.getVectorIdxConstant(i, DL)); 3606 if (N->getNumOperands() == 1) 3607 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val); 3608 else 3609 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags); 3610 } 3611 3612 return DAG.getBuildVector(WidenVT, DL, Ops); 3613} 3614 3615SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { 3616 SDLoc dl(N); 3617 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3618 ElementCount WidenNumElts = WidenVT.getVectorElementCount(); 3619 3620 SDValue Src = N->getOperand(0); 3621 EVT SrcVT = Src.getValueType(); 3622 3623 // Also widen the input. 3624 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) { 3625 Src = GetWidenedVector(Src); 3626 SrcVT = Src.getValueType(); 3627 } 3628 3629 // Input and output not widened to the same size, give up. 3630 if (WidenNumElts != SrcVT.getVectorElementCount()) 3631 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue()); 3632 3633 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1)); 3634} 3635 3636SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { 3637 SDValue InOp = N->getOperand(1); 3638 SDLoc DL(N); 3639 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 3640 3641 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3642 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3643 3644 EVT InVT = InOp.getValueType(); 3645 EVT InEltVT = InVT.getVectorElementType(); 3646 3647 unsigned Opcode = N->getOpcode(); 3648 3649 // FIXME: Optimizations need to be implemented here. 3650 3651 // Otherwise unroll into some nasty scalar code and rebuild the vector. 3652 EVT EltVT = WidenVT.getVectorElementType(); 3653 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}}; 3654 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT)); 3655 SmallVector<SDValue, 32> OpChains; 3656 // Use the original element count so we don't do more scalar opts than 3657 // necessary. 3658 unsigned MinElts = N->getValueType(0).getVectorNumElements(); 3659 for (unsigned i=0; i < MinElts; ++i) { 3660 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, 3661 DAG.getVectorIdxConstant(i, DL)); 3662 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps); 3663 OpChains.push_back(Ops[i].getValue(1)); 3664 } 3665 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); 3666 ReplaceValueWith(SDValue(N, 1), NewChain); 3667 3668 return DAG.getBuildVector(WidenVT, DL, Ops); 3669} 3670 3671SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { 3672 unsigned Opcode = N->getOpcode(); 3673 SDValue InOp = N->getOperand(0); 3674 SDLoc DL(N); 3675 3676 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3677 EVT WidenSVT = WidenVT.getVectorElementType(); 3678 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3679 3680 EVT InVT = InOp.getValueType(); 3681 EVT InSVT = InVT.getVectorElementType(); 3682 unsigned InVTNumElts = InVT.getVectorNumElements(); 3683 3684 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 3685 InOp = GetWidenedVector(InOp); 3686 InVT = InOp.getValueType(); 3687 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) { 3688 switch (Opcode) { 3689 case ISD::ANY_EXTEND_VECTOR_INREG: 3690 case ISD::SIGN_EXTEND_VECTOR_INREG: 3691 case ISD::ZERO_EXTEND_VECTOR_INREG: 3692 return DAG.getNode(Opcode, DL, WidenVT, InOp); 3693 } 3694 } 3695 } 3696 3697 // Unroll, extend the scalars and rebuild the vector. 3698 SmallVector<SDValue, 16> Ops; 3699 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) { 3700 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp, 3701 DAG.getVectorIdxConstant(i, DL)); 3702 switch (Opcode) { 3703 case ISD::ANY_EXTEND_VECTOR_INREG: 3704 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val); 3705 break; 3706 case ISD::SIGN_EXTEND_VECTOR_INREG: 3707 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val); 3708 break; 3709 case ISD::ZERO_EXTEND_VECTOR_INREG: 3710 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val); 3711 break; 3712 default: 3713 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected"); 3714 } 3715 Ops.push_back(Val); 3716 } 3717 3718 while (Ops.size() != WidenNumElts) 3719 Ops.push_back(DAG.getUNDEF(WidenSVT)); 3720 3721 return DAG.getBuildVector(WidenVT, DL, Ops); 3722} 3723 3724SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { 3725 // If this is an FCOPYSIGN with same input types, we can treat it as a 3726 // normal (can trap) binary op. 3727 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType()) 3728 return WidenVecRes_BinaryCanTrap(N); 3729 3730 // If the types are different, fall back to unrolling. 3731 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3732 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); 3733} 3734 3735SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) { 3736 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3737 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3738 SDValue ShOp = N->getOperand(1); 3739 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp); 3740} 3741 3742SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { 3743 // Unary op widening. 3744 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3745 SDValue InOp = GetWidenedVector(N->getOperand(0)); 3746 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp); 3747} 3748 3749SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { 3750 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3751 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), 3752 cast<VTSDNode>(N->getOperand(1))->getVT() 3753 .getVectorElementType(), 3754 WidenVT.getVectorNumElements()); 3755 SDValue WidenLHS = GetWidenedVector(N->getOperand(0)); 3756 return DAG.getNode(N->getOpcode(), SDLoc(N), 3757 WidenVT, WidenLHS, DAG.getValueType(ExtVT)); 3758} 3759 3760SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { 3761 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo); 3762 return GetWidenedVector(WidenVec); 3763} 3764 3765SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { 3766 SDValue InOp = N->getOperand(0); 3767 EVT InVT = InOp.getValueType(); 3768 EVT VT = N->getValueType(0); 3769 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3770 SDLoc dl(N); 3771 3772 switch (getTypeAction(InVT)) { 3773 case TargetLowering::TypeLegal: 3774 break; 3775 case TargetLowering::TypeScalarizeScalableVector: 3776 report_fatal_error("Scalarization of scalable vectors is not supported."); 3777 case TargetLowering::TypePromoteInteger: { 3778 // If the incoming type is a vector that is being promoted, then 3779 // we know that the elements are arranged differently and that we 3780 // must perform the conversion using a stack slot. 3781 if (InVT.isVector()) 3782 break; 3783 3784 // If the InOp is promoted to the same size, convert it. Otherwise, 3785 // fall out of the switch and widen the promoted input. 3786 SDValue NInOp = GetPromotedInteger(InOp); 3787 EVT NInVT = NInOp.getValueType(); 3788 if (WidenVT.bitsEq(NInVT)) { 3789 // For big endian targets we need to shift the input integer or the 3790 // interesting bits will end up at the wrong place. 3791 if (DAG.getDataLayout().isBigEndian()) { 3792 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits(); 3793 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout()); 3794 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!"); 3795 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp, 3796 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy)); 3797 } 3798 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp); 3799 } 3800 InOp = NInOp; 3801 InVT = NInVT; 3802 break; 3803 } 3804 case TargetLowering::TypeSoftenFloat: 3805 case TargetLowering::TypePromoteFloat: 3806 case TargetLowering::TypeSoftPromoteHalf: 3807 case TargetLowering::TypeExpandInteger: 3808 case TargetLowering::TypeExpandFloat: 3809 case TargetLowering::TypeScalarizeVector: 3810 case TargetLowering::TypeSplitVector: 3811 break; 3812 case TargetLowering::TypeWidenVector: 3813 // If the InOp is widened to the same size, convert it. Otherwise, fall 3814 // out of the switch and widen the widened input. 3815 InOp = GetWidenedVector(InOp); 3816 InVT = InOp.getValueType(); 3817 if (WidenVT.bitsEq(InVT)) 3818 // The input widens to the same size. Convert to the widen value. 3819 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp); 3820 break; 3821 } 3822 3823 unsigned WidenSize = WidenVT.getSizeInBits(); 3824 unsigned InSize = InVT.getSizeInBits(); 3825 // x86mmx is not an acceptable vector element type, so don't try. 3826 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) { 3827 // Determine new input vector type. The new input vector type will use 3828 // the same element type (if its a vector) or use the input type as a 3829 // vector. It is the same size as the type to widen to. 3830 EVT NewInVT; 3831 unsigned NewNumElts = WidenSize / InSize; 3832 if (InVT.isVector()) { 3833 EVT InEltVT = InVT.getVectorElementType(); 3834 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, 3835 WidenSize / InEltVT.getSizeInBits()); 3836 } else { 3837 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts); 3838 } 3839 3840 if (TLI.isTypeLegal(NewInVT)) { 3841 SDValue NewVec; 3842 if (InVT.isVector()) { 3843 // Because the result and the input are different vector types, widening 3844 // the result could create a legal type but widening the input might make 3845 // it an illegal type that might lead to repeatedly splitting the input 3846 // and then widening it. To avoid this, we widen the input only if 3847 // it results in a legal type. 3848 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT)); 3849 Ops[0] = InOp; 3850 3851 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops); 3852 } else { 3853 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp); 3854 } 3855 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec); 3856 } 3857 } 3858 3859 return CreateStackStoreLoad(InOp, WidenVT); 3860} 3861 3862SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { 3863 SDLoc dl(N); 3864 // Build a vector with undefined for the new nodes. 3865 EVT VT = N->getValueType(0); 3866 3867 // Integer BUILD_VECTOR operands may be larger than the node's vector element 3868 // type. The UNDEFs need to have the same type as the existing operands. 3869 EVT EltVT = N->getOperand(0).getValueType(); 3870 unsigned NumElts = VT.getVectorNumElements(); 3871 3872 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3873 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3874 3875 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end()); 3876 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!"); 3877 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT)); 3878 3879 return DAG.getBuildVector(WidenVT, dl, NewOps); 3880} 3881 3882SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { 3883 EVT InVT = N->getOperand(0).getValueType(); 3884 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3885 SDLoc dl(N); 3886 unsigned NumOperands = N->getNumOperands(); 3887 3888 bool InputWidened = false; // Indicates we need to widen the input. 3889 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) { 3890 unsigned WidenNumElts = WidenVT.getVectorMinNumElements(); 3891 unsigned NumInElts = InVT.getVectorMinNumElements(); 3892 if (WidenNumElts % NumInElts == 0) { 3893 // Add undef vectors to widen to correct length. 3894 unsigned NumConcat = WidenNumElts / NumInElts; 3895 SDValue UndefVal = DAG.getUNDEF(InVT); 3896 SmallVector<SDValue, 16> Ops(NumConcat); 3897 for (unsigned i=0; i < NumOperands; ++i) 3898 Ops[i] = N->getOperand(i); 3899 for (unsigned i = NumOperands; i != NumConcat; ++i) 3900 Ops[i] = UndefVal; 3901 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops); 3902 } 3903 } else { 3904 InputWidened = true; 3905 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 3906 // The inputs and the result are widen to the same value. 3907 unsigned i; 3908 for (i=1; i < NumOperands; ++i) 3909 if (!N->getOperand(i).isUndef()) 3910 break; 3911 3912 if (i == NumOperands) 3913 // Everything but the first operand is an UNDEF so just return the 3914 // widened first operand. 3915 return GetWidenedVector(N->getOperand(0)); 3916 3917 if (NumOperands == 2) { 3918 assert(!WidenVT.isScalableVector() && 3919 "Cannot use vector shuffles to widen CONCAT_VECTOR result"); 3920 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3921 unsigned NumInElts = InVT.getVectorNumElements(); 3922 3923 // Replace concat of two operands with a shuffle. 3924 SmallVector<int, 16> MaskOps(WidenNumElts, -1); 3925 for (unsigned i = 0; i < NumInElts; ++i) { 3926 MaskOps[i] = i; 3927 MaskOps[i + NumInElts] = i + WidenNumElts; 3928 } 3929 return DAG.getVectorShuffle(WidenVT, dl, 3930 GetWidenedVector(N->getOperand(0)), 3931 GetWidenedVector(N->getOperand(1)), 3932 MaskOps); 3933 } 3934 } 3935 } 3936 3937 assert(!WidenVT.isScalableVector() && 3938 "Cannot use build vectors to widen CONCAT_VECTOR result"); 3939 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3940 unsigned NumInElts = InVT.getVectorNumElements(); 3941 3942 // Fall back to use extracts and build vector. 3943 EVT EltVT = WidenVT.getVectorElementType(); 3944 SmallVector<SDValue, 16> Ops(WidenNumElts); 3945 unsigned Idx = 0; 3946 for (unsigned i=0; i < NumOperands; ++i) { 3947 SDValue InOp = N->getOperand(i); 3948 if (InputWidened) 3949 InOp = GetWidenedVector(InOp); 3950 for (unsigned j = 0; j < NumInElts; ++j) 3951 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3952 DAG.getVectorIdxConstant(j, dl)); 3953 } 3954 SDValue UndefVal = DAG.getUNDEF(EltVT); 3955 for (; Idx < WidenNumElts; ++Idx) 3956 Ops[Idx] = UndefVal; 3957 return DAG.getBuildVector(WidenVT, dl, Ops); 3958} 3959 3960SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { 3961 EVT VT = N->getValueType(0); 3962 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3963 SDValue InOp = N->getOperand(0); 3964 SDValue Idx = N->getOperand(1); 3965 SDLoc dl(N); 3966 3967 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector) 3968 InOp = GetWidenedVector(InOp); 3969 3970 EVT InVT = InOp.getValueType(); 3971 3972 // Check if we can just return the input vector after widening. 3973 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); 3974 if (IdxVal == 0 && InVT == WidenVT) 3975 return InOp; 3976 3977 if (VT.isScalableVector()) 3978 report_fatal_error("Don't know how to widen the result of " 3979 "EXTRACT_SUBVECTOR for scalable vectors"); 3980 3981 // Check if we can extract from the vector. 3982 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 3983 unsigned InNumElts = InVT.getVectorNumElements(); 3984 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts) 3985 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx); 3986 3987 // We could try widening the input to the right length but for now, extract 3988 // the original elements, fill the rest with undefs and build a vector. 3989 SmallVector<SDValue, 16> Ops(WidenNumElts); 3990 EVT EltVT = VT.getVectorElementType(); 3991 unsigned NumElts = VT.getVectorNumElements(); 3992 unsigned i; 3993 for (i = 0; i < NumElts; ++i) 3994 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 3995 DAG.getVectorIdxConstant(IdxVal + i, dl)); 3996 3997 SDValue UndefVal = DAG.getUNDEF(EltVT); 3998 for (; i < WidenNumElts; ++i) 3999 Ops[i] = UndefVal; 4000 return DAG.getBuildVector(WidenVT, dl, Ops); 4001} 4002 4003SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { 4004 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4005 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), 4006 InOp.getValueType(), InOp, 4007 N->getOperand(1), N->getOperand(2)); 4008} 4009 4010SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { 4011 LoadSDNode *LD = cast<LoadSDNode>(N); 4012 ISD::LoadExtType ExtType = LD->getExtensionType(); 4013 4014 // A vector must always be stored in memory as-is, i.e. without any padding 4015 // between the elements, since various code depend on it, e.g. in the 4016 // handling of a bitcast of a vector type to int, which may be done with a 4017 // vector store followed by an integer load. A vector that does not have 4018 // elements that are byte-sized must therefore be stored as an integer 4019 // built out of the extracted vector elements. 4020 if (!LD->getMemoryVT().isByteSized()) { 4021 SDValue Value, NewChain; 4022 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG); 4023 ReplaceValueWith(SDValue(LD, 0), Value); 4024 ReplaceValueWith(SDValue(LD, 1), NewChain); 4025 return SDValue(); 4026 } 4027 4028 SDValue Result; 4029 SmallVector<SDValue, 16> LdChain; // Chain for the series of load 4030 if (ExtType != ISD::NON_EXTLOAD) 4031 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType); 4032 else 4033 Result = GenWidenVectorLoads(LdChain, LD); 4034 4035 // If we generate a single load, we can use that for the chain. Otherwise, 4036 // build a factor node to remember the multiple loads are independent and 4037 // chain to that. 4038 SDValue NewChain; 4039 if (LdChain.size() == 1) 4040 NewChain = LdChain[0]; 4041 else 4042 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain); 4043 4044 // Modified the chain - switch anything that used the old chain to use 4045 // the new one. 4046 ReplaceValueWith(SDValue(N, 1), NewChain); 4047 4048 return Result; 4049} 4050 4051SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { 4052 4053 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0)); 4054 SDValue Mask = N->getMask(); 4055 EVT MaskVT = Mask.getValueType(); 4056 SDValue PassThru = GetWidenedVector(N->getPassThru()); 4057 ISD::LoadExtType ExtType = N->getExtensionType(); 4058 SDLoc dl(N); 4059 4060 // The mask should be widened as well 4061 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4062 MaskVT.getVectorElementType(), 4063 WidenVT.getVectorNumElements()); 4064 Mask = ModifyToType(Mask, WideMaskVT, true); 4065 4066 SDValue Res = DAG.getMaskedLoad( 4067 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, 4068 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), 4069 ExtType, N->isExpandingLoad()); 4070 // Legalize the chain result - switch anything that used the old chain to 4071 // use the new one. 4072 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4073 return Res; 4074} 4075 4076SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { 4077 4078 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4079 SDValue Mask = N->getMask(); 4080 EVT MaskVT = Mask.getValueType(); 4081 SDValue PassThru = GetWidenedVector(N->getPassThru()); 4082 SDValue Scale = N->getScale(); 4083 unsigned NumElts = WideVT.getVectorNumElements(); 4084 SDLoc dl(N); 4085 4086 // The mask should be widened as well 4087 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4088 MaskVT.getVectorElementType(), 4089 WideVT.getVectorNumElements()); 4090 Mask = ModifyToType(Mask, WideMaskVT, true); 4091 4092 // Widen the Index operand 4093 SDValue Index = N->getIndex(); 4094 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 4095 Index.getValueType().getScalarType(), 4096 NumElts); 4097 Index = ModifyToType(Index, WideIndexVT); 4098 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index, 4099 Scale }; 4100 4101 // Widen the MemoryType 4102 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(), 4103 N->getMemoryVT().getScalarType(), NumElts); 4104 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other), 4105 WideMemVT, dl, Ops, N->getMemOperand(), 4106 N->getIndexType(), N->getExtensionType()); 4107 4108 // Legalize the chain result - switch anything that used the old chain to 4109 // use the new one. 4110 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4111 return Res; 4112} 4113 4114SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) { 4115 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4116 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0)); 4117} 4118 4119// Return true is this is a SETCC node or a strict version of it. 4120static inline bool isSETCCOp(unsigned Opcode) { 4121 switch (Opcode) { 4122 case ISD::SETCC: 4123 case ISD::STRICT_FSETCC: 4124 case ISD::STRICT_FSETCCS: 4125 return true; 4126 } 4127 return false; 4128} 4129 4130// Return true if this is a node that could have two SETCCs as operands. 4131static inline bool isLogicalMaskOp(unsigned Opcode) { 4132 switch (Opcode) { 4133 case ISD::AND: 4134 case ISD::OR: 4135 case ISD::XOR: 4136 return true; 4137 } 4138 return false; 4139} 4140 4141// If N is a SETCC or a strict variant of it, return the type 4142// of the compare operands. 4143static inline EVT getSETCCOperandType(SDValue N) { 4144 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; 4145 return N->getOperand(OpNo).getValueType(); 4146} 4147 4148// This is used just for the assert in convertMask(). Check that this either 4149// a SETCC or a previously handled SETCC by convertMask(). 4150#ifndef NDEBUG 4151static inline bool isSETCCorConvertedSETCC(SDValue N) { 4152 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) 4153 N = N.getOperand(0); 4154 else if (N.getOpcode() == ISD::CONCAT_VECTORS) { 4155 for (unsigned i = 1; i < N->getNumOperands(); ++i) 4156 if (!N->getOperand(i)->isUndef()) 4157 return false; 4158 N = N.getOperand(0); 4159 } 4160 4161 if (N.getOpcode() == ISD::TRUNCATE) 4162 N = N.getOperand(0); 4163 else if (N.getOpcode() == ISD::SIGN_EXTEND) 4164 N = N.getOperand(0); 4165 4166 if (isLogicalMaskOp(N.getOpcode())) 4167 return isSETCCorConvertedSETCC(N.getOperand(0)) && 4168 isSETCCorConvertedSETCC(N.getOperand(1)); 4169 4170 return (isSETCCOp(N.getOpcode()) || 4171 ISD::isBuildVectorOfConstantSDNodes(N.getNode())); 4172} 4173#endif 4174 4175// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT 4176// to ToMaskVT if needed with vector extension or truncation. 4177SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, 4178 EVT ToMaskVT) { 4179 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled. 4180 // FIXME: This code seems to be too restrictive, we might consider 4181 // generalizing it or dropping it. 4182 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument."); 4183 4184 // Make a new Mask node, with a legal result VT. 4185 SDValue Mask; 4186 SmallVector<SDValue, 4> Ops; 4187 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i) 4188 Ops.push_back(InMask->getOperand(i)); 4189 if (InMask->isStrictFPOpcode()) { 4190 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), 4191 { MaskVT, MVT::Other }, Ops); 4192 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1)); 4193 } 4194 else 4195 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops); 4196 4197 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign 4198 // extend or truncate is needed. 4199 LLVMContext &Ctx = *DAG.getContext(); 4200 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits(); 4201 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits(); 4202 if (MaskScalarBits < ToMaskScalBits) { 4203 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 4204 MaskVT.getVectorNumElements()); 4205 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask); 4206 } else if (MaskScalarBits > ToMaskScalBits) { 4207 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(), 4208 MaskVT.getVectorNumElements()); 4209 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask); 4210 } 4211 4212 assert(Mask->getValueType(0).getScalarSizeInBits() == 4213 ToMaskVT.getScalarSizeInBits() && 4214 "Mask should have the right element size by now."); 4215 4216 // Adjust Mask to the right number of elements. 4217 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements(); 4218 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) { 4219 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask)); 4220 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask, 4221 ZeroIdx); 4222 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) { 4223 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls); 4224 EVT SubVT = Mask->getValueType(0); 4225 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT)); 4226 SubOps[0] = Mask; 4227 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps); 4228 } 4229 4230 assert((Mask->getValueType(0) == ToMaskVT) && 4231 "A mask of ToMaskVT should have been produced by now."); 4232 4233 return Mask; 4234} 4235 4236// This method tries to handle some special cases for the vselect mask 4237// and if needed adjusting the mask vector type to match that of the VSELECT. 4238// Without it, many cases end up with scalarization of the SETCC, with many 4239// unnecessary instructions. 4240SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) { 4241 LLVMContext &Ctx = *DAG.getContext(); 4242 SDValue Cond = N->getOperand(0); 4243 4244 if (N->getOpcode() != ISD::VSELECT) 4245 return SDValue(); 4246 4247 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode())) 4248 return SDValue(); 4249 4250 // If this is a splitted VSELECT that was previously already handled, do 4251 // nothing. 4252 EVT CondVT = Cond->getValueType(0); 4253 if (CondVT.getScalarSizeInBits() != 1) 4254 return SDValue(); 4255 4256 EVT VSelVT = N->getValueType(0); 4257 // Only handle vector types which are a power of 2. 4258 if (!isPowerOf2_64(VSelVT.getSizeInBits())) 4259 return SDValue(); 4260 4261 // Don't touch if this will be scalarized. 4262 EVT FinalVT = VSelVT; 4263 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector) 4264 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx); 4265 4266 if (FinalVT.getVectorNumElements() == 1) 4267 return SDValue(); 4268 4269 // If there is support for an i1 vector mask, don't touch. 4270 if (isSETCCOp(Cond.getOpcode())) { 4271 EVT SetCCOpVT = getSETCCOperandType(Cond); 4272 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal) 4273 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT); 4274 EVT SetCCResVT = getSetCCResultType(SetCCOpVT); 4275 if (SetCCResVT.getScalarSizeInBits() == 1) 4276 return SDValue(); 4277 } else if (CondVT.getScalarType() == MVT::i1) { 4278 // If there is support for an i1 vector mask (or only scalar i1 conditions), 4279 // don't touch. 4280 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal) 4281 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT); 4282 4283 if (CondVT.getScalarType() == MVT::i1) 4284 return SDValue(); 4285 } 4286 4287 // Widen the vselect result type if needed. 4288 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) 4289 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT); 4290 4291 // The mask of the VSELECT should have integer elements. 4292 EVT ToMaskVT = VSelVT; 4293 if (!ToMaskVT.getScalarType().isInteger()) 4294 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger(); 4295 4296 SDValue Mask; 4297 if (isSETCCOp(Cond->getOpcode())) { 4298 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond)); 4299 Mask = convertMask(Cond, MaskVT, ToMaskVT); 4300 } else if (isLogicalMaskOp(Cond->getOpcode()) && 4301 isSETCCOp(Cond->getOperand(0).getOpcode()) && 4302 isSETCCOp(Cond->getOperand(1).getOpcode())) { 4303 // Cond is (AND/OR/XOR (SETCC, SETCC)) 4304 SDValue SETCC0 = Cond->getOperand(0); 4305 SDValue SETCC1 = Cond->getOperand(1); 4306 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0)); 4307 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1)); 4308 unsigned ScalarBits0 = VT0.getScalarSizeInBits(); 4309 unsigned ScalarBits1 = VT1.getScalarSizeInBits(); 4310 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits(); 4311 EVT MaskVT; 4312 // If the two SETCCs have different VTs, either extend/truncate one of 4313 // them to the other "towards" ToMaskVT, or truncate one and extend the 4314 // other to ToMaskVT. 4315 if (ScalarBits0 != ScalarBits1) { 4316 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1); 4317 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0); 4318 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits()) 4319 MaskVT = WideVT; 4320 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits()) 4321 MaskVT = NarrowVT; 4322 else 4323 MaskVT = ToMaskVT; 4324 } else 4325 // If the two SETCCs have the same VT, don't change it. 4326 MaskVT = VT0; 4327 4328 // Make new SETCCs and logical nodes. 4329 SETCC0 = convertMask(SETCC0, VT0, MaskVT); 4330 SETCC1 = convertMask(SETCC1, VT1, MaskVT); 4331 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1); 4332 4333 // Convert the logical op for VSELECT if needed. 4334 Mask = convertMask(Cond, MaskVT, ToMaskVT); 4335 } else 4336 return SDValue(); 4337 4338 return Mask; 4339} 4340 4341SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { 4342 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4343 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4344 4345 SDValue Cond1 = N->getOperand(0); 4346 EVT CondVT = Cond1.getValueType(); 4347 if (CondVT.isVector()) { 4348 if (SDValue WideCond = WidenVSELECTMask(N)) { 4349 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 4350 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 4351 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 4352 return DAG.getNode(N->getOpcode(), SDLoc(N), 4353 WidenVT, WideCond, InOp1, InOp2); 4354 } 4355 4356 EVT CondEltVT = CondVT.getVectorElementType(); 4357 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(), 4358 CondEltVT, WidenNumElts); 4359 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector) 4360 Cond1 = GetWidenedVector(Cond1); 4361 4362 // If we have to split the condition there is no point in widening the 4363 // select. This would result in an cycle of widening the select -> 4364 // widening the condition operand -> splitting the condition operand -> 4365 // splitting the select -> widening the select. Instead split this select 4366 // further and widen the resulting type. 4367 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) { 4368 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0); 4369 SDValue Res = ModifyToType(SplitSelect, WidenVT); 4370 return Res; 4371 } 4372 4373 if (Cond1.getValueType() != CondWidenVT) 4374 Cond1 = ModifyToType(Cond1, CondWidenVT); 4375 } 4376 4377 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 4378 SDValue InOp2 = GetWidenedVector(N->getOperand(2)); 4379 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT); 4380 return DAG.getNode(N->getOpcode(), SDLoc(N), 4381 WidenVT, Cond1, InOp1, InOp2); 4382} 4383 4384SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { 4385 SDValue InOp1 = GetWidenedVector(N->getOperand(2)); 4386 SDValue InOp2 = GetWidenedVector(N->getOperand(3)); 4387 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 4388 InOp1.getValueType(), N->getOperand(0), 4389 N->getOperand(1), InOp1, InOp2, N->getOperand(4)); 4390} 4391 4392SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { 4393 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4394 return DAG.getUNDEF(WidenVT); 4395} 4396 4397SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { 4398 EVT VT = N->getValueType(0); 4399 SDLoc dl(N); 4400 4401 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4402 unsigned NumElts = VT.getVectorNumElements(); 4403 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4404 4405 SDValue InOp1 = GetWidenedVector(N->getOperand(0)); 4406 SDValue InOp2 = GetWidenedVector(N->getOperand(1)); 4407 4408 // Adjust mask based on new input vector length. 4409 SmallVector<int, 16> NewMask; 4410 for (unsigned i = 0; i != NumElts; ++i) { 4411 int Idx = N->getMaskElt(i); 4412 if (Idx < (int)NumElts) 4413 NewMask.push_back(Idx); 4414 else 4415 NewMask.push_back(Idx - NumElts + WidenNumElts); 4416 } 4417 for (unsigned i = NumElts; i != WidenNumElts; ++i) 4418 NewMask.push_back(-1); 4419 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask); 4420} 4421 4422SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { 4423 assert(N->getValueType(0).isVector() && 4424 N->getOperand(0).getValueType().isVector() && 4425 "Operands must be vectors"); 4426 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 4427 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4428 4429 SDValue InOp1 = N->getOperand(0); 4430 EVT InVT = InOp1.getValueType(); 4431 assert(InVT.isVector() && "can not widen non-vector type"); 4432 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(), 4433 InVT.getVectorElementType(), WidenNumElts); 4434 4435 // The input and output types often differ here, and it could be that while 4436 // we'd prefer to widen the result type, the input operands have been split. 4437 // In this case, we also need to split the result of this node as well. 4438 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) { 4439 SDValue SplitVSetCC = SplitVecOp_VSETCC(N); 4440 SDValue Res = ModifyToType(SplitVSetCC, WidenVT); 4441 return Res; 4442 } 4443 4444 // If the inputs also widen, handle them directly. Otherwise widen by hand. 4445 SDValue InOp2 = N->getOperand(1); 4446 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { 4447 InOp1 = GetWidenedVector(InOp1); 4448 InOp2 = GetWidenedVector(InOp2); 4449 } else { 4450 InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); 4451 InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); 4452 } 4453 4454 // Assume that the input and output will be widen appropriately. If not, 4455 // we will have to unroll it at some point. 4456 assert(InOp1.getValueType() == WidenInVT && 4457 InOp2.getValueType() == WidenInVT && 4458 "Input not widened to expected type!"); 4459 (void)WidenInVT; 4460 return DAG.getNode(ISD::SETCC, SDLoc(N), 4461 WidenVT, InOp1, InOp2, N->getOperand(2)); 4462} 4463 4464SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { 4465 assert(N->getValueType(0).isVector() && 4466 N->getOperand(1).getValueType().isVector() && 4467 "Operands must be vectors"); 4468 EVT VT = N->getValueType(0); 4469 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 4470 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 4471 unsigned NumElts = VT.getVectorNumElements(); 4472 EVT EltVT = VT.getVectorElementType(); 4473 4474 SDLoc dl(N); 4475 SDValue Chain = N->getOperand(0); 4476 SDValue LHS = N->getOperand(1); 4477 SDValue RHS = N->getOperand(2); 4478 SDValue CC = N->getOperand(3); 4479 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 4480 4481 // Fully unroll and reassemble. 4482 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT)); 4483 SmallVector<SDValue, 8> Chains(NumElts); 4484 for (unsigned i = 0; i != NumElts; ++i) { 4485 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 4486 DAG.getVectorIdxConstant(i, dl)); 4487 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 4488 DAG.getVectorIdxConstant(i, dl)); 4489 4490 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 4491 {Chain, LHSElem, RHSElem, CC}); 4492 Chains[i] = Scalars[i].getValue(1); 4493 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 4494 DAG.getBoolConstant(true, dl, EltVT, VT), 4495 DAG.getBoolConstant(false, dl, EltVT, VT)); 4496 } 4497 4498 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 4499 ReplaceValueWith(SDValue(N, 1), NewChain); 4500 4501 return DAG.getBuildVector(WidenVT, dl, Scalars); 4502} 4503 4504//===----------------------------------------------------------------------===// 4505// Widen Vector Operand 4506//===----------------------------------------------------------------------===// 4507bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { 4508 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG); 4509 dbgs() << "\n"); 4510 SDValue Res = SDValue(); 4511 4512 // See if the target wants to custom widen this node. 4513 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 4514 return false; 4515 4516 switch (N->getOpcode()) { 4517 default: 4518#ifndef NDEBUG 4519 dbgs() << "WidenVectorOperand op #" << OpNo << ": "; 4520 N->dump(&DAG); 4521 dbgs() << "\n"; 4522#endif 4523 llvm_unreachable("Do not know how to widen this operator's operand!"); 4524 4525 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break; 4526 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break; 4527 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break; 4528 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break; 4529 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break; 4530 case ISD::STORE: Res = WidenVecOp_STORE(N); break; 4531 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break; 4532 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; 4533 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; 4534 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; 4535 case ISD::STRICT_FSETCC: 4536 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break; 4537 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; 4538 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; 4539 4540 case ISD::ANY_EXTEND: 4541 case ISD::SIGN_EXTEND: 4542 case ISD::ZERO_EXTEND: 4543 Res = WidenVecOp_EXTEND(N); 4544 break; 4545 4546 case ISD::FP_EXTEND: 4547 case ISD::STRICT_FP_EXTEND: 4548 case ISD::FP_ROUND: 4549 case ISD::STRICT_FP_ROUND: 4550 case ISD::FP_TO_SINT: 4551 case ISD::STRICT_FP_TO_SINT: 4552 case ISD::FP_TO_UINT: 4553 case ISD::STRICT_FP_TO_UINT: 4554 case ISD::SINT_TO_FP: 4555 case ISD::STRICT_SINT_TO_FP: 4556 case ISD::UINT_TO_FP: 4557 case ISD::STRICT_UINT_TO_FP: 4558 case ISD::TRUNCATE: 4559 Res = WidenVecOp_Convert(N); 4560 break; 4561 4562 case ISD::FP_TO_SINT_SAT: 4563 case ISD::FP_TO_UINT_SAT: 4564 Res = WidenVecOp_FP_TO_XINT_SAT(N); 4565 break; 4566 4567 case ISD::VECREDUCE_FADD: 4568 case ISD::VECREDUCE_FMUL: 4569 case ISD::VECREDUCE_ADD: 4570 case ISD::VECREDUCE_MUL: 4571 case ISD::VECREDUCE_AND: 4572 case ISD::VECREDUCE_OR: 4573 case ISD::VECREDUCE_XOR: 4574 case ISD::VECREDUCE_SMAX: 4575 case ISD::VECREDUCE_SMIN: 4576 case ISD::VECREDUCE_UMAX: 4577 case ISD::VECREDUCE_UMIN: 4578 case ISD::VECREDUCE_FMAX: 4579 case ISD::VECREDUCE_FMIN: 4580 Res = WidenVecOp_VECREDUCE(N); 4581 break; 4582 case ISD::VECREDUCE_SEQ_FADD: 4583 case ISD::VECREDUCE_SEQ_FMUL: 4584 Res = WidenVecOp_VECREDUCE_SEQ(N); 4585 break; 4586 } 4587 4588 // If Res is null, the sub-method took care of registering the result. 4589 if (!Res.getNode()) return false; 4590 4591 // If the result is N, the sub-method updated N in place. Tell the legalizer 4592 // core about this. 4593 if (Res.getNode() == N) 4594 return true; 4595 4596 4597 if (N->isStrictFPOpcode()) 4598 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && 4599 "Invalid operand expansion"); 4600 else 4601 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 4602 "Invalid operand expansion"); 4603 4604 ReplaceValueWith(SDValue(N, 0), Res); 4605 return false; 4606} 4607 4608SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { 4609 SDLoc DL(N); 4610 EVT VT = N->getValueType(0); 4611 4612 SDValue InOp = N->getOperand(0); 4613 assert(getTypeAction(InOp.getValueType()) == 4614 TargetLowering::TypeWidenVector && 4615 "Unexpected type action"); 4616 InOp = GetWidenedVector(InOp); 4617 assert(VT.getVectorNumElements() < 4618 InOp.getValueType().getVectorNumElements() && 4619 "Input wasn't widened!"); 4620 4621 // We may need to further widen the operand until it has the same total 4622 // vector size as the result. 4623 EVT InVT = InOp.getValueType(); 4624 if (InVT.getSizeInBits() != VT.getSizeInBits()) { 4625 EVT InEltVT = InVT.getVectorElementType(); 4626 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) { 4627 EVT FixedVT = (MVT::SimpleValueType)i; 4628 EVT FixedEltVT = FixedVT.getVectorElementType(); 4629 if (TLI.isTypeLegal(FixedVT) && 4630 FixedVT.getSizeInBits() == VT.getSizeInBits() && 4631 FixedEltVT == InEltVT) { 4632 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() && 4633 "Not enough elements in the fixed type for the operand!"); 4634 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() && 4635 "We can't have the same type as we started with!"); 4636 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements()) 4637 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT, 4638 DAG.getUNDEF(FixedVT), InOp, 4639 DAG.getVectorIdxConstant(0, DL)); 4640 else 4641 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp, 4642 DAG.getVectorIdxConstant(0, DL)); 4643 break; 4644 } 4645 } 4646 InVT = InOp.getValueType(); 4647 if (InVT.getSizeInBits() != VT.getSizeInBits()) 4648 // We couldn't find a legal vector type that was a widening of the input 4649 // and could be extended in-register to the result type, so we have to 4650 // scalarize. 4651 return WidenVecOp_Convert(N); 4652 } 4653 4654 // Use special DAG nodes to represent the operation of extending the 4655 // low lanes. 4656 switch (N->getOpcode()) { 4657 default: 4658 llvm_unreachable("Extend legalization on extend operation!"); 4659 case ISD::ANY_EXTEND: 4660 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp); 4661 case ISD::SIGN_EXTEND: 4662 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp); 4663 case ISD::ZERO_EXTEND: 4664 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp); 4665 } 4666} 4667 4668SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) { 4669 // The result (and first input) is legal, but the second input is illegal. 4670 // We can't do much to fix that, so just unroll and let the extracts off of 4671 // the second input be widened as needed later. 4672 return DAG.UnrollVectorOp(N); 4673} 4674 4675SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { 4676 // Since the result is legal and the input is illegal. 4677 EVT VT = N->getValueType(0); 4678 EVT EltVT = VT.getVectorElementType(); 4679 SDLoc dl(N); 4680 unsigned NumElts = VT.getVectorNumElements(); 4681 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); 4682 assert(getTypeAction(InOp.getValueType()) == 4683 TargetLowering::TypeWidenVector && 4684 "Unexpected type action"); 4685 InOp = GetWidenedVector(InOp); 4686 EVT InVT = InOp.getValueType(); 4687 unsigned Opcode = N->getOpcode(); 4688 4689 // See if a widened result type would be legal, if so widen the node. 4690 // FIXME: This isn't safe for StrictFP. Other optimization here is needed. 4691 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 4692 InVT.getVectorNumElements()); 4693 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) { 4694 SDValue Res; 4695 if (N->isStrictFPOpcode()) { 4696 if (Opcode == ISD::STRICT_FP_ROUND) 4697 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 4698 { N->getOperand(0), InOp, N->getOperand(2) }); 4699 else 4700 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 4701 { N->getOperand(0), InOp }); 4702 // Legalize the chain result - switch anything that used the old chain to 4703 // use the new one. 4704 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4705 } else { 4706 if (Opcode == ISD::FP_ROUND) 4707 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1)); 4708 else 4709 Res = DAG.getNode(Opcode, dl, WideVT, InOp); 4710 } 4711 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, 4712 DAG.getVectorIdxConstant(0, dl)); 4713 } 4714 4715 EVT InEltVT = InVT.getVectorElementType(); 4716 4717 // Unroll the convert into some scalar code and create a nasty build vector. 4718 SmallVector<SDValue, 16> Ops(NumElts); 4719 if (N->isStrictFPOpcode()) { 4720 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); 4721 SmallVector<SDValue, 32> OpChains; 4722 for (unsigned i=0; i < NumElts; ++i) { 4723 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, 4724 DAG.getVectorIdxConstant(i, dl)); 4725 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); 4726 OpChains.push_back(Ops[i].getValue(1)); 4727 } 4728 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); 4729 ReplaceValueWith(SDValue(N, 1), NewChain); 4730 } else { 4731 for (unsigned i = 0; i < NumElts; ++i) 4732 Ops[i] = DAG.getNode(Opcode, dl, EltVT, 4733 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, 4734 InOp, DAG.getVectorIdxConstant(i, dl))); 4735 } 4736 4737 return DAG.getBuildVector(VT, dl, Ops); 4738} 4739 4740SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { 4741 EVT DstVT = N->getValueType(0); 4742 SDValue Src = GetWidenedVector(N->getOperand(0)); 4743 EVT SrcVT = Src.getValueType(); 4744 ElementCount WideNumElts = SrcVT.getVectorElementCount(); 4745 SDLoc dl(N); 4746 4747 // See if a widened result type would be legal, if so widen the node. 4748 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(), 4749 DstVT.getVectorElementType(), WideNumElts); 4750 if (TLI.isTypeLegal(WideDstVT)) { 4751 SDValue Res = 4752 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1)); 4753 return DAG.getNode( 4754 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res, 4755 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); 4756 } 4757 4758 // Give up and unroll. 4759 return DAG.UnrollVectorOp(N); 4760} 4761 4762SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { 4763 EVT VT = N->getValueType(0); 4764 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4765 EVT InWidenVT = InOp.getValueType(); 4766 SDLoc dl(N); 4767 4768 // Check if we can convert between two legal vector types and extract. 4769 unsigned InWidenSize = InWidenVT.getSizeInBits(); 4770 unsigned Size = VT.getSizeInBits(); 4771 // x86mmx is not an acceptable vector element type, so don't try. 4772 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) { 4773 unsigned NewNumElts = InWidenSize / Size; 4774 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts); 4775 if (TLI.isTypeLegal(NewVT)) { 4776 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 4777 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp, 4778 DAG.getVectorIdxConstant(0, dl)); 4779 } 4780 } 4781 4782 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened 4783 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, 4784 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids 4785 // having to copy via memory. 4786 if (VT.isVector()) { 4787 EVT EltVT = VT.getVectorElementType(); 4788 unsigned EltSize = EltVT.getSizeInBits(); 4789 if (InWidenSize % EltSize == 0) { 4790 unsigned NewNumElts = InWidenSize / EltSize; 4791 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts); 4792 if (TLI.isTypeLegal(NewVT)) { 4793 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); 4794 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp, 4795 DAG.getVectorIdxConstant(0, dl)); 4796 } 4797 } 4798 } 4799 4800 return CreateStackStoreLoad(InOp, VT); 4801} 4802 4803SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { 4804 EVT VT = N->getValueType(0); 4805 EVT EltVT = VT.getVectorElementType(); 4806 EVT InVT = N->getOperand(0).getValueType(); 4807 SDLoc dl(N); 4808 4809 // If the widen width for this operand is the same as the width of the concat 4810 // and all but the first operand is undef, just use the widened operand. 4811 unsigned NumOperands = N->getNumOperands(); 4812 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) { 4813 unsigned i; 4814 for (i = 1; i < NumOperands; ++i) 4815 if (!N->getOperand(i).isUndef()) 4816 break; 4817 4818 if (i == NumOperands) 4819 return GetWidenedVector(N->getOperand(0)); 4820 } 4821 4822 // Otherwise, fall back to a nasty build vector. 4823 unsigned NumElts = VT.getVectorNumElements(); 4824 SmallVector<SDValue, 16> Ops(NumElts); 4825 4826 unsigned NumInElts = InVT.getVectorNumElements(); 4827 4828 unsigned Idx = 0; 4829 for (unsigned i=0; i < NumOperands; ++i) { 4830 SDValue InOp = N->getOperand(i); 4831 assert(getTypeAction(InOp.getValueType()) == 4832 TargetLowering::TypeWidenVector && 4833 "Unexpected type action"); 4834 InOp = GetWidenedVector(InOp); 4835 for (unsigned j = 0; j < NumInElts; ++j) 4836 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 4837 DAG.getVectorIdxConstant(j, dl)); 4838 } 4839 return DAG.getBuildVector(VT, dl, Ops); 4840} 4841 4842SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) { 4843 SDValue SubVec = N->getOperand(1); 4844 SDValue InVec = N->getOperand(0); 4845 4846 if (getTypeAction(InVec.getValueType()) == TargetLowering::TypeWidenVector) 4847 InVec = GetWidenedVector(InVec); 4848 4849 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector) 4850 SubVec = GetWidenedVector(SubVec); 4851 4852 if (SubVec.getValueType() == InVec.getValueType() && InVec.isUndef() && 4853 N->getConstantOperandVal(2) == 0) 4854 return SubVec; 4855 4856 report_fatal_error("Don't know how to widen the operands for " 4857 "INSERT_SUBVECTOR"); 4858} 4859 4860SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { 4861 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4862 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), 4863 N->getValueType(0), InOp, N->getOperand(1)); 4864} 4865 4866SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { 4867 SDValue InOp = GetWidenedVector(N->getOperand(0)); 4868 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 4869 N->getValueType(0), InOp, N->getOperand(1)); 4870} 4871 4872SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { 4873 // We have to widen the value, but we want only to store the original 4874 // vector type. 4875 StoreSDNode *ST = cast<StoreSDNode>(N); 4876 4877 if (!ST->getMemoryVT().getScalarType().isByteSized()) 4878 return TLI.scalarizeVectorStore(ST, DAG); 4879 4880 if (ST->isTruncatingStore()) 4881 return TLI.scalarizeVectorStore(ST, DAG); 4882 4883 SmallVector<SDValue, 16> StChain; 4884 GenWidenVectorStores(StChain, ST); 4885 4886 if (StChain.size() == 1) 4887 return StChain[0]; 4888 else 4889 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain); 4890} 4891 4892SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { 4893 assert((OpNo == 1 || OpNo == 3) && 4894 "Can widen only data or mask operand of mstore"); 4895 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); 4896 SDValue Mask = MST->getMask(); 4897 EVT MaskVT = Mask.getValueType(); 4898 SDValue StVal = MST->getValue(); 4899 SDLoc dl(N); 4900 4901 if (OpNo == 1) { 4902 // Widen the value. 4903 StVal = GetWidenedVector(StVal); 4904 4905 // The mask should be widened as well. 4906 EVT WideVT = StVal.getValueType(); 4907 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4908 MaskVT.getVectorElementType(), 4909 WideVT.getVectorNumElements()); 4910 Mask = ModifyToType(Mask, WideMaskVT, true); 4911 } else { 4912 // Widen the mask. 4913 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT); 4914 Mask = ModifyToType(Mask, WideMaskVT, true); 4915 4916 EVT ValueVT = StVal.getValueType(); 4917 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), 4918 ValueVT.getVectorElementType(), 4919 WideMaskVT.getVectorNumElements()); 4920 StVal = ModifyToType(StVal, WideVT); 4921 } 4922 4923 assert(Mask.getValueType().getVectorNumElements() == 4924 StVal.getValueType().getVectorNumElements() && 4925 "Mask and data vectors should have the same number of elements"); 4926 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(), 4927 MST->getOffset(), Mask, MST->getMemoryVT(), 4928 MST->getMemOperand(), MST->getAddressingMode(), 4929 false, MST->isCompressingStore()); 4930} 4931 4932SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { 4933 assert(OpNo == 4 && "Can widen only the index of mgather"); 4934 auto *MG = cast<MaskedGatherSDNode>(N); 4935 SDValue DataOp = MG->getPassThru(); 4936 SDValue Mask = MG->getMask(); 4937 SDValue Scale = MG->getScale(); 4938 4939 // Just widen the index. It's allowed to have extra elements. 4940 SDValue Index = GetWidenedVector(MG->getIndex()); 4941 4942 SDLoc dl(N); 4943 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index, 4944 Scale}; 4945 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops, 4946 MG->getMemOperand(), MG->getIndexType(), 4947 MG->getExtensionType()); 4948 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 4949 ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); 4950 return SDValue(); 4951} 4952 4953SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { 4954 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N); 4955 SDValue DataOp = MSC->getValue(); 4956 SDValue Mask = MSC->getMask(); 4957 SDValue Index = MSC->getIndex(); 4958 SDValue Scale = MSC->getScale(); 4959 EVT WideMemVT = MSC->getMemoryVT(); 4960 4961 if (OpNo == 1) { 4962 DataOp = GetWidenedVector(DataOp); 4963 unsigned NumElts = DataOp.getValueType().getVectorNumElements(); 4964 4965 // Widen index. 4966 EVT IndexVT = Index.getValueType(); 4967 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(), 4968 IndexVT.getVectorElementType(), NumElts); 4969 Index = ModifyToType(Index, WideIndexVT); 4970 4971 // The mask should be widened as well. 4972 EVT MaskVT = Mask.getValueType(); 4973 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(), 4974 MaskVT.getVectorElementType(), NumElts); 4975 Mask = ModifyToType(Mask, WideMaskVT, true); 4976 4977 // Widen the MemoryType 4978 WideMemVT = EVT::getVectorVT(*DAG.getContext(), 4979 MSC->getMemoryVT().getScalarType(), NumElts); 4980 } else if (OpNo == 4) { 4981 // Just widen the index. It's allowed to have extra elements. 4982 Index = GetWidenedVector(Index); 4983 } else 4984 llvm_unreachable("Can't widen this operand of mscatter"); 4985 4986 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index, 4987 Scale}; 4988 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N), 4989 Ops, MSC->getMemOperand(), MSC->getIndexType(), 4990 MSC->isTruncatingStore()); 4991} 4992 4993SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { 4994 SDValue InOp0 = GetWidenedVector(N->getOperand(0)); 4995 SDValue InOp1 = GetWidenedVector(N->getOperand(1)); 4996 SDLoc dl(N); 4997 EVT VT = N->getValueType(0); 4998 4999 // WARNING: In this code we widen the compare instruction with garbage. 5000 // This garbage may contain denormal floats which may be slow. Is this a real 5001 // concern ? Should we zero the unused lanes if this is a float compare ? 5002 5003 // Get a new SETCC node to compare the newly widened operands. 5004 // Only some of the compared elements are legal. 5005 EVT SVT = getSetCCResultType(InOp0.getValueType()); 5006 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. 5007 if (VT.getScalarType() == MVT::i1) 5008 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 5009 SVT.getVectorNumElements()); 5010 5011 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N), 5012 SVT, InOp0, InOp1, N->getOperand(2)); 5013 5014 // Extract the needed results from the result vector. 5015 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), 5016 SVT.getVectorElementType(), 5017 VT.getVectorNumElements()); 5018 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC, 5019 DAG.getVectorIdxConstant(0, dl)); 5020 5021 EVT OpVT = N->getOperand(0).getValueType(); 5022 ISD::NodeType ExtendCode = 5023 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT)); 5024 return DAG.getNode(ExtendCode, dl, VT, CC); 5025} 5026 5027SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { 5028 SDValue Chain = N->getOperand(0); 5029 SDValue LHS = GetWidenedVector(N->getOperand(1)); 5030 SDValue RHS = GetWidenedVector(N->getOperand(2)); 5031 SDValue CC = N->getOperand(3); 5032 SDLoc dl(N); 5033 5034 EVT VT = N->getValueType(0); 5035 EVT EltVT = VT.getVectorElementType(); 5036 EVT TmpEltVT = LHS.getValueType().getVectorElementType(); 5037 unsigned NumElts = VT.getVectorNumElements(); 5038 5039 // Unroll into a build vector. 5040 SmallVector<SDValue, 8> Scalars(NumElts); 5041 SmallVector<SDValue, 8> Chains(NumElts); 5042 5043 for (unsigned i = 0; i != NumElts; ++i) { 5044 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, 5045 DAG.getVectorIdxConstant(i, dl)); 5046 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, 5047 DAG.getVectorIdxConstant(i, dl)); 5048 5049 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other}, 5050 {Chain, LHSElem, RHSElem, CC}); 5051 Chains[i] = Scalars[i].getValue(1); 5052 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i], 5053 DAG.getBoolConstant(true, dl, EltVT, VT), 5054 DAG.getBoolConstant(false, dl, EltVT, VT)); 5055 } 5056 5057 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 5058 ReplaceValueWith(SDValue(N, 1), NewChain); 5059 5060 return DAG.getBuildVector(VT, dl, Scalars); 5061} 5062 5063SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { 5064 SDLoc dl(N); 5065 SDValue Op = GetWidenedVector(N->getOperand(0)); 5066 EVT OrigVT = N->getOperand(0).getValueType(); 5067 EVT WideVT = Op.getValueType(); 5068 EVT ElemVT = OrigVT.getVectorElementType(); 5069 SDNodeFlags Flags = N->getFlags(); 5070 5071 unsigned Opc = N->getOpcode(); 5072 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 5073 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 5074 assert(NeutralElem && "Neutral element must exist"); 5075 5076 // Pad the vector with the neutral element. 5077 unsigned OrigElts = OrigVT.getVectorNumElements(); 5078 unsigned WideElts = WideVT.getVectorNumElements(); 5079 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 5080 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 5081 DAG.getVectorIdxConstant(Idx, dl)); 5082 5083 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags); 5084} 5085 5086SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) { 5087 SDLoc dl(N); 5088 SDValue AccOp = N->getOperand(0); 5089 SDValue VecOp = N->getOperand(1); 5090 SDValue Op = GetWidenedVector(VecOp); 5091 5092 EVT OrigVT = VecOp.getValueType(); 5093 EVT WideVT = Op.getValueType(); 5094 EVT ElemVT = OrigVT.getVectorElementType(); 5095 SDNodeFlags Flags = N->getFlags(); 5096 5097 unsigned Opc = N->getOpcode(); 5098 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc); 5099 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags); 5100 5101 // Pad the vector with the neutral element. 5102 unsigned OrigElts = OrigVT.getVectorNumElements(); 5103 unsigned WideElts = WideVT.getVectorNumElements(); 5104 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) 5105 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, 5106 DAG.getVectorIdxConstant(Idx, dl)); 5107 5108 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags); 5109} 5110 5111SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { 5112 // This only gets called in the case that the left and right inputs and 5113 // result are of a legal odd vector type, and the condition is illegal i1 of 5114 // the same odd width that needs widening. 5115 EVT VT = N->getValueType(0); 5116 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); 5117 5118 SDValue Cond = GetWidenedVector(N->getOperand(0)); 5119 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); 5120 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); 5121 SDLoc DL(N); 5122 5123 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, 5124 LeftIn, RightIn); 5125 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select, 5126 DAG.getVectorIdxConstant(0, DL)); 5127} 5128 5129//===----------------------------------------------------------------------===// 5130// Vector Widening Utilities 5131//===----------------------------------------------------------------------===// 5132 5133// Utility function to find the type to chop up a widen vector for load/store 5134// TLI: Target lowering used to determine legal types. 5135// Width: Width left need to load/store. 5136// WidenVT: The widen vector type to load to/store from 5137// Align: If 0, don't allow use of a wider type 5138// WidenEx: If Align is not 0, the amount additional we can load/store from. 5139 5140static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, 5141 unsigned Width, EVT WidenVT, 5142 unsigned Align = 0, unsigned WidenEx = 0) { 5143 EVT WidenEltVT = WidenVT.getVectorElementType(); 5144 const bool Scalable = WidenVT.isScalableVector(); 5145 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize(); 5146 unsigned WidenEltWidth = WidenEltVT.getSizeInBits(); 5147 unsigned AlignInBits = Align*8; 5148 5149 // If we have one element to load/store, return it. 5150 EVT RetVT = WidenEltVT; 5151 if (!Scalable && Width == WidenEltWidth) 5152 return RetVT; 5153 5154 // See if there is larger legal integer than the element type to load/store. 5155 unsigned VT; 5156 // Don't bother looking for an integer type if the vector is scalable, skip 5157 // to vector types. 5158 if (!Scalable) { 5159 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE; 5160 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) { 5161 EVT MemVT((MVT::SimpleValueType) VT); 5162 unsigned MemVTWidth = MemVT.getSizeInBits(); 5163 if (MemVT.getSizeInBits() <= WidenEltWidth) 5164 break; 5165 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 5166 if ((Action == TargetLowering::TypeLegal || 5167 Action == TargetLowering::TypePromoteInteger) && 5168 (WidenWidth % MemVTWidth) == 0 && 5169 isPowerOf2_32(WidenWidth / MemVTWidth) && 5170 (MemVTWidth <= Width || 5171 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 5172 if (MemVTWidth == WidenWidth) 5173 return MemVT; 5174 RetVT = MemVT; 5175 break; 5176 } 5177 } 5178 } 5179 5180 // See if there is a larger vector type to load/store that has the same vector 5181 // element type and is evenly divisible with the WidenVT. 5182 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 5183 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { 5184 EVT MemVT = (MVT::SimpleValueType) VT; 5185 // Skip vector MVTs which don't match the scalable property of WidenVT. 5186 if (Scalable != MemVT.isScalableVector()) 5187 continue; 5188 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize(); 5189 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); 5190 if ((Action == TargetLowering::TypeLegal || 5191 Action == TargetLowering::TypePromoteInteger) && 5192 WidenEltVT == MemVT.getVectorElementType() && 5193 (WidenWidth % MemVTWidth) == 0 && 5194 isPowerOf2_32(WidenWidth / MemVTWidth) && 5195 (MemVTWidth <= Width || 5196 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { 5197 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT) 5198 return MemVT; 5199 } 5200 } 5201 5202 if (Scalable) 5203 report_fatal_error("Using element-wise loads and stores for widening " 5204 "operations is not supported for scalable vectors"); 5205 return RetVT; 5206} 5207 5208// Builds a vector type from scalar loads 5209// VecTy: Resulting Vector type 5210// LDOps: Load operators to build a vector type 5211// [Start,End) the list of loads to use. 5212static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, 5213 SmallVectorImpl<SDValue> &LdOps, 5214 unsigned Start, unsigned End) { 5215 SDLoc dl(LdOps[Start]); 5216 EVT LdTy = LdOps[Start].getValueType(); 5217 unsigned Width = VecTy.getSizeInBits(); 5218 unsigned NumElts = Width / LdTy.getSizeInBits(); 5219 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts); 5220 5221 unsigned Idx = 1; 5222 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]); 5223 5224 for (unsigned i = Start + 1; i != End; ++i) { 5225 EVT NewLdTy = LdOps[i].getValueType(); 5226 if (NewLdTy != LdTy) { 5227 NumElts = Width / NewLdTy.getSizeInBits(); 5228 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts); 5229 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp); 5230 // Readjust position and vector position based on new load type. 5231 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits(); 5232 LdTy = NewLdTy; 5233 } 5234 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i], 5235 DAG.getVectorIdxConstant(Idx++, dl)); 5236 } 5237 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp); 5238} 5239 5240SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, 5241 LoadSDNode *LD) { 5242 // The strategy assumes that we can efficiently load power-of-two widths. 5243 // The routine chops the vector into the largest vector loads with the same 5244 // element type or scalar loads and then recombines it to the widen vector 5245 // type. 5246 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 5247 EVT LdVT = LD->getMemoryVT(); 5248 SDLoc dl(LD); 5249 assert(LdVT.isVector() && WidenVT.isVector()); 5250 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 5251 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType()); 5252 5253 // Load information 5254 SDValue Chain = LD->getChain(); 5255 SDValue BasePtr = LD->getBasePtr(); 5256 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 5257 AAMDNodes AAInfo = LD->getAAInfo(); 5258 5259 TypeSize LdWidth = LdVT.getSizeInBits(); 5260 TypeSize WidenWidth = WidenVT.getSizeInBits(); 5261 TypeSize WidthDiff = WidenWidth - LdWidth; 5262 // Allow wider loads if they are sufficiently aligned to avoid memory faults 5263 // and if the original load is simple. 5264 unsigned LdAlign = (!LD->isSimple()) ? 0 : LD->getAlignment(); 5265 5266 // Find the vector type that can load from. 5267 EVT NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign, 5268 WidthDiff.getKnownMinSize()); 5269 TypeSize NewVTWidth = NewVT.getSizeInBits(); 5270 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(), 5271 LD->getOriginalAlign(), MMOFlags, AAInfo); 5272 LdChain.push_back(LdOp.getValue(1)); 5273 5274 // Check if we can load the element with one instruction. 5275 if (TypeSize::isKnownLE(LdWidth, NewVTWidth)) { 5276 if (!NewVT.isVector()) { 5277 unsigned NumElts = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize(); 5278 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 5279 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp); 5280 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp); 5281 } 5282 if (NewVT == WidenVT) 5283 return LdOp; 5284 5285 // TODO: We don't currently have any tests that exercise this code path. 5286 assert(WidenWidth.getFixedSize() % NewVTWidth.getFixedSize() == 0); 5287 unsigned NumConcat = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize(); 5288 SmallVector<SDValue, 16> ConcatOps(NumConcat); 5289 SDValue UndefVal = DAG.getUNDEF(NewVT); 5290 ConcatOps[0] = LdOp; 5291 for (unsigned i = 1; i != NumConcat; ++i) 5292 ConcatOps[i] = UndefVal; 5293 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps); 5294 } 5295 5296 // Load vector by using multiple loads from largest vector to scalar. 5297 SmallVector<SDValue, 16> LdOps; 5298 LdOps.push_back(LdOp); 5299 5300 uint64_t ScaledOffset = 0; 5301 MachinePointerInfo MPI = LD->getPointerInfo(); 5302 do { 5303 LdWidth -= NewVTWidth; 5304 IncrementPointer(cast<LoadSDNode>(LdOp), NewVT, MPI, BasePtr, 5305 &ScaledOffset); 5306 5307 if (TypeSize::isKnownLT(LdWidth, NewVTWidth)) { 5308 // The current type we are using is too large. Find a better size. 5309 NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign, 5310 WidthDiff.getKnownMinSize()); 5311 NewVTWidth = NewVT.getSizeInBits(); 5312 } 5313 5314 Align NewAlign = ScaledOffset == 0 5315 ? LD->getOriginalAlign() 5316 : commonAlignment(LD->getAlign(), ScaledOffset); 5317 SDValue L = 5318 DAG.getLoad(NewVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo); 5319 LdChain.push_back(L.getValue(1)); 5320 5321 LdOps.push_back(L); 5322 LdOp = L; 5323 } while (TypeSize::isKnownGT(LdWidth, NewVTWidth)); 5324 5325 // Build the vector from the load operations. 5326 unsigned End = LdOps.size(); 5327 if (!LdOps[0].getValueType().isVector()) 5328 // All the loads are scalar loads. 5329 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End); 5330 5331 // If the load contains vectors, build the vector using concat vector. 5332 // All of the vectors used to load are power-of-2, and the scalar loads can be 5333 // combined to make a power-of-2 vector. 5334 SmallVector<SDValue, 16> ConcatOps(End); 5335 int i = End - 1; 5336 int Idx = End; 5337 EVT LdTy = LdOps[i].getValueType(); 5338 // First, combine the scalar loads to a vector. 5339 if (!LdTy.isVector()) { 5340 for (--i; i >= 0; --i) { 5341 LdTy = LdOps[i].getValueType(); 5342 if (LdTy.isVector()) 5343 break; 5344 } 5345 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End); 5346 } 5347 5348 ConcatOps[--Idx] = LdOps[i]; 5349 for (--i; i >= 0; --i) { 5350 EVT NewLdTy = LdOps[i].getValueType(); 5351 if (NewLdTy != LdTy) { 5352 // Create a larger vector. 5353 TypeSize LdTySize = LdTy.getSizeInBits(); 5354 TypeSize NewLdTySize = NewLdTy.getSizeInBits(); 5355 assert(NewLdTySize.isScalable() == LdTySize.isScalable() && 5356 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize())); 5357 unsigned NumOps = 5358 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize(); 5359 SmallVector<SDValue, 16> WidenOps(NumOps); 5360 unsigned j = 0; 5361 for (; j != End-Idx; ++j) 5362 WidenOps[j] = ConcatOps[Idx+j]; 5363 for (; j != NumOps; ++j) 5364 WidenOps[j] = DAG.getUNDEF(LdTy); 5365 5366 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy, 5367 WidenOps); 5368 Idx = End - 1; 5369 LdTy = NewLdTy; 5370 } 5371 ConcatOps[--Idx] = LdOps[i]; 5372 } 5373 5374 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx)) 5375 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, 5376 makeArrayRef(&ConcatOps[Idx], End - Idx)); 5377 5378 // We need to fill the rest with undefs to build the vector. 5379 unsigned NumOps = 5380 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize(); 5381 SmallVector<SDValue, 16> WidenOps(NumOps); 5382 SDValue UndefVal = DAG.getUNDEF(LdTy); 5383 { 5384 unsigned i = 0; 5385 for (; i != End-Idx; ++i) 5386 WidenOps[i] = ConcatOps[Idx+i]; 5387 for (; i != NumOps; ++i) 5388 WidenOps[i] = UndefVal; 5389 } 5390 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps); 5391} 5392 5393SDValue 5394DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, 5395 LoadSDNode *LD, 5396 ISD::LoadExtType ExtType) { 5397 // For extension loads, it may not be more efficient to chop up the vector 5398 // and then extend it. Instead, we unroll the load and build a new vector. 5399 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0)); 5400 EVT LdVT = LD->getMemoryVT(); 5401 SDLoc dl(LD); 5402 assert(LdVT.isVector() && WidenVT.isVector()); 5403 assert(LdVT.isScalableVector() == WidenVT.isScalableVector()); 5404 5405 // Load information 5406 SDValue Chain = LD->getChain(); 5407 SDValue BasePtr = LD->getBasePtr(); 5408 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); 5409 AAMDNodes AAInfo = LD->getAAInfo(); 5410 5411 if (LdVT.isScalableVector()) 5412 report_fatal_error("Generating widen scalable extending vector loads is " 5413 "not yet supported"); 5414 5415 EVT EltVT = WidenVT.getVectorElementType(); 5416 EVT LdEltVT = LdVT.getVectorElementType(); 5417 unsigned NumElts = LdVT.getVectorNumElements(); 5418 5419 // Load each element and widen. 5420 unsigned WidenNumElts = WidenVT.getVectorNumElements(); 5421 SmallVector<SDValue, 16> Ops(WidenNumElts); 5422 unsigned Increment = LdEltVT.getSizeInBits() / 8; 5423 Ops[0] = 5424 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(), 5425 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo); 5426 LdChain.push_back(Ops[0].getValue(1)); 5427 unsigned i = 0, Offset = Increment; 5428 for (i=1; i < NumElts; ++i, Offset += Increment) { 5429 SDValue NewBasePtr = 5430 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset)); 5431 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, 5432 LD->getPointerInfo().getWithOffset(Offset), LdEltVT, 5433 LD->getOriginalAlign(), MMOFlags, AAInfo); 5434 LdChain.push_back(Ops[i].getValue(1)); 5435 } 5436 5437 // Fill the rest with undefs. 5438 SDValue UndefVal = DAG.getUNDEF(EltVT); 5439 for (; i != WidenNumElts; ++i) 5440 Ops[i] = UndefVal; 5441 5442 return DAG.getBuildVector(WidenVT, dl, Ops); 5443} 5444 5445void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, 5446 StoreSDNode *ST) { 5447 // The strategy assumes that we can efficiently store power-of-two widths. 5448 // The routine chops the vector into the largest vector stores with the same 5449 // element type or scalar stores. 5450 SDValue Chain = ST->getChain(); 5451 SDValue BasePtr = ST->getBasePtr(); 5452 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); 5453 AAMDNodes AAInfo = ST->getAAInfo(); 5454 SDValue ValOp = GetWidenedVector(ST->getValue()); 5455 SDLoc dl(ST); 5456 5457 EVT StVT = ST->getMemoryVT(); 5458 TypeSize StWidth = StVT.getSizeInBits(); 5459 EVT ValVT = ValOp.getValueType(); 5460 TypeSize ValWidth = ValVT.getSizeInBits(); 5461 EVT ValEltVT = ValVT.getVectorElementType(); 5462 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits(); 5463 assert(StVT.getVectorElementType() == ValEltVT); 5464 assert(StVT.isScalableVector() == ValVT.isScalableVector() && 5465 "Mismatch between store and value types"); 5466 5467 int Idx = 0; // current index to store 5468 5469 MachinePointerInfo MPI = ST->getPointerInfo(); 5470 uint64_t ScaledOffset = 0; 5471 while (StWidth.isNonZero()) { 5472 // Find the largest vector type we can store with. 5473 EVT NewVT = FindMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT); 5474 TypeSize NewVTWidth = NewVT.getSizeInBits(); 5475 5476 if (NewVT.isVector()) { 5477 unsigned NumVTElts = NewVT.getVectorMinNumElements(); 5478 do { 5479 Align NewAlign = ScaledOffset == 0 5480 ? ST->getOriginalAlign() 5481 : commonAlignment(ST->getAlign(), ScaledOffset); 5482 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp, 5483 DAG.getVectorIdxConstant(Idx, dl)); 5484 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign, 5485 MMOFlags, AAInfo); 5486 StChain.push_back(PartStore); 5487 5488 StWidth -= NewVTWidth; 5489 Idx += NumVTElts; 5490 5491 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr, 5492 &ScaledOffset); 5493 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth)); 5494 } else { 5495 // Cast the vector to the scalar type we can store. 5496 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize(); 5497 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts); 5498 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp); 5499 // Readjust index position based on new vector type. 5500 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize(); 5501 do { 5502 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp, 5503 DAG.getVectorIdxConstant(Idx++, dl)); 5504 SDValue PartStore = 5505 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(), 5506 MMOFlags, AAInfo); 5507 StChain.push_back(PartStore); 5508 5509 StWidth -= NewVTWidth; 5510 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr); 5511 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth)); 5512 // Restore index back to be relative to the original widen element type. 5513 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth; 5514 } 5515 } 5516} 5517 5518/// Modifies a vector input (widen or narrows) to a vector of NVT. The 5519/// input vector must have the same element type as NVT. 5520/// FillWithZeroes specifies that the vector should be widened with zeroes. 5521SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, 5522 bool FillWithZeroes) { 5523 // Note that InOp might have been widened so it might already have 5524 // the right width or it might need be narrowed. 5525 EVT InVT = InOp.getValueType(); 5526 assert(InVT.getVectorElementType() == NVT.getVectorElementType() && 5527 "input and widen element type must match"); 5528 SDLoc dl(InOp); 5529 5530 // Check if InOp already has the right width. 5531 if (InVT == NVT) 5532 return InOp; 5533 5534 unsigned InNumElts = InVT.getVectorNumElements(); 5535 unsigned WidenNumElts = NVT.getVectorNumElements(); 5536 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) { 5537 unsigned NumConcat = WidenNumElts / InNumElts; 5538 SmallVector<SDValue, 16> Ops(NumConcat); 5539 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) : 5540 DAG.getUNDEF(InVT); 5541 Ops[0] = InOp; 5542 for (unsigned i = 1; i != NumConcat; ++i) 5543 Ops[i] = FillVal; 5544 5545 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops); 5546 } 5547 5548 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts) 5549 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp, 5550 DAG.getVectorIdxConstant(0, dl)); 5551 5552 // Fall back to extract and build. 5553 SmallVector<SDValue, 16> Ops(WidenNumElts); 5554 EVT EltVT = NVT.getVectorElementType(); 5555 unsigned MinNumElts = std::min(WidenNumElts, InNumElts); 5556 unsigned Idx; 5557 for (Idx = 0; Idx < MinNumElts; ++Idx) 5558 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, 5559 DAG.getVectorIdxConstant(Idx, dl)); 5560 5561 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) : 5562 DAG.getUNDEF(EltVT); 5563 for ( ; Idx < WidenNumElts; ++Idx) 5564 Ops[Idx] = FillVal; 5565 return DAG.getBuildVector(NVT, dl, Ops); 5566} 5567 5568void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, 5569 SDValue &Hi) { 5570 SDValue InLo, InHi; 5571 GetSplitVector(N->getOperand(0), InLo, InHi); 5572 SDLoc DL(N); 5573 5574 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi); 5575 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo); 5576} 5577 5578void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, 5579 SDValue &Hi) { 5580 EVT VT = N->getValueType(0); 5581 SDLoc DL(N); 5582 5583 EVT LoVT, HiVT; 5584 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); 5585 5586 SDValue Expanded = TLI.expandVectorSplice(N, DAG); 5587 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded, 5588 DAG.getVectorIdxConstant(0, DL)); 5589 Hi = 5590 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded, 5591 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); 5592} 5593