1285163Sdim//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==// 2285163Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6285163Sdim// 7285163Sdim//===----------------------------------------------------------------------===// 8285163Sdim/// 9285163Sdim/// \file 10341825Sdim/// This file implements the WebAssemblyTargetLowering class. 11285163Sdim/// 12285163Sdim//===----------------------------------------------------------------------===// 13285163Sdim 14285163Sdim#include "WebAssemblyISelLowering.h" 15285163Sdim#include "MCTargetDesc/WebAssemblyMCTargetDesc.h" 16285163Sdim#include "WebAssemblyMachineFunctionInfo.h" 17285163Sdim#include "WebAssemblySubtarget.h" 18285163Sdim#include "WebAssemblyTargetMachine.h" 19285163Sdim#include "llvm/CodeGen/Analysis.h" 20296417Sdim#include "llvm/CodeGen/CallingConvLower.h" 21327952Sdim#include "llvm/CodeGen/MachineInstrBuilder.h" 22296417Sdim#include "llvm/CodeGen/MachineJumpTableInfo.h" 23344779Sdim#include "llvm/CodeGen/MachineModuleInfo.h" 24285163Sdim#include "llvm/CodeGen/MachineRegisterInfo.h" 25285163Sdim#include "llvm/CodeGen/SelectionDAG.h" 26344779Sdim#include "llvm/CodeGen/WasmEHFuncInfo.h" 27296417Sdim#include "llvm/IR/DiagnosticInfo.h" 28296417Sdim#include "llvm/IR/DiagnosticPrinter.h" 29285163Sdim#include "llvm/IR/Function.h" 30285163Sdim#include "llvm/IR/Intrinsics.h" 31360784Sdim#include "llvm/IR/IntrinsicsWebAssembly.h" 32285163Sdim#include "llvm/Support/Debug.h" 33285163Sdim#include "llvm/Support/ErrorHandling.h" 34285163Sdim#include "llvm/Support/raw_ostream.h" 35285163Sdim#include "llvm/Target/TargetOptions.h" 36285163Sdimusing namespace llvm; 37285163Sdim 38285163Sdim#define DEBUG_TYPE "wasm-lower" 39285163Sdim 40285163SdimWebAssemblyTargetLowering::WebAssemblyTargetLowering( 41285163Sdim const TargetMachine &TM, const WebAssemblySubtarget &STI) 42285163Sdim : TargetLowering(TM), Subtarget(&STI) { 43296417Sdim auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32; 44296417Sdim 45296417Sdim // Booleans always contain 0 or 1. 46296417Sdim setBooleanContents(ZeroOrOneBooleanContent); 47344779Sdim // Except in SIMD vectors 48344779Sdim setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 49286684Sdim // We don't know the microarchitecture here, so just reduce register pressure. 50286684Sdim setSchedulingPreference(Sched::RegPressure); 51296417Sdim // Tell ISel that we have a stack pointer. 52296417Sdim setStackPointerRegisterToSaveRestore( 53296417Sdim Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32); 54296417Sdim // Set up the register classes. 55296417Sdim addRegisterClass(MVT::i32, &WebAssembly::I32RegClass); 56296417Sdim addRegisterClass(MVT::i64, &WebAssembly::I64RegClass); 57296417Sdim addRegisterClass(MVT::f32, &WebAssembly::F32RegClass); 58296417Sdim addRegisterClass(MVT::f64, &WebAssembly::F64RegClass); 59314564Sdim if (Subtarget->hasSIMD128()) { 60314564Sdim addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass); 61314564Sdim addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass); 62314564Sdim addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass); 63314564Sdim addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass); 64314564Sdim } 65353358Sdim if (Subtarget->hasUnimplementedSIMD128()) { 66353358Sdim addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass); 67353358Sdim addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass); 68353358Sdim } 69296417Sdim // Compute derived properties from the register classes. 70296417Sdim computeRegisterProperties(Subtarget->getRegisterInfo()); 71296417Sdim 72296417Sdim setOperationAction(ISD::GlobalAddress, MVTPtr, Custom); 73296417Sdim setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom); 74296417Sdim setOperationAction(ISD::JumpTable, MVTPtr, Custom); 75309124Sdim setOperationAction(ISD::BlockAddress, MVTPtr, Custom); 76309124Sdim setOperationAction(ISD::BRIND, MVT::Other, Custom); 77296417Sdim 78296417Sdim // Take the default expansion for va_arg, va_copy, and va_end. There is no 79296417Sdim // default action for va_start, so we do that custom. 80296417Sdim setOperationAction(ISD::VASTART, MVT::Other, Custom); 81296417Sdim setOperationAction(ISD::VAARG, MVT::Other, Expand); 82296417Sdim setOperationAction(ISD::VACOPY, MVT::Other, Expand); 83296417Sdim setOperationAction(ISD::VAEND, MVT::Other, Expand); 84296417Sdim 85344779Sdim for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { 86296417Sdim // Don't expand the floating-point types to constant pools. 87296417Sdim setOperationAction(ISD::ConstantFP, T, Legal); 88296417Sdim // Expand floating-point comparisons. 89296417Sdim for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE, 90296417Sdim ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE}) 91296417Sdim setCondCodeAction(CC, T, Expand); 92296417Sdim // Expand floating-point library function operators. 93344779Sdim for (auto Op : 94344779Sdim {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA}) 95296417Sdim setOperationAction(Op, T, Expand); 96296417Sdim // Note supported floating-point library function operators that otherwise 97296417Sdim // default to expand. 98296417Sdim for (auto Op : 99296417Sdim {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT}) 100296417Sdim setOperationAction(Op, T, Legal); 101344779Sdim // Support minimum and maximum, which otherwise default to expand. 102344779Sdim setOperationAction(ISD::FMINIMUM, T, Legal); 103344779Sdim setOperationAction(ISD::FMAXIMUM, T, Legal); 104321369Sdim // WebAssembly currently has no builtin f16 support. 105321369Sdim setOperationAction(ISD::FP16_TO_FP, T, Expand); 106321369Sdim setOperationAction(ISD::FP_TO_FP16, T, Expand); 107321369Sdim setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand); 108321369Sdim setTruncStoreAction(T, MVT::f16, Expand); 109296417Sdim } 110296417Sdim 111344779Sdim // Expand unavailable integer operations. 112344779Sdim for (auto Op : 113344779Sdim {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU, 114344779Sdim ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS, 115344779Sdim ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) { 116353358Sdim for (auto T : {MVT::i32, MVT::i64}) 117296417Sdim setOperationAction(Op, T, Expand); 118353358Sdim if (Subtarget->hasSIMD128()) 119353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) 120344779Sdim setOperationAction(Op, T, Expand); 121353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 122353358Sdim setOperationAction(Op, MVT::v2i64, Expand); 123296417Sdim } 124296417Sdim 125353358Sdim // SIMD-specific configuration 126353358Sdim if (Subtarget->hasSIMD128()) { 127353358Sdim // Support saturating add for i8x16 and i16x8 128353358Sdim for (auto Op : {ISD::SADDSAT, ISD::UADDSAT}) 129353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16}) 130353358Sdim setOperationAction(Op, T, Legal); 131344779Sdim 132353358Sdim // Custom lower BUILD_VECTORs to minimize number of replace_lanes 133353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) 134353358Sdim setOperationAction(ISD::BUILD_VECTOR, T, Custom); 135353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 136353358Sdim for (auto T : {MVT::v2i64, MVT::v2f64}) 137353358Sdim setOperationAction(ISD::BUILD_VECTOR, T, Custom); 138353358Sdim 139353358Sdim // We have custom shuffle lowering to expose the shuffle mask 140353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) 141344779Sdim setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom); 142353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 143353358Sdim for (auto T: {MVT::v2i64, MVT::v2f64}) 144353358Sdim setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom); 145344779Sdim 146353358Sdim // Custom lowering since wasm shifts must have a scalar shift amount 147353358Sdim for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) { 148353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) 149344779Sdim setOperationAction(Op, T, Custom); 150353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 151344779Sdim setOperationAction(Op, MVT::v2i64, Custom); 152353358Sdim } 153344779Sdim 154353358Sdim // Custom lower lane accesses to expand out variable indices 155353358Sdim for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) { 156353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) 157353358Sdim setOperationAction(Op, T, Custom); 158353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 159353358Sdim for (auto T : {MVT::v2i64, MVT::v2f64}) 160353358Sdim setOperationAction(Op, T, Custom); 161353358Sdim } 162353358Sdim 163353358Sdim // There is no i64x2.mul instruction 164353358Sdim setOperationAction(ISD::MUL, MVT::v2i64, Expand); 165353358Sdim 166353358Sdim // There are no vector select instructions 167344779Sdim for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) { 168344779Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) 169344779Sdim setOperationAction(Op, T, Expand); 170344779Sdim if (Subtarget->hasUnimplementedSIMD128()) 171344779Sdim for (auto T : {MVT::v2i64, MVT::v2f64}) 172344779Sdim setOperationAction(Op, T, Expand); 173344779Sdim } 174344779Sdim 175353358Sdim // Expand integer operations supported for scalars but not SIMD 176353358Sdim for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV, 177353358Sdim ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) { 178353358Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) 179353358Sdim setOperationAction(Op, T, Expand); 180353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 181353358Sdim setOperationAction(Op, MVT::v2i64, Expand); 182353358Sdim } 183353358Sdim 184360784Sdim // But we do have integer min and max operations 185360784Sdim if (Subtarget->hasUnimplementedSIMD128()) { 186360784Sdim for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) 187360784Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) 188360784Sdim setOperationAction(Op, T, Legal); 189360784Sdim } 190360784Sdim 191353358Sdim // Expand float operations supported for scalars but not SIMD 192353358Sdim for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, 193353358Sdim ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, 194353358Sdim ISD::FEXP, ISD::FEXP2, ISD::FRINT}) { 195353358Sdim setOperationAction(Op, MVT::v4f32, Expand); 196353358Sdim if (Subtarget->hasUnimplementedSIMD128()) 197353358Sdim setOperationAction(Op, MVT::v2f64, Expand); 198353358Sdim } 199353358Sdim 200360784Sdim // Expand operations not supported for i64x2 vectors 201360784Sdim if (Subtarget->hasUnimplementedSIMD128()) 202360784Sdim for (unsigned CC = 0; CC < ISD::SETCC_INVALID; ++CC) 203360784Sdim setCondCodeAction(static_cast<ISD::CondCode>(CC), MVT::v2i64, Custom); 204360784Sdim 205353358Sdim // Expand additional SIMD ops that V8 hasn't implemented yet 206353358Sdim if (!Subtarget->hasUnimplementedSIMD128()) { 207353358Sdim setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); 208353358Sdim setOperationAction(ISD::FDIV, MVT::v4f32, Expand); 209353358Sdim } 210353358Sdim } 211353358Sdim 212296417Sdim // As a special case, these operators use the type to mean the type to 213296417Sdim // sign-extend from. 214327952Sdim setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 215341825Sdim if (!Subtarget->hasSignExt()) { 216344779Sdim // Sign extends are legal only when extending a vector extract 217344779Sdim auto Action = Subtarget->hasSIMD128() ? Custom : Expand; 218327952Sdim for (auto T : {MVT::i8, MVT::i16, MVT::i32}) 219344779Sdim setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action); 220327952Sdim } 221360784Sdim for (auto T : MVT::integer_fixedlen_vector_valuetypes()) 222344779Sdim setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand); 223296417Sdim 224296417Sdim // Dynamic stack allocation: use the default expansion. 225296417Sdim setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); 226296417Sdim setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); 227296417Sdim setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand); 228296417Sdim 229296417Sdim setOperationAction(ISD::FrameIndex, MVT::i32, Custom); 230309124Sdim setOperationAction(ISD::CopyToReg, MVT::Other, Custom); 231296417Sdim 232296417Sdim // Expand these forms; we pattern-match the forms that we can handle in isel. 233296417Sdim for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) 234296417Sdim for (auto Op : {ISD::BR_CC, ISD::SELECT_CC}) 235296417Sdim setOperationAction(Op, T, Expand); 236296417Sdim 237296417Sdim // We have custom switch handling. 238296417Sdim setOperationAction(ISD::BR_JT, MVT::Other, Custom); 239296417Sdim 240296417Sdim // WebAssembly doesn't have: 241296417Sdim // - Floating-point extending loads. 242296417Sdim // - Floating-point truncating stores. 243296417Sdim // - i1 extending loads. 244360784Sdim // - truncating SIMD stores and most extending loads 245296417Sdim setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 246296417Sdim setTruncStoreAction(MVT::f64, MVT::f32, Expand); 247296417Sdim for (auto T : MVT::integer_valuetypes()) 248296417Sdim for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) 249296417Sdim setLoadExtAction(Ext, T, MVT::i1, Promote); 250344779Sdim if (Subtarget->hasSIMD128()) { 251344779Sdim for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, 252344779Sdim MVT::v2f64}) { 253360784Sdim for (auto MemT : MVT::fixedlen_vector_valuetypes()) { 254344779Sdim if (MVT(T) != MemT) { 255344779Sdim setTruncStoreAction(T, MemT, Expand); 256344779Sdim for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) 257344779Sdim setLoadExtAction(Ext, T, MemT, Expand); 258344779Sdim } 259344779Sdim } 260344779Sdim } 261360784Sdim // But some vector extending loads are legal 262360784Sdim if (Subtarget->hasUnimplementedSIMD128()) { 263360784Sdim for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) { 264360784Sdim setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal); 265360784Sdim setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal); 266360784Sdim setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal); 267360784Sdim } 268360784Sdim } 269344779Sdim } 270296417Sdim 271353358Sdim // Don't do anything clever with build_pairs 272353358Sdim setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); 273344779Sdim 274296417Sdim // Trap lowers to wasm unreachable 275296417Sdim setOperationAction(ISD::TRAP, MVT::Other, Legal); 276327952Sdim 277341825Sdim // Exception handling intrinsics 278341825Sdim setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 279344779Sdim setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 280341825Sdim 281327952Sdim setMaxAtomicSizeInBitsSupported(64); 282353358Sdim 283353358Sdim // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is 284353358Sdim // consistent with the f64 and f128 names. 285353358Sdim setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); 286353358Sdim setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); 287353358Sdim 288353358Sdim // Define the emscripten name for return address helper. 289353358Sdim // TODO: when implementing other WASM backends, make this generic or only do 290353358Sdim // this on emscripten depending on what they end up doing. 291353358Sdim setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address"); 292353358Sdim 293353358Sdim // Always convert switches to br_tables unless there is only one case, which 294353358Sdim // is equivalent to a simple branch. This reduces code size for wasm, and we 295353358Sdim // defer possible jump table optimizations to the VM. 296353358Sdim setMinimumJumpTableEntries(2); 297285163Sdim} 298285163Sdim 299344779SdimTargetLowering::AtomicExpansionKind 300344779SdimWebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { 301344779Sdim // We have wasm instructions for these 302344779Sdim switch (AI->getOperation()) { 303344779Sdim case AtomicRMWInst::Add: 304344779Sdim case AtomicRMWInst::Sub: 305344779Sdim case AtomicRMWInst::And: 306344779Sdim case AtomicRMWInst::Or: 307344779Sdim case AtomicRMWInst::Xor: 308344779Sdim case AtomicRMWInst::Xchg: 309344779Sdim return AtomicExpansionKind::None; 310344779Sdim default: 311344779Sdim break; 312344779Sdim } 313344779Sdim return AtomicExpansionKind::CmpXChg; 314344779Sdim} 315344779Sdim 316296417SdimFastISel *WebAssemblyTargetLowering::createFastISel( 317296417Sdim FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const { 318296417Sdim return WebAssembly::createFastISel(FuncInfo, LibInfo); 319296417Sdim} 320296417Sdim 321296417SdimMVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/, 322296417Sdim EVT VT) const { 323296417Sdim unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1); 324344779Sdim if (BitWidth > 1 && BitWidth < 8) 325344779Sdim BitWidth = 8; 326296417Sdim 327296417Sdim if (BitWidth > 64) { 328309124Sdim // The shift will be lowered to a libcall, and compiler-rt libcalls expect 329309124Sdim // the count to be an i32. 330309124Sdim BitWidth = 32; 331296417Sdim assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && 332309124Sdim "32-bit shift counts ought to be enough for anyone"); 333296417Sdim } 334296417Sdim 335296417Sdim MVT Result = MVT::getIntegerVT(BitWidth); 336296417Sdim assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE && 337296417Sdim "Unable to represent scalar shift amount type"); 338296417Sdim return Result; 339296417Sdim} 340296417Sdim 341327952Sdim// Lower an fp-to-int conversion operator from the LLVM opcode, which has an 342327952Sdim// undefined result on invalid/overflow, to the WebAssembly opcode, which 343327952Sdim// traps on invalid/overflow. 344344779Sdimstatic MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL, 345344779Sdim MachineBasicBlock *BB, 346344779Sdim const TargetInstrInfo &TII, 347344779Sdim bool IsUnsigned, bool Int64, 348344779Sdim bool Float64, unsigned LoweredOpcode) { 349327952Sdim MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); 350327952Sdim 351360784Sdim Register OutReg = MI.getOperand(0).getReg(); 352360784Sdim Register InReg = MI.getOperand(1).getReg(); 353327952Sdim 354327952Sdim unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32; 355327952Sdim unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32; 356327952Sdim unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32; 357327952Sdim unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32; 358327952Sdim unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32; 359327952Sdim unsigned Eqz = WebAssembly::EQZ_I32; 360327952Sdim unsigned And = WebAssembly::AND_I32; 361327952Sdim int64_t Limit = Int64 ? INT64_MIN : INT32_MIN; 362327952Sdim int64_t Substitute = IsUnsigned ? 0 : Limit; 363327952Sdim double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit; 364327952Sdim auto &Context = BB->getParent()->getFunction().getContext(); 365327952Sdim Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context); 366327952Sdim 367353358Sdim const BasicBlock *LLVMBB = BB->getBasicBlock(); 368327952Sdim MachineFunction *F = BB->getParent(); 369353358Sdim MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB); 370353358Sdim MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB); 371353358Sdim MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB); 372327952Sdim 373327952Sdim MachineFunction::iterator It = ++BB->getIterator(); 374327952Sdim F->insert(It, FalseMBB); 375327952Sdim F->insert(It, TrueMBB); 376327952Sdim F->insert(It, DoneMBB); 377327952Sdim 378327952Sdim // Transfer the remainder of BB and its successor edges to DoneMBB. 379353358Sdim DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end()); 380327952Sdim DoneMBB->transferSuccessorsAndUpdatePHIs(BB); 381327952Sdim 382327952Sdim BB->addSuccessor(TrueMBB); 383327952Sdim BB->addSuccessor(FalseMBB); 384327952Sdim TrueMBB->addSuccessor(DoneMBB); 385327952Sdim FalseMBB->addSuccessor(DoneMBB); 386327952Sdim 387327952Sdim unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg; 388327952Sdim Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 389327952Sdim Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 390327952Sdim CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 391327952Sdim EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 392327952Sdim FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); 393327952Sdim TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); 394327952Sdim 395327952Sdim MI.eraseFromParent(); 396327952Sdim // For signed numbers, we can do a single comparison to determine whether 397327952Sdim // fabs(x) is within range. 398327952Sdim if (IsUnsigned) { 399327952Sdim Tmp0 = InReg; 400327952Sdim } else { 401344779Sdim BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg); 402327952Sdim } 403327952Sdim BuildMI(BB, DL, TII.get(FConst), Tmp1) 404327952Sdim .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal))); 405344779Sdim BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1); 406327952Sdim 407327952Sdim // For unsigned numbers, we have to do a separate comparison with zero. 408327952Sdim if (IsUnsigned) { 409327952Sdim Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); 410360784Sdim Register SecondCmpReg = 411344779Sdim MRI.createVirtualRegister(&WebAssembly::I32RegClass); 412360784Sdim Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); 413327952Sdim BuildMI(BB, DL, TII.get(FConst), Tmp1) 414327952Sdim .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0))); 415344779Sdim BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1); 416344779Sdim BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg); 417327952Sdim CmpReg = AndReg; 418327952Sdim } 419327952Sdim 420344779Sdim BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg); 421327952Sdim 422327952Sdim // Create the CFG diamond to select between doing the conversion or using 423327952Sdim // the substitute value. 424344779Sdim BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg); 425344779Sdim BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg); 426344779Sdim BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB); 427344779Sdim BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute); 428327952Sdim BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg) 429327952Sdim .addReg(FalseReg) 430327952Sdim .addMBB(FalseMBB) 431327952Sdim .addReg(TrueReg) 432327952Sdim .addMBB(TrueMBB); 433327952Sdim 434327952Sdim return DoneMBB; 435327952Sdim} 436327952Sdim 437344779SdimMachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter( 438344779Sdim MachineInstr &MI, MachineBasicBlock *BB) const { 439327952Sdim const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); 440327952Sdim DebugLoc DL = MI.getDebugLoc(); 441327952Sdim 442327952Sdim switch (MI.getOpcode()) { 443344779Sdim default: 444344779Sdim llvm_unreachable("Unexpected instr type to insert"); 445327952Sdim case WebAssembly::FP_TO_SINT_I32_F32: 446327952Sdim return LowerFPToInt(MI, DL, BB, TII, false, false, false, 447327952Sdim WebAssembly::I32_TRUNC_S_F32); 448327952Sdim case WebAssembly::FP_TO_UINT_I32_F32: 449327952Sdim return LowerFPToInt(MI, DL, BB, TII, true, false, false, 450327952Sdim WebAssembly::I32_TRUNC_U_F32); 451327952Sdim case WebAssembly::FP_TO_SINT_I64_F32: 452327952Sdim return LowerFPToInt(MI, DL, BB, TII, false, true, false, 453327952Sdim WebAssembly::I64_TRUNC_S_F32); 454327952Sdim case WebAssembly::FP_TO_UINT_I64_F32: 455327952Sdim return LowerFPToInt(MI, DL, BB, TII, true, true, false, 456327952Sdim WebAssembly::I64_TRUNC_U_F32); 457327952Sdim case WebAssembly::FP_TO_SINT_I32_F64: 458327952Sdim return LowerFPToInt(MI, DL, BB, TII, false, false, true, 459327952Sdim WebAssembly::I32_TRUNC_S_F64); 460327952Sdim case WebAssembly::FP_TO_UINT_I32_F64: 461327952Sdim return LowerFPToInt(MI, DL, BB, TII, true, false, true, 462327952Sdim WebAssembly::I32_TRUNC_U_F64); 463327952Sdim case WebAssembly::FP_TO_SINT_I64_F64: 464327952Sdim return LowerFPToInt(MI, DL, BB, TII, false, true, true, 465327952Sdim WebAssembly::I64_TRUNC_S_F64); 466327952Sdim case WebAssembly::FP_TO_UINT_I64_F64: 467327952Sdim return LowerFPToInt(MI, DL, BB, TII, true, true, true, 468327952Sdim WebAssembly::I64_TRUNC_U_F64); 469344779Sdim llvm_unreachable("Unexpected instruction to emit with custom inserter"); 470327952Sdim } 471327952Sdim} 472327952Sdim 473344779Sdimconst char * 474344779SdimWebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const { 475296417Sdim switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) { 476344779Sdim case WebAssemblyISD::FIRST_NUMBER: 477360784Sdim case WebAssemblyISD::FIRST_MEM_OPCODE: 478344779Sdim break; 479344779Sdim#define HANDLE_NODETYPE(NODE) \ 480344779Sdim case WebAssemblyISD::NODE: \ 481296417Sdim return "WebAssemblyISD::" #NODE; 482360784Sdim#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE) 483296417Sdim#include "WebAssemblyISD.def" 484360784Sdim#undef HANDLE_MEM_NODETYPE 485296417Sdim#undef HANDLE_NODETYPE 486296417Sdim } 487296417Sdim return nullptr; 488296417Sdim} 489296417Sdim 490296417Sdimstd::pair<unsigned, const TargetRegisterClass *> 491296417SdimWebAssemblyTargetLowering::getRegForInlineAsmConstraint( 492296417Sdim const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 493296417Sdim // First, see if this is a constraint that directly corresponds to a 494296417Sdim // WebAssembly register class. 495296417Sdim if (Constraint.size() == 1) { 496296417Sdim switch (Constraint[0]) { 497344779Sdim case 'r': 498344779Sdim assert(VT != MVT::iPTR && "Pointer MVT not expected here"); 499344779Sdim if (Subtarget->hasSIMD128() && VT.isVector()) { 500344779Sdim if (VT.getSizeInBits() == 128) 501344779Sdim return std::make_pair(0U, &WebAssembly::V128RegClass); 502344779Sdim } 503344779Sdim if (VT.isInteger() && !VT.isVector()) { 504344779Sdim if (VT.getSizeInBits() <= 32) 505344779Sdim return std::make_pair(0U, &WebAssembly::I32RegClass); 506344779Sdim if (VT.getSizeInBits() <= 64) 507344779Sdim return std::make_pair(0U, &WebAssembly::I64RegClass); 508344779Sdim } 509344779Sdim break; 510344779Sdim default: 511344779Sdim break; 512296417Sdim } 513296417Sdim } 514296417Sdim 515296417Sdim return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 516296417Sdim} 517296417Sdim 518296417Sdimbool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const { 519296417Sdim // Assume ctz is a relatively cheap operation. 520296417Sdim return true; 521296417Sdim} 522296417Sdim 523296417Sdimbool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const { 524296417Sdim // Assume clz is a relatively cheap operation. 525296417Sdim return true; 526296417Sdim} 527296417Sdim 528296417Sdimbool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL, 529296417Sdim const AddrMode &AM, 530344779Sdim Type *Ty, unsigned AS, 531327952Sdim Instruction *I) const { 532296417Sdim // WebAssembly offsets are added as unsigned without wrapping. The 533296417Sdim // isLegalAddressingMode gives us no way to determine if wrapping could be 534296417Sdim // happening, so we approximate this by accepting only non-negative offsets. 535344779Sdim if (AM.BaseOffs < 0) 536344779Sdim return false; 537296417Sdim 538296417Sdim // WebAssembly has no scale register operands. 539344779Sdim if (AM.Scale != 0) 540344779Sdim return false; 541296417Sdim 542296417Sdim // Everything else is legal. 543296417Sdim return true; 544296417Sdim} 545296417Sdim 546309124Sdimbool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses( 547353358Sdim EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/, 548353358Sdim MachineMemOperand::Flags /*Flags*/, bool *Fast) const { 549309124Sdim // WebAssembly supports unaligned accesses, though it should be declared 550309124Sdim // with the p2align attribute on loads and stores which do so, and there 551309124Sdim // may be a performance impact. We tell LLVM they're "fast" because 552309124Sdim // for the kinds of things that LLVM uses this for (merging adjacent stores 553309124Sdim // of constants, etc.), WebAssembly implementations will either want the 554309124Sdim // unaligned access or they'll split anyway. 555344779Sdim if (Fast) 556344779Sdim *Fast = true; 557309124Sdim return true; 558309124Sdim} 559309124Sdim 560321369Sdimbool WebAssemblyTargetLowering::isIntDivCheap(EVT VT, 561321369Sdim AttributeList Attr) const { 562309124Sdim // The current thinking is that wasm engines will perform this optimization, 563309124Sdim // so we can save on code size. 564309124Sdim return true; 565309124Sdim} 566309124Sdim 567360784Sdimbool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const { 568360784Sdim if (!Subtarget->hasUnimplementedSIMD128()) 569360784Sdim return false; 570360784Sdim MVT ExtT = ExtVal.getSimpleValueType(); 571360784Sdim MVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getSimpleValueType(0); 572360784Sdim return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) || 573360784Sdim (ExtT == MVT::v4i32 && MemT == MVT::v4i16) || 574360784Sdim (ExtT == MVT::v2i64 && MemT == MVT::v2i32); 575360784Sdim} 576360784Sdim 577341825SdimEVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL, 578341825Sdim LLVMContext &C, 579341825Sdim EVT VT) const { 580341825Sdim if (VT.isVector()) 581341825Sdim return VT.changeVectorElementTypeToInteger(); 582341825Sdim 583341825Sdim return TargetLowering::getSetCCResultType(DL, C, VT); 584341825Sdim} 585341825Sdim 586344779Sdimbool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, 587344779Sdim const CallInst &I, 588344779Sdim MachineFunction &MF, 589344779Sdim unsigned Intrinsic) const { 590344779Sdim switch (Intrinsic) { 591344779Sdim case Intrinsic::wasm_atomic_notify: 592344779Sdim Info.opc = ISD::INTRINSIC_W_CHAIN; 593344779Sdim Info.memVT = MVT::i32; 594344779Sdim Info.ptrVal = I.getArgOperand(0); 595344779Sdim Info.offset = 0; 596360784Sdim Info.align = Align(4); 597344779Sdim // atomic.notify instruction does not really load the memory specified with 598344779Sdim // this argument, but MachineMemOperand should either be load or store, so 599344779Sdim // we set this to a load. 600344779Sdim // FIXME Volatile isn't really correct, but currently all LLVM atomic 601344779Sdim // instructions are treated as volatiles in the backend, so we should be 602344779Sdim // consistent. The same applies for wasm_atomic_wait intrinsics too. 603344779Sdim Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 604344779Sdim return true; 605344779Sdim case Intrinsic::wasm_atomic_wait_i32: 606344779Sdim Info.opc = ISD::INTRINSIC_W_CHAIN; 607344779Sdim Info.memVT = MVT::i32; 608344779Sdim Info.ptrVal = I.getArgOperand(0); 609344779Sdim Info.offset = 0; 610360784Sdim Info.align = Align(4); 611344779Sdim Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 612344779Sdim return true; 613344779Sdim case Intrinsic::wasm_atomic_wait_i64: 614344779Sdim Info.opc = ISD::INTRINSIC_W_CHAIN; 615344779Sdim Info.memVT = MVT::i64; 616344779Sdim Info.ptrVal = I.getArgOperand(0); 617344779Sdim Info.offset = 0; 618360784Sdim Info.align = Align(8); 619344779Sdim Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; 620344779Sdim return true; 621344779Sdim default: 622344779Sdim return false; 623344779Sdim } 624344779Sdim} 625344779Sdim 626285163Sdim//===----------------------------------------------------------------------===// 627285163Sdim// WebAssembly Lowering private implementation. 628285163Sdim//===----------------------------------------------------------------------===// 629285163Sdim 630285163Sdim//===----------------------------------------------------------------------===// 631285163Sdim// Lowering Code 632285163Sdim//===----------------------------------------------------------------------===// 633285163Sdim 634353358Sdimstatic void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) { 635296417Sdim MachineFunction &MF = DAG.getMachineFunction(); 636296417Sdim DAG.getContext()->diagnose( 637353358Sdim DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc())); 638296417Sdim} 639296417Sdim 640296417Sdim// Test whether the given calling convention is supported. 641353358Sdimstatic bool callingConvSupported(CallingConv::ID CallConv) { 642296417Sdim // We currently support the language-independent target-independent 643296417Sdim // conventions. We don't yet have a way to annotate calls with properties like 644296417Sdim // "cold", and we don't have any call-clobbered registers, so these are mostly 645296417Sdim // all handled the same. 646296417Sdim return CallConv == CallingConv::C || CallConv == CallingConv::Fast || 647296417Sdim CallConv == CallingConv::Cold || 648296417Sdim CallConv == CallingConv::PreserveMost || 649296417Sdim CallConv == CallingConv::PreserveAll || 650360784Sdim CallConv == CallingConv::CXX_FAST_TLS || 651360784Sdim CallConv == CallingConv::WASM_EmscriptenInvoke; 652296417Sdim} 653296417Sdim 654344779SdimSDValue 655344779SdimWebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI, 656344779Sdim SmallVectorImpl<SDValue> &InVals) const { 657296417Sdim SelectionDAG &DAG = CLI.DAG; 658296417Sdim SDLoc DL = CLI.DL; 659296417Sdim SDValue Chain = CLI.Chain; 660296417Sdim SDValue Callee = CLI.Callee; 661296417Sdim MachineFunction &MF = DAG.getMachineFunction(); 662309124Sdim auto Layout = MF.getDataLayout(); 663296417Sdim 664296417Sdim CallingConv::ID CallConv = CLI.CallConv; 665353358Sdim if (!callingConvSupported(CallConv)) 666296417Sdim fail(DL, DAG, 667296417Sdim "WebAssembly doesn't support language-specific or target-specific " 668296417Sdim "calling conventions yet"); 669296417Sdim if (CLI.IsPatchPoint) 670296417Sdim fail(DL, DAG, "WebAssembly doesn't support patch point yet"); 671296417Sdim 672360784Sdim if (CLI.IsTailCall) { 673360784Sdim bool MustTail = CLI.CS && CLI.CS.isMustTailCall(); 674360784Sdim if (Subtarget->hasTailCall() && !CLI.IsVarArg) { 675360784Sdim // Do not tail call unless caller and callee return types match 676360784Sdim const Function &F = MF.getFunction(); 677360784Sdim const TargetMachine &TM = getTargetMachine(); 678360784Sdim Type *RetTy = F.getReturnType(); 679360784Sdim SmallVector<MVT, 4> CallerRetTys; 680360784Sdim SmallVector<MVT, 4> CalleeRetTys; 681360784Sdim computeLegalValueVTs(F, TM, RetTy, CallerRetTys); 682360784Sdim computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys); 683360784Sdim bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() && 684360784Sdim std::equal(CallerRetTys.begin(), CallerRetTys.end(), 685360784Sdim CalleeRetTys.begin()); 686360784Sdim if (!TypesMatch) { 687360784Sdim // musttail in this case would be an LLVM IR validation failure 688360784Sdim assert(!MustTail); 689360784Sdim CLI.IsTailCall = false; 690360784Sdim } 691360784Sdim } else { 692360784Sdim CLI.IsTailCall = false; 693360784Sdim if (MustTail) { 694360784Sdim if (CLI.IsVarArg) { 695360784Sdim // The return would pop the argument buffer 696360784Sdim fail(DL, DAG, "WebAssembly does not support varargs tail calls"); 697360784Sdim } else { 698360784Sdim fail(DL, DAG, "WebAssembly 'tail-call' feature not enabled"); 699360784Sdim } 700360784Sdim } 701360784Sdim } 702353358Sdim } 703296417Sdim 704296417Sdim SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 705296417Sdim if (Ins.size() > 1) 706296417Sdim fail(DL, DAG, "WebAssembly doesn't support more than 1 returned value yet"); 707296417Sdim 708296417Sdim SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 709309124Sdim SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 710360784Sdim 711360784Sdim // The generic code may have added an sret argument. If we're lowering an 712360784Sdim // invoke function, the ABI requires that the function pointer be the first 713360784Sdim // argument, so we may have to swap the arguments. 714360784Sdim if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 && 715360784Sdim Outs[0].Flags.isSRet()) { 716360784Sdim std::swap(Outs[0], Outs[1]); 717360784Sdim std::swap(OutVals[0], OutVals[1]); 718360784Sdim } 719360784Sdim 720341825Sdim unsigned NumFixedArgs = 0; 721353358Sdim for (unsigned I = 0; I < Outs.size(); ++I) { 722353358Sdim const ISD::OutputArg &Out = Outs[I]; 723353358Sdim SDValue &OutVal = OutVals[I]; 724296417Sdim if (Out.Flags.isNest()) 725296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); 726296417Sdim if (Out.Flags.isInAlloca()) 727296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); 728296417Sdim if (Out.Flags.isInConsecutiveRegs()) 729296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); 730296417Sdim if (Out.Flags.isInConsecutiveRegsLast()) 731296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); 732309124Sdim if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) { 733314564Sdim auto &MFI = MF.getFrameInfo(); 734314564Sdim int FI = MFI.CreateStackObject(Out.Flags.getByValSize(), 735314564Sdim Out.Flags.getByValAlign(), 736314564Sdim /*isSS=*/false); 737309124Sdim SDValue SizeNode = 738309124Sdim DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32); 739309124Sdim SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); 740309124Sdim Chain = DAG.getMemcpy( 741309124Sdim Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getByValAlign(), 742309124Sdim /*isVolatile*/ false, /*AlwaysInline=*/false, 743309124Sdim /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo()); 744309124Sdim OutVal = FINode; 745309124Sdim } 746341825Sdim // Count the number of fixed args *after* legalization. 747341825Sdim NumFixedArgs += Out.IsFixed; 748296417Sdim } 749296417Sdim 750296417Sdim bool IsVarArg = CLI.IsVarArg; 751309124Sdim auto PtrVT = getPointerTy(Layout); 752309124Sdim 753296417Sdim // Analyze operands of the call, assigning locations to each operand. 754296417Sdim SmallVector<CCValAssign, 16> ArgLocs; 755296417Sdim CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 756296417Sdim 757296417Sdim if (IsVarArg) { 758309124Sdim // Outgoing non-fixed arguments are placed in a buffer. First 759309124Sdim // compute their offsets and the total amount of buffer space needed. 760349004Sdim for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) { 761349004Sdim const ISD::OutputArg &Out = Outs[I]; 762349004Sdim SDValue &Arg = OutVals[I]; 763296417Sdim EVT VT = Arg.getValueType(); 764296417Sdim assert(VT != MVT::iPTR && "Legalized args should be concrete"); 765296417Sdim Type *Ty = VT.getTypeForEVT(*DAG.getContext()); 766349004Sdim unsigned Align = std::max(Out.Flags.getOrigAlign(), 767349004Sdim Layout.getABITypeAlignment(Ty)); 768309124Sdim unsigned Offset = CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), 769349004Sdim Align); 770296417Sdim CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(), 771296417Sdim Offset, VT.getSimpleVT(), 772296417Sdim CCValAssign::Full)); 773296417Sdim } 774296417Sdim } 775296417Sdim 776296417Sdim unsigned NumBytes = CCInfo.getAlignedCallFrameSize(); 777296417Sdim 778309124Sdim SDValue FINode; 779309124Sdim if (IsVarArg && NumBytes) { 780296417Sdim // For non-fixed arguments, next emit stores to store the argument values 781309124Sdim // to the stack buffer at the offsets computed above. 782314564Sdim int FI = MF.getFrameInfo().CreateStackObject(NumBytes, 783314564Sdim Layout.getStackAlignment(), 784314564Sdim /*isSS=*/false); 785296417Sdim unsigned ValNo = 0; 786296417Sdim SmallVector<SDValue, 8> Chains; 787296417Sdim for (SDValue Arg : 788296417Sdim make_range(OutVals.begin() + NumFixedArgs, OutVals.end())) { 789296417Sdim assert(ArgLocs[ValNo].getValNo() == ValNo && 790296417Sdim "ArgLocs should remain in order and only hold varargs args"); 791296417Sdim unsigned Offset = ArgLocs[ValNo++].getLocMemOffset(); 792309124Sdim FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); 793309124Sdim SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode, 794296417Sdim DAG.getConstant(Offset, DL, PtrVT)); 795344779Sdim Chains.push_back( 796344779Sdim DAG.getStore(Chain, DL, Arg, Add, 797344779Sdim MachinePointerInfo::getFixedStack(MF, FI, Offset), 0)); 798296417Sdim } 799296417Sdim if (!Chains.empty()) 800296417Sdim Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); 801309124Sdim } else if (IsVarArg) { 802309124Sdim FINode = DAG.getIntPtrConstant(0, DL); 803296417Sdim } 804296417Sdim 805353358Sdim if (Callee->getOpcode() == ISD::GlobalAddress) { 806353358Sdim // If the callee is a GlobalAddress node (quite common, every direct call 807353358Sdim // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress 808353358Sdim // doesn't at MO_GOT which is not needed for direct calls. 809353358Sdim GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee); 810353358Sdim Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, 811353358Sdim getPointerTy(DAG.getDataLayout()), 812353358Sdim GA->getOffset()); 813353358Sdim Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL, 814353358Sdim getPointerTy(DAG.getDataLayout()), Callee); 815353358Sdim } 816353358Sdim 817296417Sdim // Compute the operands for the CALLn node. 818296417Sdim SmallVector<SDValue, 16> Ops; 819296417Sdim Ops.push_back(Chain); 820296417Sdim Ops.push_back(Callee); 821296417Sdim 822296417Sdim // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs 823296417Sdim // isn't reliable. 824296417Sdim Ops.append(OutVals.begin(), 825296417Sdim IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end()); 826309124Sdim // Add a pointer to the vararg buffer. 827344779Sdim if (IsVarArg) 828344779Sdim Ops.push_back(FINode); 829296417Sdim 830309124Sdim SmallVector<EVT, 8> InTys; 831296417Sdim for (const auto &In : Ins) { 832296417Sdim assert(!In.Flags.isByVal() && "byval is not valid for return values"); 833296417Sdim assert(!In.Flags.isNest() && "nest is not valid for return values"); 834296417Sdim if (In.Flags.isInAlloca()) 835296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values"); 836296417Sdim if (In.Flags.isInConsecutiveRegs()) 837296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values"); 838296417Sdim if (In.Flags.isInConsecutiveRegsLast()) 839296417Sdim fail(DL, DAG, 840296417Sdim "WebAssembly hasn't implemented cons regs last return values"); 841296417Sdim // Ignore In.getOrigAlign() because all our arguments are passed in 842296417Sdim // registers. 843309124Sdim InTys.push_back(In.VT); 844296417Sdim } 845353358Sdim 846353358Sdim if (CLI.IsTailCall) { 847353358Sdim // ret_calls do not return values to the current frame 848353358Sdim SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 849353358Sdim return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops); 850353358Sdim } 851353358Sdim 852309124Sdim InTys.push_back(MVT::Other); 853309124Sdim SDVTList InTyList = DAG.getVTList(InTys); 854296417Sdim SDValue Res = 855296417Sdim DAG.getNode(Ins.empty() ? WebAssemblyISD::CALL0 : WebAssemblyISD::CALL1, 856309124Sdim DL, InTyList, Ops); 857296417Sdim if (Ins.empty()) { 858296417Sdim Chain = Res; 859296417Sdim } else { 860296417Sdim InVals.push_back(Res); 861296417Sdim Chain = Res.getValue(1); 862296417Sdim } 863296417Sdim 864296417Sdim return Chain; 865296417Sdim} 866296417Sdim 867296417Sdimbool WebAssemblyTargetLowering::CanLowerReturn( 868296417Sdim CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/, 869296417Sdim const SmallVectorImpl<ISD::OutputArg> &Outs, 870296417Sdim LLVMContext & /*Context*/) const { 871360784Sdim // WebAssembly can only handle returning tuples with multivalue enabled 872360784Sdim return Subtarget->hasMultivalue() || Outs.size() <= 1; 873296417Sdim} 874296417Sdim 875296417SdimSDValue WebAssemblyTargetLowering::LowerReturn( 876296417Sdim SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/, 877296417Sdim const SmallVectorImpl<ISD::OutputArg> &Outs, 878309124Sdim const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, 879296417Sdim SelectionDAG &DAG) const { 880360784Sdim assert((Subtarget->hasMultivalue() || Outs.size() <= 1) && 881360784Sdim "MVP WebAssembly can only return up to one value"); 882353358Sdim if (!callingConvSupported(CallConv)) 883296417Sdim fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); 884296417Sdim 885296417Sdim SmallVector<SDValue, 4> RetOps(1, Chain); 886296417Sdim RetOps.append(OutVals.begin(), OutVals.end()); 887296417Sdim Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps); 888296417Sdim 889296417Sdim // Record the number and types of the return values. 890296417Sdim for (const ISD::OutputArg &Out : Outs) { 891296417Sdim assert(!Out.Flags.isByVal() && "byval is not valid for return values"); 892296417Sdim assert(!Out.Flags.isNest() && "nest is not valid for return values"); 893296417Sdim assert(Out.IsFixed && "non-fixed return value is not valid"); 894296417Sdim if (Out.Flags.isInAlloca()) 895296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented inalloca results"); 896296417Sdim if (Out.Flags.isInConsecutiveRegs()) 897296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs results"); 898296417Sdim if (Out.Flags.isInConsecutiveRegsLast()) 899296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results"); 900296417Sdim } 901296417Sdim 902296417Sdim return Chain; 903296417Sdim} 904296417Sdim 905296417SdimSDValue WebAssemblyTargetLowering::LowerFormalArguments( 906309124Sdim SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 907309124Sdim const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 908309124Sdim SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 909353358Sdim if (!callingConvSupported(CallConv)) 910314564Sdim fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); 911314564Sdim 912296417Sdim MachineFunction &MF = DAG.getMachineFunction(); 913309124Sdim auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>(); 914296417Sdim 915296417Sdim // Set up the incoming ARGUMENTS value, which serves to represent the liveness 916296417Sdim // of the incoming values before they're represented by virtual registers. 917296417Sdim MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS); 918296417Sdim 919296417Sdim for (const ISD::InputArg &In : Ins) { 920296417Sdim if (In.Flags.isInAlloca()) 921296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); 922296417Sdim if (In.Flags.isNest()) 923296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); 924296417Sdim if (In.Flags.isInConsecutiveRegs()) 925296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); 926296417Sdim if (In.Flags.isInConsecutiveRegsLast()) 927296417Sdim fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); 928296417Sdim // Ignore In.getOrigAlign() because all our arguments are passed in 929296417Sdim // registers. 930344779Sdim InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT, 931344779Sdim DAG.getTargetConstant(InVals.size(), 932344779Sdim DL, MVT::i32)) 933344779Sdim : DAG.getUNDEF(In.VT)); 934296417Sdim 935296417Sdim // Record the number and types of arguments. 936309124Sdim MFI->addParam(In.VT); 937296417Sdim } 938296417Sdim 939309124Sdim // Varargs are copied into a buffer allocated by the caller, and a pointer to 940309124Sdim // the buffer is passed as an argument. 941309124Sdim if (IsVarArg) { 942309124Sdim MVT PtrVT = getPointerTy(MF.getDataLayout()); 943360784Sdim Register VarargVreg = 944309124Sdim MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT)); 945309124Sdim MFI->setVarargBufferVreg(VarargVreg); 946309124Sdim Chain = DAG.getCopyToReg( 947309124Sdim Chain, DL, VarargVreg, 948309124Sdim DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT, 949309124Sdim DAG.getTargetConstant(Ins.size(), DL, MVT::i32))); 950309124Sdim MFI->addParam(PtrVT); 951309124Sdim } 952296417Sdim 953344779Sdim // Record the number and types of arguments and results. 954314564Sdim SmallVector<MVT, 4> Params; 955314564Sdim SmallVector<MVT, 4> Results; 956353358Sdim computeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(), 957344779Sdim DAG.getTarget(), Params, Results); 958314564Sdim for (MVT VT : Results) 959314564Sdim MFI->addResult(VT); 960344779Sdim // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify 961344779Sdim // the param logic here with ComputeSignatureVTs 962344779Sdim assert(MFI->getParams().size() == Params.size() && 963344779Sdim std::equal(MFI->getParams().begin(), MFI->getParams().end(), 964344779Sdim Params.begin())); 965314564Sdim 966296417Sdim return Chain; 967296417Sdim} 968296417Sdim 969353358Sdimvoid WebAssemblyTargetLowering::ReplaceNodeResults( 970353358Sdim SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 971353358Sdim switch (N->getOpcode()) { 972353358Sdim case ISD::SIGN_EXTEND_INREG: 973353358Sdim // Do not add any results, signifying that N should not be custom lowered 974353358Sdim // after all. This happens because simd128 turns on custom lowering for 975353358Sdim // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an 976353358Sdim // illegal type. 977353358Sdim break; 978353358Sdim default: 979353358Sdim llvm_unreachable( 980353358Sdim "ReplaceNodeResults not implemented for this op for WebAssembly!"); 981353358Sdim } 982353358Sdim} 983353358Sdim 984285163Sdim//===----------------------------------------------------------------------===// 985296417Sdim// Custom lowering hooks. 986285163Sdim//===----------------------------------------------------------------------===// 987285163Sdim 988296417SdimSDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op, 989296417Sdim SelectionDAG &DAG) const { 990309124Sdim SDLoc DL(Op); 991296417Sdim switch (Op.getOpcode()) { 992344779Sdim default: 993344779Sdim llvm_unreachable("unimplemented operation lowering"); 994344779Sdim return SDValue(); 995344779Sdim case ISD::FrameIndex: 996344779Sdim return LowerFrameIndex(Op, DAG); 997344779Sdim case ISD::GlobalAddress: 998344779Sdim return LowerGlobalAddress(Op, DAG); 999344779Sdim case ISD::ExternalSymbol: 1000344779Sdim return LowerExternalSymbol(Op, DAG); 1001344779Sdim case ISD::JumpTable: 1002344779Sdim return LowerJumpTable(Op, DAG); 1003344779Sdim case ISD::BR_JT: 1004344779Sdim return LowerBR_JT(Op, DAG); 1005344779Sdim case ISD::VASTART: 1006344779Sdim return LowerVASTART(Op, DAG); 1007344779Sdim case ISD::BlockAddress: 1008344779Sdim case ISD::BRIND: 1009344779Sdim fail(DL, DAG, "WebAssembly hasn't implemented computed gotos"); 1010344779Sdim return SDValue(); 1011353358Sdim case ISD::RETURNADDR: 1012353358Sdim return LowerRETURNADDR(Op, DAG); 1013344779Sdim case ISD::FRAMEADDR: 1014344779Sdim return LowerFRAMEADDR(Op, DAG); 1015344779Sdim case ISD::CopyToReg: 1016344779Sdim return LowerCopyToReg(Op, DAG); 1017344779Sdim case ISD::EXTRACT_VECTOR_ELT: 1018344779Sdim case ISD::INSERT_VECTOR_ELT: 1019344779Sdim return LowerAccessVectorElement(Op, DAG); 1020344779Sdim case ISD::INTRINSIC_VOID: 1021353358Sdim case ISD::INTRINSIC_WO_CHAIN: 1022353358Sdim case ISD::INTRINSIC_W_CHAIN: 1023353358Sdim return LowerIntrinsic(Op, DAG); 1024344779Sdim case ISD::SIGN_EXTEND_INREG: 1025344779Sdim return LowerSIGN_EXTEND_INREG(Op, DAG); 1026353358Sdim case ISD::BUILD_VECTOR: 1027353358Sdim return LowerBUILD_VECTOR(Op, DAG); 1028344779Sdim case ISD::VECTOR_SHUFFLE: 1029344779Sdim return LowerVECTOR_SHUFFLE(Op, DAG); 1030360784Sdim case ISD::SETCC: 1031360784Sdim return LowerSETCC(Op, DAG); 1032344779Sdim case ISD::SHL: 1033344779Sdim case ISD::SRA: 1034344779Sdim case ISD::SRL: 1035344779Sdim return LowerShift(Op, DAG); 1036296417Sdim } 1037296417Sdim} 1038296417Sdim 1039309124SdimSDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op, 1040309124Sdim SelectionDAG &DAG) const { 1041309124Sdim SDValue Src = Op.getOperand(2); 1042309124Sdim if (isa<FrameIndexSDNode>(Src.getNode())) { 1043309124Sdim // CopyToReg nodes don't support FrameIndex operands. Other targets select 1044309124Sdim // the FI to some LEA-like instruction, but since we don't have that, we 1045309124Sdim // need to insert some kind of instruction that can take an FI operand and 1046309124Sdim // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy 1047344779Sdim // local.copy between Op and its FI operand. 1048309124Sdim SDValue Chain = Op.getOperand(0); 1049309124Sdim SDLoc DL(Op); 1050309124Sdim unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg(); 1051309124Sdim EVT VT = Src.getValueType(); 1052344779Sdim SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32 1053344779Sdim : WebAssembly::COPY_I64, 1054344779Sdim DL, VT, Src), 1055344779Sdim 0); 1056309124Sdim return Op.getNode()->getNumValues() == 1 1057309124Sdim ? DAG.getCopyToReg(Chain, DL, Reg, Copy) 1058344779Sdim : DAG.getCopyToReg(Chain, DL, Reg, Copy, 1059344779Sdim Op.getNumOperands() == 4 ? Op.getOperand(3) 1060344779Sdim : SDValue()); 1061309124Sdim } 1062309124Sdim return SDValue(); 1063309124Sdim} 1064309124Sdim 1065296417SdimSDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op, 1066296417Sdim SelectionDAG &DAG) const { 1067296417Sdim int FI = cast<FrameIndexSDNode>(Op)->getIndex(); 1068296417Sdim return DAG.getTargetFrameIndex(FI, Op.getValueType()); 1069296417Sdim} 1070296417Sdim 1071353358SdimSDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op, 1072353358Sdim SelectionDAG &DAG) const { 1073353358Sdim SDLoc DL(Op); 1074353358Sdim 1075353358Sdim if (!Subtarget->getTargetTriple().isOSEmscripten()) { 1076353358Sdim fail(DL, DAG, 1077353358Sdim "Non-Emscripten WebAssembly hasn't implemented " 1078353358Sdim "__builtin_return_address"); 1079353358Sdim return SDValue(); 1080353358Sdim } 1081353358Sdim 1082353358Sdim if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 1083353358Sdim return SDValue(); 1084353358Sdim 1085353358Sdim unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 1086360784Sdim MakeLibCallOptions CallOptions; 1087353358Sdim return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(), 1088360784Sdim {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL) 1089353358Sdim .first; 1090353358Sdim} 1091353358Sdim 1092309124SdimSDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op, 1093309124Sdim SelectionDAG &DAG) const { 1094309124Sdim // Non-zero depths are not supported by WebAssembly currently. Use the 1095309124Sdim // legalizer's default expansion, which is to return 0 (what this function is 1096309124Sdim // documented to do). 1097309124Sdim if (Op.getConstantOperandVal(0) > 0) 1098309124Sdim return SDValue(); 1099309124Sdim 1100314564Sdim DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true); 1101309124Sdim EVT VT = Op.getValueType(); 1102360784Sdim Register FP = 1103309124Sdim Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction()); 1104309124Sdim return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT); 1105309124Sdim} 1106309124Sdim 1107296417SdimSDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op, 1108296417Sdim SelectionDAG &DAG) const { 1109296417Sdim SDLoc DL(Op); 1110296417Sdim const auto *GA = cast<GlobalAddressSDNode>(Op); 1111296417Sdim EVT VT = Op.getValueType(); 1112296417Sdim assert(GA->getTargetFlags() == 0 && 1113296417Sdim "Unexpected target flags on generic GlobalAddressSDNode"); 1114296417Sdim if (GA->getAddressSpace() != 0) 1115296417Sdim fail(DL, DAG, "WebAssembly only expects the 0 address space"); 1116353358Sdim 1117353358Sdim unsigned OperandFlags = 0; 1118353358Sdim if (isPositionIndependent()) { 1119353358Sdim const GlobalValue *GV = GA->getGlobal(); 1120353358Sdim if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) { 1121353358Sdim MachineFunction &MF = DAG.getMachineFunction(); 1122353358Sdim MVT PtrVT = getPointerTy(MF.getDataLayout()); 1123353358Sdim const char *BaseName; 1124353358Sdim if (GV->getValueType()->isFunctionTy()) { 1125353358Sdim BaseName = MF.createExternalSymbolName("__table_base"); 1126353358Sdim OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL; 1127353358Sdim } 1128353358Sdim else { 1129353358Sdim BaseName = MF.createExternalSymbolName("__memory_base"); 1130353358Sdim OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL; 1131353358Sdim } 1132353358Sdim SDValue BaseAddr = 1133353358Sdim DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, 1134353358Sdim DAG.getTargetExternalSymbol(BaseName, PtrVT)); 1135353358Sdim 1136353358Sdim SDValue SymAddr = DAG.getNode( 1137353358Sdim WebAssemblyISD::WrapperPIC, DL, VT, 1138353358Sdim DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(), 1139353358Sdim OperandFlags)); 1140353358Sdim 1141353358Sdim return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr); 1142353358Sdim } else { 1143353358Sdim OperandFlags = WebAssemblyII::MO_GOT; 1144353358Sdim } 1145353358Sdim } 1146353358Sdim 1147353358Sdim return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1148353358Sdim DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, 1149353358Sdim GA->getOffset(), OperandFlags)); 1150296417Sdim} 1151296417Sdim 1152344779SdimSDValue 1153344779SdimWebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op, 1154344779Sdim SelectionDAG &DAG) const { 1155296417Sdim SDLoc DL(Op); 1156296417Sdim const auto *ES = cast<ExternalSymbolSDNode>(Op); 1157296417Sdim EVT VT = Op.getValueType(); 1158296417Sdim assert(ES->getTargetFlags() == 0 && 1159296417Sdim "Unexpected target flags on generic ExternalSymbolSDNode"); 1160353358Sdim return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1161353358Sdim DAG.getTargetExternalSymbol(ES->getSymbol(), VT)); 1162296417Sdim} 1163296417Sdim 1164296417SdimSDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op, 1165296417Sdim SelectionDAG &DAG) const { 1166296417Sdim // There's no need for a Wrapper node because we always incorporate a jump 1167309124Sdim // table operand into a BR_TABLE instruction, rather than ever 1168296417Sdim // materializing it in a register. 1169296417Sdim const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); 1170296417Sdim return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(), 1171296417Sdim JT->getTargetFlags()); 1172296417Sdim} 1173296417Sdim 1174296417SdimSDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op, 1175296417Sdim SelectionDAG &DAG) const { 1176296417Sdim SDLoc DL(Op); 1177296417Sdim SDValue Chain = Op.getOperand(0); 1178296417Sdim const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1)); 1179296417Sdim SDValue Index = Op.getOperand(2); 1180296417Sdim assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags"); 1181296417Sdim 1182296417Sdim SmallVector<SDValue, 8> Ops; 1183296417Sdim Ops.push_back(Chain); 1184296417Sdim Ops.push_back(Index); 1185296417Sdim 1186296417Sdim MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo(); 1187296417Sdim const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs; 1188296417Sdim 1189309124Sdim // Add an operand for each case. 1190344779Sdim for (auto MBB : MBBs) 1191344779Sdim Ops.push_back(DAG.getBasicBlock(MBB)); 1192309124Sdim 1193296417Sdim // TODO: For now, we just pick something arbitrary for a default case for now. 1194296417Sdim // We really want to sniff out the guard and put in the real default case (and 1195296417Sdim // delete the guard). 1196296417Sdim Ops.push_back(DAG.getBasicBlock(MBBs[0])); 1197296417Sdim 1198309124Sdim return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops); 1199296417Sdim} 1200296417Sdim 1201296417SdimSDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op, 1202296417Sdim SelectionDAG &DAG) const { 1203296417Sdim SDLoc DL(Op); 1204296417Sdim EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout()); 1205296417Sdim 1206309124Sdim auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>(); 1207296417Sdim const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 1208309124Sdim 1209309124Sdim SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL, 1210309124Sdim MFI->getVarargBufferVreg(), PtrVT); 1211309124Sdim return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1), 1212309124Sdim MachinePointerInfo(SV), 0); 1213296417Sdim} 1214296417Sdim 1215353358SdimSDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op, 1216353358Sdim SelectionDAG &DAG) const { 1217353358Sdim MachineFunction &MF = DAG.getMachineFunction(); 1218353358Sdim unsigned IntNo; 1219353358Sdim switch (Op.getOpcode()) { 1220353358Sdim case ISD::INTRINSIC_VOID: 1221353358Sdim case ISD::INTRINSIC_W_CHAIN: 1222353358Sdim IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 1223353358Sdim break; 1224353358Sdim case ISD::INTRINSIC_WO_CHAIN: 1225353358Sdim IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 1226353358Sdim break; 1227353358Sdim default: 1228353358Sdim llvm_unreachable("Invalid intrinsic"); 1229353358Sdim } 1230341825Sdim SDLoc DL(Op); 1231353358Sdim 1232341825Sdim switch (IntNo) { 1233341825Sdim default: 1234353358Sdim return SDValue(); // Don't custom lower most intrinsics. 1235341825Sdim 1236344779Sdim case Intrinsic::wasm_lsda: { 1237344779Sdim EVT VT = Op.getValueType(); 1238344779Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1239344779Sdim MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 1240344779Sdim auto &Context = MF.getMMI().getContext(); 1241344779Sdim MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") + 1242344779Sdim Twine(MF.getFunctionNumber())); 1243344779Sdim return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, 1244344779Sdim DAG.getMCSymbol(S, PtrVT)); 1245341825Sdim } 1246341825Sdim 1247344779Sdim case Intrinsic::wasm_throw: { 1248353358Sdim // We only support C++ exceptions for now 1249344779Sdim int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue(); 1250353358Sdim if (Tag != CPP_EXCEPTION) 1251344779Sdim llvm_unreachable("Invalid tag!"); 1252353358Sdim const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1253353358Sdim MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 1254353358Sdim const char *SymName = MF.createExternalSymbolName("__cpp_exception"); 1255353358Sdim SDValue SymNode = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, 1256353358Sdim DAG.getTargetExternalSymbol(SymName, PtrVT)); 1257353358Sdim return DAG.getNode(WebAssemblyISD::THROW, DL, 1258353358Sdim MVT::Other, // outchain type 1259353358Sdim { 1260353358Sdim Op.getOperand(0), // inchain 1261353358Sdim SymNode, // exception symbol 1262353358Sdim Op.getOperand(3) // thrown value 1263353358Sdim }); 1264344779Sdim } 1265344779Sdim } 1266344779Sdim} 1267344779Sdim 1268344779SdimSDValue 1269344779SdimWebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, 1270344779Sdim SelectionDAG &DAG) const { 1271353358Sdim SDLoc DL(Op); 1272344779Sdim // If sign extension operations are disabled, allow sext_inreg only if operand 1273344779Sdim // is a vector extract. SIMD does not depend on sign extension operations, but 1274344779Sdim // allowing sext_inreg in this context lets us have simple patterns to select 1275344779Sdim // extract_lane_s instructions. Expanding sext_inreg everywhere would be 1276344779Sdim // simpler in this file, but would necessitate large and brittle patterns to 1277344779Sdim // undo the expansion and select extract_lane_s instructions. 1278344779Sdim assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128()); 1279353358Sdim if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT) { 1280353358Sdim const SDValue &Extract = Op.getOperand(0); 1281353358Sdim MVT VecT = Extract.getOperand(0).getSimpleValueType(); 1282353358Sdim MVT ExtractedLaneT = static_cast<VTSDNode *>(Op.getOperand(1).getNode()) 1283353358Sdim ->getVT() 1284353358Sdim .getSimpleVT(); 1285353358Sdim MVT ExtractedVecT = 1286353358Sdim MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits()); 1287353358Sdim if (ExtractedVecT == VecT) 1288353358Sdim return Op; 1289353358Sdim // Bitcast vector to appropriate type to ensure ISel pattern coverage 1290353358Sdim const SDValue &Index = Extract.getOperand(1); 1291353358Sdim unsigned IndexVal = 1292353358Sdim static_cast<ConstantSDNode *>(Index.getNode())->getZExtValue(); 1293353358Sdim unsigned Scale = 1294353358Sdim ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements(); 1295353358Sdim assert(Scale > 1); 1296353358Sdim SDValue NewIndex = 1297353358Sdim DAG.getConstant(IndexVal * Scale, DL, Index.getValueType()); 1298353358Sdim SDValue NewExtract = DAG.getNode( 1299353358Sdim ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(), 1300353358Sdim DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex); 1301353358Sdim return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), 1302353358Sdim NewExtract, Op.getOperand(1)); 1303353358Sdim } 1304344779Sdim // Otherwise expand 1305344779Sdim return SDValue(); 1306344779Sdim} 1307344779Sdim 1308353358SdimSDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op, 1309353358Sdim SelectionDAG &DAG) const { 1310353358Sdim SDLoc DL(Op); 1311353358Sdim const EVT VecT = Op.getValueType(); 1312353358Sdim const EVT LaneT = Op.getOperand(0).getValueType(); 1313353358Sdim const size_t Lanes = Op.getNumOperands(); 1314360784Sdim bool CanSwizzle = Subtarget->hasUnimplementedSIMD128() && VecT == MVT::v16i8; 1315360784Sdim 1316360784Sdim // BUILD_VECTORs are lowered to the instruction that initializes the highest 1317360784Sdim // possible number of lanes at once followed by a sequence of replace_lane 1318360784Sdim // instructions to individually initialize any remaining lanes. 1319360784Sdim 1320360784Sdim // TODO: Tune this. For example, lanewise swizzling is very expensive, so 1321360784Sdim // swizzled lanes should be given greater weight. 1322360784Sdim 1323360784Sdim // TODO: Investigate building vectors by shuffling together vectors built by 1324360784Sdim // separately specialized means. 1325360784Sdim 1326353358Sdim auto IsConstant = [](const SDValue &V) { 1327353358Sdim return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP; 1328353358Sdim }; 1329353358Sdim 1330360784Sdim // Returns the source vector and index vector pair if they exist. Checks for: 1331360784Sdim // (extract_vector_elt 1332360784Sdim // $src, 1333360784Sdim // (sign_extend_inreg (extract_vector_elt $indices, $i)) 1334360784Sdim // ) 1335360784Sdim auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) { 1336360784Sdim auto Bail = std::make_pair(SDValue(), SDValue()); 1337360784Sdim if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1338360784Sdim return Bail; 1339360784Sdim const SDValue &SwizzleSrc = Lane->getOperand(0); 1340360784Sdim const SDValue &IndexExt = Lane->getOperand(1); 1341360784Sdim if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG) 1342360784Sdim return Bail; 1343360784Sdim const SDValue &Index = IndexExt->getOperand(0); 1344360784Sdim if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1345360784Sdim return Bail; 1346360784Sdim const SDValue &SwizzleIndices = Index->getOperand(0); 1347360784Sdim if (SwizzleSrc.getValueType() != MVT::v16i8 || 1348360784Sdim SwizzleIndices.getValueType() != MVT::v16i8 || 1349360784Sdim Index->getOperand(1)->getOpcode() != ISD::Constant || 1350360784Sdim Index->getConstantOperandVal(1) != I) 1351360784Sdim return Bail; 1352360784Sdim return std::make_pair(SwizzleSrc, SwizzleIndices); 1353360784Sdim }; 1354360784Sdim 1355360784Sdim using ValueEntry = std::pair<SDValue, size_t>; 1356360784Sdim SmallVector<ValueEntry, 16> SplatValueCounts; 1357360784Sdim 1358360784Sdim using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>; 1359360784Sdim SmallVector<SwizzleEntry, 16> SwizzleCounts; 1360360784Sdim 1361360784Sdim auto AddCount = [](auto &Counts, const auto &Val) { 1362360784Sdim auto CountIt = std::find_if(Counts.begin(), Counts.end(), 1363360784Sdim [&Val](auto E) { return E.first == Val; }); 1364360784Sdim if (CountIt == Counts.end()) { 1365360784Sdim Counts.emplace_back(Val, 1); 1366353358Sdim } else { 1367353358Sdim CountIt->second++; 1368353358Sdim } 1369360784Sdim }; 1370360784Sdim 1371360784Sdim auto GetMostCommon = [](auto &Counts) { 1372360784Sdim auto CommonIt = 1373360784Sdim std::max_element(Counts.begin(), Counts.end(), 1374360784Sdim [](auto A, auto B) { return A.second < B.second; }); 1375360784Sdim assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector"); 1376360784Sdim return *CommonIt; 1377360784Sdim }; 1378360784Sdim 1379360784Sdim size_t NumConstantLanes = 0; 1380360784Sdim 1381360784Sdim // Count eligible lanes for each type of vector creation op 1382360784Sdim for (size_t I = 0; I < Lanes; ++I) { 1383360784Sdim const SDValue &Lane = Op->getOperand(I); 1384360784Sdim if (Lane.isUndef()) 1385360784Sdim continue; 1386360784Sdim 1387360784Sdim AddCount(SplatValueCounts, Lane); 1388360784Sdim 1389360784Sdim if (IsConstant(Lane)) { 1390360784Sdim NumConstantLanes++; 1391360784Sdim } else if (CanSwizzle) { 1392360784Sdim auto SwizzleSrcs = GetSwizzleSrcs(I, Lane); 1393360784Sdim if (SwizzleSrcs.first) 1394360784Sdim AddCount(SwizzleCounts, SwizzleSrcs); 1395360784Sdim } 1396353358Sdim } 1397353358Sdim 1398360784Sdim SDValue SplatValue; 1399360784Sdim size_t NumSplatLanes; 1400360784Sdim std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts); 1401360784Sdim 1402360784Sdim SDValue SwizzleSrc; 1403360784Sdim SDValue SwizzleIndices; 1404360784Sdim size_t NumSwizzleLanes = 0; 1405360784Sdim if (SwizzleCounts.size()) 1406360784Sdim std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices), 1407360784Sdim NumSwizzleLanes) = GetMostCommon(SwizzleCounts); 1408360784Sdim 1409360784Sdim // Predicate returning true if the lane is properly initialized by the 1410360784Sdim // original instruction 1411360784Sdim std::function<bool(size_t, const SDValue &)> IsLaneConstructed; 1412360784Sdim SDValue Result; 1413353358Sdim if (Subtarget->hasUnimplementedSIMD128()) { 1414360784Sdim // Prefer swizzles over vector consts over splats 1415360784Sdim if (NumSwizzleLanes >= NumSplatLanes && 1416360784Sdim NumSwizzleLanes >= NumConstantLanes) { 1417360784Sdim Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc, 1418360784Sdim SwizzleIndices); 1419360784Sdim auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices); 1420360784Sdim IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) { 1421360784Sdim return Swizzled == GetSwizzleSrcs(I, Lane); 1422360784Sdim }; 1423360784Sdim } else if (NumConstantLanes >= NumSplatLanes) { 1424353358Sdim SmallVector<SDValue, 16> ConstLanes; 1425353358Sdim for (const SDValue &Lane : Op->op_values()) { 1426353358Sdim if (IsConstant(Lane)) { 1427353358Sdim ConstLanes.push_back(Lane); 1428353358Sdim } else if (LaneT.isFloatingPoint()) { 1429353358Sdim ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT)); 1430353358Sdim } else { 1431353358Sdim ConstLanes.push_back(DAG.getConstant(0, DL, LaneT)); 1432353358Sdim } 1433353358Sdim } 1434360784Sdim Result = DAG.getBuildVector(VecT, DL, ConstLanes); 1435360784Sdim IsLaneConstructed = [&](size_t _, const SDValue &Lane) { 1436360784Sdim return IsConstant(Lane); 1437360784Sdim }; 1438353358Sdim } 1439353358Sdim } 1440360784Sdim if (!Result) { 1441360784Sdim // Use a splat, but possibly a load_splat 1442360784Sdim LoadSDNode *SplattedLoad; 1443360784Sdim if (Subtarget->hasUnimplementedSIMD128() && 1444360784Sdim (SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) && 1445360784Sdim SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) { 1446360784Sdim Result = DAG.getMemIntrinsicNode( 1447360784Sdim WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT), 1448360784Sdim {SplattedLoad->getChain(), SplattedLoad->getBasePtr(), 1449360784Sdim SplattedLoad->getOffset()}, 1450360784Sdim SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand()); 1451360784Sdim } else { 1452360784Sdim Result = DAG.getSplatBuildVector(VecT, DL, SplatValue); 1453360784Sdim } 1454360784Sdim IsLaneConstructed = [&](size_t _, const SDValue &Lane) { 1455360784Sdim return Lane == SplatValue; 1456360784Sdim }; 1457360784Sdim } 1458360784Sdim 1459360784Sdim // Add replace_lane instructions for any unhandled values 1460353358Sdim for (size_t I = 0; I < Lanes; ++I) { 1461353358Sdim const SDValue &Lane = Op->getOperand(I); 1462360784Sdim if (!Lane.isUndef() && !IsLaneConstructed(I, Lane)) 1463353358Sdim Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane, 1464353358Sdim DAG.getConstant(I, DL, MVT::i32)); 1465353358Sdim } 1466360784Sdim 1467353358Sdim return Result; 1468353358Sdim} 1469353358Sdim 1470344779SdimSDValue 1471344779SdimWebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, 1472344779Sdim SelectionDAG &DAG) const { 1473344779Sdim SDLoc DL(Op); 1474344779Sdim ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask(); 1475344779Sdim MVT VecType = Op.getOperand(0).getSimpleValueType(); 1476344779Sdim assert(VecType.is128BitVector() && "Unexpected shuffle vector type"); 1477344779Sdim size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8; 1478344779Sdim 1479344779Sdim // Space for two vector args and sixteen mask indices 1480344779Sdim SDValue Ops[18]; 1481344779Sdim size_t OpIdx = 0; 1482344779Sdim Ops[OpIdx++] = Op.getOperand(0); 1483344779Sdim Ops[OpIdx++] = Op.getOperand(1); 1484344779Sdim 1485344779Sdim // Expand mask indices to byte indices and materialize them as operands 1486353358Sdim for (int M : Mask) { 1487344779Sdim for (size_t J = 0; J < LaneBytes; ++J) { 1488344779Sdim // Lower undefs (represented by -1 in mask) to zero 1489353358Sdim uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J; 1490344779Sdim Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32); 1491344779Sdim } 1492344779Sdim } 1493344779Sdim 1494344779Sdim return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops); 1495344779Sdim} 1496344779Sdim 1497360784SdimSDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op, 1498360784Sdim SelectionDAG &DAG) const { 1499360784Sdim SDLoc DL(Op); 1500360784Sdim // The legalizer does not know how to expand the comparison modes of i64x2 1501360784Sdim // vectors because no comparison modes are supported. We could solve this by 1502360784Sdim // expanding all i64x2 SETCC nodes, but that seems to expand f64x2 SETCC nodes 1503360784Sdim // (which return i64x2 results) as well. So instead we manually unroll i64x2 1504360784Sdim // comparisons here. 1505360784Sdim assert(Subtarget->hasUnimplementedSIMD128()); 1506360784Sdim assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64); 1507360784Sdim SmallVector<SDValue, 2> LHS, RHS; 1508360784Sdim DAG.ExtractVectorElements(Op->getOperand(0), LHS); 1509360784Sdim DAG.ExtractVectorElements(Op->getOperand(1), RHS); 1510360784Sdim const SDValue &CC = Op->getOperand(2); 1511360784Sdim auto MakeLane = [&](unsigned I) { 1512360784Sdim return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I], 1513360784Sdim DAG.getConstant(uint64_t(-1), DL, MVT::i64), 1514360784Sdim DAG.getConstant(uint64_t(0), DL, MVT::i64), CC); 1515360784Sdim }; 1516360784Sdim return DAG.getBuildVector(Op->getValueType(0), DL, 1517360784Sdim {MakeLane(0), MakeLane(1)}); 1518360784Sdim} 1519360784Sdim 1520344779SdimSDValue 1521344779SdimWebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op, 1522344779Sdim SelectionDAG &DAG) const { 1523344779Sdim // Allow constant lane indices, expand variable lane indices 1524344779Sdim SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode(); 1525344779Sdim if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef()) 1526344779Sdim return Op; 1527344779Sdim else 1528344779Sdim // Perform default expansion 1529344779Sdim return SDValue(); 1530344779Sdim} 1531344779Sdim 1532353358Sdimstatic SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) { 1533344779Sdim EVT LaneT = Op.getSimpleValueType().getVectorElementType(); 1534344779Sdim // 32-bit and 64-bit unrolled shifts will have proper semantics 1535344779Sdim if (LaneT.bitsGE(MVT::i32)) 1536344779Sdim return DAG.UnrollVectorOp(Op.getNode()); 1537344779Sdim // Otherwise mask the shift value to get proper semantics from 32-bit shift 1538344779Sdim SDLoc DL(Op); 1539344779Sdim SDValue ShiftVal = Op.getOperand(1); 1540344779Sdim uint64_t MaskVal = LaneT.getSizeInBits() - 1; 1541344779Sdim SDValue MaskedShiftVal = DAG.getNode( 1542344779Sdim ISD::AND, // mask opcode 1543344779Sdim DL, ShiftVal.getValueType(), // masked value type 1544344779Sdim ShiftVal, // original shift value operand 1545344779Sdim DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand 1546344779Sdim ); 1547344779Sdim 1548344779Sdim return DAG.UnrollVectorOp( 1549344779Sdim DAG.getNode(Op.getOpcode(), // original shift opcode 1550344779Sdim DL, Op.getValueType(), // original return type 1551344779Sdim Op.getOperand(0), // original vector operand, 1552344779Sdim MaskedShiftVal // new masked shift value operand 1553344779Sdim ) 1554344779Sdim .getNode()); 1555344779Sdim} 1556344779Sdim 1557344779SdimSDValue WebAssemblyTargetLowering::LowerShift(SDValue Op, 1558344779Sdim SelectionDAG &DAG) const { 1559344779Sdim SDLoc DL(Op); 1560344779Sdim 1561344779Sdim // Only manually lower vector shifts 1562344779Sdim assert(Op.getSimpleValueType().isVector()); 1563344779Sdim 1564344779Sdim // Unroll non-splat vector shifts 1565344779Sdim BuildVectorSDNode *ShiftVec; 1566344779Sdim SDValue SplatVal; 1567344779Sdim if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) || 1568344779Sdim !(SplatVal = ShiftVec->getSplatValue())) 1569353358Sdim return unrollVectorShift(Op, DAG); 1570344779Sdim 1571344779Sdim // All splats except i64x2 const splats are handled by patterns 1572353358Sdim auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal); 1573344779Sdim if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64) 1574344779Sdim return Op; 1575344779Sdim 1576344779Sdim // i64x2 const splats are custom lowered to avoid unnecessary wraps 1577344779Sdim unsigned Opcode; 1578344779Sdim switch (Op.getOpcode()) { 1579344779Sdim case ISD::SHL: 1580344779Sdim Opcode = WebAssemblyISD::VEC_SHL; 1581344779Sdim break; 1582344779Sdim case ISD::SRA: 1583344779Sdim Opcode = WebAssemblyISD::VEC_SHR_S; 1584344779Sdim break; 1585344779Sdim case ISD::SRL: 1586344779Sdim Opcode = WebAssemblyISD::VEC_SHR_U; 1587344779Sdim break; 1588344779Sdim default: 1589344779Sdim llvm_unreachable("unexpected opcode"); 1590344779Sdim } 1591344779Sdim APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32); 1592344779Sdim return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), 1593344779Sdim DAG.getConstant(Shift, DL, MVT::i32)); 1594344779Sdim} 1595344779Sdim 1596285163Sdim//===----------------------------------------------------------------------===// 1597285163Sdim// WebAssembly Optimization Hooks 1598285163Sdim//===----------------------------------------------------------------------===// 1599