1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// Pass to verify generated machine code. The following is checked: 10// 11// Operand counts: All explicit operands must be present. 12// 13// Register classes: All physical and virtual register operands must be 14// compatible with the register class required by the instruction descriptor. 15// 16// Register live intervals: Registers must be defined only once, and must be 17// defined before use. 18// 19// The machine code verifier is enabled with the command-line option 20// -verify-machineinstrs. 21//===----------------------------------------------------------------------===// 22 23#include "llvm/ADT/BitVector.h" 24#include "llvm/ADT/DenseMap.h" 25#include "llvm/ADT/DenseSet.h" 26#include "llvm/ADT/DepthFirstIterator.h" 27#include "llvm/ADT/PostOrderIterator.h" 28#include "llvm/ADT/STLExtras.h" 29#include "llvm/ADT/SetOperations.h" 30#include "llvm/ADT/SmallPtrSet.h" 31#include "llvm/ADT/SmallVector.h" 32#include "llvm/ADT/StringRef.h" 33#include "llvm/ADT/Twine.h" 34#include "llvm/Analysis/EHPersonalities.h" 35#include "llvm/CodeGen/GlobalISel/RegisterBank.h" 36#include "llvm/CodeGen/LiveInterval.h" 37#include "llvm/CodeGen/LiveIntervalCalc.h" 38#include "llvm/CodeGen/LiveIntervals.h" 39#include "llvm/CodeGen/LiveStacks.h" 40#include "llvm/CodeGen/LiveVariables.h" 41#include "llvm/CodeGen/MachineBasicBlock.h" 42#include "llvm/CodeGen/MachineFrameInfo.h" 43#include "llvm/CodeGen/MachineFunction.h" 44#include "llvm/CodeGen/MachineFunctionPass.h" 45#include "llvm/CodeGen/MachineInstr.h" 46#include "llvm/CodeGen/MachineInstrBundle.h" 47#include "llvm/CodeGen/MachineMemOperand.h" 48#include "llvm/CodeGen/MachineOperand.h" 49#include "llvm/CodeGen/MachineRegisterInfo.h" 50#include "llvm/CodeGen/PseudoSourceValue.h" 51#include "llvm/CodeGen/SlotIndexes.h" 52#include "llvm/CodeGen/StackMaps.h" 53#include "llvm/CodeGen/TargetInstrInfo.h" 54#include "llvm/CodeGen/TargetOpcodes.h" 55#include "llvm/CodeGen/TargetRegisterInfo.h" 56#include "llvm/CodeGen/TargetSubtargetInfo.h" 57#include "llvm/IR/BasicBlock.h" 58#include "llvm/IR/Function.h" 59#include "llvm/IR/InlineAsm.h" 60#include "llvm/IR/Instructions.h" 61#include "llvm/InitializePasses.h" 62#include "llvm/MC/LaneBitmask.h" 63#include "llvm/MC/MCAsmInfo.h" 64#include "llvm/MC/MCInstrDesc.h" 65#include "llvm/MC/MCRegisterInfo.h" 66#include "llvm/MC/MCTargetOptions.h" 67#include "llvm/Pass.h" 68#include "llvm/Support/Casting.h" 69#include "llvm/Support/ErrorHandling.h" 70#include "llvm/Support/LowLevelTypeImpl.h" 71#include "llvm/Support/MathExtras.h" 72#include "llvm/Support/raw_ostream.h" 73#include "llvm/Target/TargetMachine.h" 74#include <algorithm> 75#include <cassert> 76#include <cstddef> 77#include <cstdint> 78#include <iterator> 79#include <string> 80#include <utility> 81 82using namespace llvm; 83 84namespace { 85 86 struct MachineVerifier { 87 MachineVerifier(Pass *pass, const char *b) : PASS(pass), Banner(b) {} 88 89 unsigned verify(const MachineFunction &MF); 90 91 Pass *const PASS; 92 const char *Banner; 93 const MachineFunction *MF; 94 const TargetMachine *TM; 95 const TargetInstrInfo *TII; 96 const TargetRegisterInfo *TRI; 97 const MachineRegisterInfo *MRI; 98 99 unsigned foundErrors; 100 101 // Avoid querying the MachineFunctionProperties for each operand. 102 bool isFunctionRegBankSelected; 103 bool isFunctionSelected; 104 105 using RegVector = SmallVector<Register, 16>; 106 using RegMaskVector = SmallVector<const uint32_t *, 4>; 107 using RegSet = DenseSet<Register>; 108 using RegMap = DenseMap<Register, const MachineInstr *>; 109 using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>; 110 111 const MachineInstr *FirstNonPHI; 112 const MachineInstr *FirstTerminator; 113 BlockSet FunctionBlocks; 114 115 BitVector regsReserved; 116 RegSet regsLive; 117 RegVector regsDefined, regsDead, regsKilled; 118 RegMaskVector regMasks; 119 120 SlotIndex lastIndex; 121 122 // Add Reg and any sub-registers to RV 123 void addRegWithSubRegs(RegVector &RV, Register Reg) { 124 RV.push_back(Reg); 125 if (Reg.isPhysical()) 126 append_range(RV, TRI->subregs(Reg.asMCReg())); 127 } 128 129 struct BBInfo { 130 // Is this MBB reachable from the MF entry point? 131 bool reachable = false; 132 133 // Vregs that must be live in because they are used without being 134 // defined. Map value is the user. vregsLiveIn doesn't include regs 135 // that only are used by PHI nodes. 136 RegMap vregsLiveIn; 137 138 // Regs killed in MBB. They may be defined again, and will then be in both 139 // regsKilled and regsLiveOut. 140 RegSet regsKilled; 141 142 // Regs defined in MBB and live out. Note that vregs passing through may 143 // be live out without being mentioned here. 144 RegSet regsLiveOut; 145 146 // Vregs that pass through MBB untouched. This set is disjoint from 147 // regsKilled and regsLiveOut. 148 RegSet vregsPassed; 149 150 // Vregs that must pass through MBB because they are needed by a successor 151 // block. This set is disjoint from regsLiveOut. 152 RegSet vregsRequired; 153 154 // Set versions of block's predecessor and successor lists. 155 BlockSet Preds, Succs; 156 157 BBInfo() = default; 158 159 // Add register to vregsRequired if it belongs there. Return true if 160 // anything changed. 161 bool addRequired(Register Reg) { 162 if (!Reg.isVirtual()) 163 return false; 164 if (regsLiveOut.count(Reg)) 165 return false; 166 return vregsRequired.insert(Reg).second; 167 } 168 169 // Same for a full set. 170 bool addRequired(const RegSet &RS) { 171 bool Changed = false; 172 for (Register Reg : RS) 173 Changed |= addRequired(Reg); 174 return Changed; 175 } 176 177 // Same for a full map. 178 bool addRequired(const RegMap &RM) { 179 bool Changed = false; 180 for (const auto &I : RM) 181 Changed |= addRequired(I.first); 182 return Changed; 183 } 184 185 // Live-out registers are either in regsLiveOut or vregsPassed. 186 bool isLiveOut(Register Reg) const { 187 return regsLiveOut.count(Reg) || vregsPassed.count(Reg); 188 } 189 }; 190 191 // Extra register info per MBB. 192 DenseMap<const MachineBasicBlock*, BBInfo> MBBInfoMap; 193 194 bool isReserved(Register Reg) { 195 return Reg.id() < regsReserved.size() && regsReserved.test(Reg.id()); 196 } 197 198 bool isAllocatable(Register Reg) const { 199 return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) && 200 !regsReserved.test(Reg.id()); 201 } 202 203 // Analysis information if available 204 LiveVariables *LiveVars; 205 LiveIntervals *LiveInts; 206 LiveStacks *LiveStks; 207 SlotIndexes *Indexes; 208 209 void visitMachineFunctionBefore(); 210 void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); 211 void visitMachineBundleBefore(const MachineInstr *MI); 212 213 bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI); 214 void verifyPreISelGenericInstruction(const MachineInstr *MI); 215 void visitMachineInstrBefore(const MachineInstr *MI); 216 void visitMachineOperand(const MachineOperand *MO, unsigned MONum); 217 void visitMachineBundleAfter(const MachineInstr *MI); 218 void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB); 219 void visitMachineFunctionAfter(); 220 221 void report(const char *msg, const MachineFunction *MF); 222 void report(const char *msg, const MachineBasicBlock *MBB); 223 void report(const char *msg, const MachineInstr *MI); 224 void report(const char *msg, const MachineOperand *MO, unsigned MONum, 225 LLT MOVRegType = LLT{}); 226 void report(const Twine &Msg, const MachineInstr *MI); 227 228 void report_context(const LiveInterval &LI) const; 229 void report_context(const LiveRange &LR, Register VRegUnit, 230 LaneBitmask LaneMask) const; 231 void report_context(const LiveRange::Segment &S) const; 232 void report_context(const VNInfo &VNI) const; 233 void report_context(SlotIndex Pos) const; 234 void report_context(MCPhysReg PhysReg) const; 235 void report_context_liverange(const LiveRange &LR) const; 236 void report_context_lanemask(LaneBitmask LaneMask) const; 237 void report_context_vreg(Register VReg) const; 238 void report_context_vreg_regunit(Register VRegOrUnit) const; 239 240 void verifyInlineAsm(const MachineInstr *MI); 241 242 void checkLiveness(const MachineOperand *MO, unsigned MONum); 243 void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum, 244 SlotIndex UseIdx, const LiveRange &LR, 245 Register VRegOrUnit, 246 LaneBitmask LaneMask = LaneBitmask::getNone()); 247 void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum, 248 SlotIndex DefIdx, const LiveRange &LR, 249 Register VRegOrUnit, bool SubRangeCheck = false, 250 LaneBitmask LaneMask = LaneBitmask::getNone()); 251 252 void markReachable(const MachineBasicBlock *MBB); 253 void calcRegsPassed(); 254 void checkPHIOps(const MachineBasicBlock &MBB); 255 256 void calcRegsRequired(); 257 void verifyLiveVariables(); 258 void verifyLiveIntervals(); 259 void verifyLiveInterval(const LiveInterval&); 260 void verifyLiveRangeValue(const LiveRange &, const VNInfo *, Register, 261 LaneBitmask); 262 void verifyLiveRangeSegment(const LiveRange &, 263 const LiveRange::const_iterator I, Register, 264 LaneBitmask); 265 void verifyLiveRange(const LiveRange &, Register, 266 LaneBitmask LaneMask = LaneBitmask::getNone()); 267 268 void verifyStackFrame(); 269 270 void verifySlotIndexes() const; 271 void verifyProperties(const MachineFunction &MF); 272 }; 273 274 struct MachineVerifierPass : public MachineFunctionPass { 275 static char ID; // Pass ID, replacement for typeid 276 277 const std::string Banner; 278 279 MachineVerifierPass(std::string banner = std::string()) 280 : MachineFunctionPass(ID), Banner(std::move(banner)) { 281 initializeMachineVerifierPassPass(*PassRegistry::getPassRegistry()); 282 } 283 284 void getAnalysisUsage(AnalysisUsage &AU) const override { 285 AU.setPreservesAll(); 286 MachineFunctionPass::getAnalysisUsage(AU); 287 } 288 289 bool runOnMachineFunction(MachineFunction &MF) override { 290 unsigned FoundErrors = MachineVerifier(this, Banner.c_str()).verify(MF); 291 if (FoundErrors) 292 report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors."); 293 return false; 294 } 295 }; 296 297} // end anonymous namespace 298 299char MachineVerifierPass::ID = 0; 300 301INITIALIZE_PASS(MachineVerifierPass, "machineverifier", 302 "Verify generated machine code", false, false) 303 304FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) { 305 return new MachineVerifierPass(Banner); 306} 307 308void llvm::verifyMachineFunction(MachineFunctionAnalysisManager *, 309 const std::string &Banner, 310 const MachineFunction &MF) { 311 // TODO: Use MFAM after porting below analyses. 312 // LiveVariables *LiveVars; 313 // LiveIntervals *LiveInts; 314 // LiveStacks *LiveStks; 315 // SlotIndexes *Indexes; 316 unsigned FoundErrors = MachineVerifier(nullptr, Banner.c_str()).verify(MF); 317 if (FoundErrors) 318 report_fatal_error("Found " + Twine(FoundErrors) + " machine code errors."); 319} 320 321bool MachineFunction::verify(Pass *p, const char *Banner, bool AbortOnErrors) 322 const { 323 MachineFunction &MF = const_cast<MachineFunction&>(*this); 324 unsigned FoundErrors = MachineVerifier(p, Banner).verify(MF); 325 if (AbortOnErrors && FoundErrors) 326 report_fatal_error("Found "+Twine(FoundErrors)+" machine code errors."); 327 return FoundErrors == 0; 328} 329 330void MachineVerifier::verifySlotIndexes() const { 331 if (Indexes == nullptr) 332 return; 333 334 // Ensure the IdxMBB list is sorted by slot indexes. 335 SlotIndex Last; 336 for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(), 337 E = Indexes->MBBIndexEnd(); I != E; ++I) { 338 assert(!Last.isValid() || I->first > Last); 339 Last = I->first; 340 } 341} 342 343void MachineVerifier::verifyProperties(const MachineFunction &MF) { 344 // If a pass has introduced virtual registers without clearing the 345 // NoVRegs property (or set it without allocating the vregs) 346 // then report an error. 347 if (MF.getProperties().hasProperty( 348 MachineFunctionProperties::Property::NoVRegs) && 349 MRI->getNumVirtRegs()) 350 report("Function has NoVRegs property but there are VReg operands", &MF); 351} 352 353unsigned MachineVerifier::verify(const MachineFunction &MF) { 354 foundErrors = 0; 355 356 this->MF = &MF; 357 TM = &MF.getTarget(); 358 TII = MF.getSubtarget().getInstrInfo(); 359 TRI = MF.getSubtarget().getRegisterInfo(); 360 MRI = &MF.getRegInfo(); 361 362 const bool isFunctionFailedISel = MF.getProperties().hasProperty( 363 MachineFunctionProperties::Property::FailedISel); 364 365 // If we're mid-GlobalISel and we already triggered the fallback path then 366 // it's expected that the MIR is somewhat broken but that's ok since we'll 367 // reset it and clear the FailedISel attribute in ResetMachineFunctions. 368 if (isFunctionFailedISel) 369 return foundErrors; 370 371 isFunctionRegBankSelected = MF.getProperties().hasProperty( 372 MachineFunctionProperties::Property::RegBankSelected); 373 isFunctionSelected = MF.getProperties().hasProperty( 374 MachineFunctionProperties::Property::Selected); 375 376 LiveVars = nullptr; 377 LiveInts = nullptr; 378 LiveStks = nullptr; 379 Indexes = nullptr; 380 if (PASS) { 381 LiveInts = PASS->getAnalysisIfAvailable<LiveIntervals>(); 382 // We don't want to verify LiveVariables if LiveIntervals is available. 383 if (!LiveInts) 384 LiveVars = PASS->getAnalysisIfAvailable<LiveVariables>(); 385 LiveStks = PASS->getAnalysisIfAvailable<LiveStacks>(); 386 Indexes = PASS->getAnalysisIfAvailable<SlotIndexes>(); 387 } 388 389 verifySlotIndexes(); 390 391 verifyProperties(MF); 392 393 visitMachineFunctionBefore(); 394 for (const MachineBasicBlock &MBB : MF) { 395 visitMachineBasicBlockBefore(&MBB); 396 // Keep track of the current bundle header. 397 const MachineInstr *CurBundle = nullptr; 398 // Do we expect the next instruction to be part of the same bundle? 399 bool InBundle = false; 400 401 for (const MachineInstr &MI : MBB.instrs()) { 402 if (MI.getParent() != &MBB) { 403 report("Bad instruction parent pointer", &MBB); 404 errs() << "Instruction: " << MI; 405 continue; 406 } 407 408 // Check for consistent bundle flags. 409 if (InBundle && !MI.isBundledWithPred()) 410 report("Missing BundledPred flag, " 411 "BundledSucc was set on predecessor", 412 &MI); 413 if (!InBundle && MI.isBundledWithPred()) 414 report("BundledPred flag is set, " 415 "but BundledSucc not set on predecessor", 416 &MI); 417 418 // Is this a bundle header? 419 if (!MI.isInsideBundle()) { 420 if (CurBundle) 421 visitMachineBundleAfter(CurBundle); 422 CurBundle = &MI; 423 visitMachineBundleBefore(CurBundle); 424 } else if (!CurBundle) 425 report("No bundle header", &MI); 426 visitMachineInstrBefore(&MI); 427 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 428 const MachineOperand &Op = MI.getOperand(I); 429 if (Op.getParent() != &MI) { 430 // Make sure to use correct addOperand / RemoveOperand / ChangeTo 431 // functions when replacing operands of a MachineInstr. 432 report("Instruction has operand with wrong parent set", &MI); 433 } 434 435 visitMachineOperand(&Op, I); 436 } 437 438 // Was this the last bundled instruction? 439 InBundle = MI.isBundledWithSucc(); 440 } 441 if (CurBundle) 442 visitMachineBundleAfter(CurBundle); 443 if (InBundle) 444 report("BundledSucc flag set on last instruction in block", &MBB.back()); 445 visitMachineBasicBlockAfter(&MBB); 446 } 447 visitMachineFunctionAfter(); 448 449 // Clean up. 450 regsLive.clear(); 451 regsDefined.clear(); 452 regsDead.clear(); 453 regsKilled.clear(); 454 regMasks.clear(); 455 MBBInfoMap.clear(); 456 457 return foundErrors; 458} 459 460void MachineVerifier::report(const char *msg, const MachineFunction *MF) { 461 assert(MF); 462 errs() << '\n'; 463 if (!foundErrors++) { 464 if (Banner) 465 errs() << "# " << Banner << '\n'; 466 if (LiveInts != nullptr) 467 LiveInts->print(errs()); 468 else 469 MF->print(errs(), Indexes); 470 } 471 errs() << "*** Bad machine code: " << msg << " ***\n" 472 << "- function: " << MF->getName() << "\n"; 473} 474 475void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) { 476 assert(MBB); 477 report(msg, MBB->getParent()); 478 errs() << "- basic block: " << printMBBReference(*MBB) << ' ' 479 << MBB->getName() << " (" << (const void *)MBB << ')'; 480 if (Indexes) 481 errs() << " [" << Indexes->getMBBStartIdx(MBB) 482 << ';' << Indexes->getMBBEndIdx(MBB) << ')'; 483 errs() << '\n'; 484} 485 486void MachineVerifier::report(const char *msg, const MachineInstr *MI) { 487 assert(MI); 488 report(msg, MI->getParent()); 489 errs() << "- instruction: "; 490 if (Indexes && Indexes->hasIndex(*MI)) 491 errs() << Indexes->getInstructionIndex(*MI) << '\t'; 492 MI->print(errs(), /*IsStandalone=*/true); 493} 494 495void MachineVerifier::report(const char *msg, const MachineOperand *MO, 496 unsigned MONum, LLT MOVRegType) { 497 assert(MO); 498 report(msg, MO->getParent()); 499 errs() << "- operand " << MONum << ": "; 500 MO->print(errs(), MOVRegType, TRI); 501 errs() << "\n"; 502} 503 504void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) { 505 report(Msg.str().c_str(), MI); 506} 507 508void MachineVerifier::report_context(SlotIndex Pos) const { 509 errs() << "- at: " << Pos << '\n'; 510} 511 512void MachineVerifier::report_context(const LiveInterval &LI) const { 513 errs() << "- interval: " << LI << '\n'; 514} 515 516void MachineVerifier::report_context(const LiveRange &LR, Register VRegUnit, 517 LaneBitmask LaneMask) const { 518 report_context_liverange(LR); 519 report_context_vreg_regunit(VRegUnit); 520 if (LaneMask.any()) 521 report_context_lanemask(LaneMask); 522} 523 524void MachineVerifier::report_context(const LiveRange::Segment &S) const { 525 errs() << "- segment: " << S << '\n'; 526} 527 528void MachineVerifier::report_context(const VNInfo &VNI) const { 529 errs() << "- ValNo: " << VNI.id << " (def " << VNI.def << ")\n"; 530} 531 532void MachineVerifier::report_context_liverange(const LiveRange &LR) const { 533 errs() << "- liverange: " << LR << '\n'; 534} 535 536void MachineVerifier::report_context(MCPhysReg PReg) const { 537 errs() << "- p. register: " << printReg(PReg, TRI) << '\n'; 538} 539 540void MachineVerifier::report_context_vreg(Register VReg) const { 541 errs() << "- v. register: " << printReg(VReg, TRI) << '\n'; 542} 543 544void MachineVerifier::report_context_vreg_regunit(Register VRegOrUnit) const { 545 if (Register::isVirtualRegister(VRegOrUnit)) { 546 report_context_vreg(VRegOrUnit); 547 } else { 548 errs() << "- regunit: " << printRegUnit(VRegOrUnit, TRI) << '\n'; 549 } 550} 551 552void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const { 553 errs() << "- lanemask: " << PrintLaneMask(LaneMask) << '\n'; 554} 555 556void MachineVerifier::markReachable(const MachineBasicBlock *MBB) { 557 BBInfo &MInfo = MBBInfoMap[MBB]; 558 if (!MInfo.reachable) { 559 MInfo.reachable = true; 560 for (const MachineBasicBlock *Succ : MBB->successors()) 561 markReachable(Succ); 562 } 563} 564 565void MachineVerifier::visitMachineFunctionBefore() { 566 lastIndex = SlotIndex(); 567 regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs() 568 : TRI->getReservedRegs(*MF); 569 570 if (!MF->empty()) 571 markReachable(&MF->front()); 572 573 // Build a set of the basic blocks in the function. 574 FunctionBlocks.clear(); 575 for (const auto &MBB : *MF) { 576 FunctionBlocks.insert(&MBB); 577 BBInfo &MInfo = MBBInfoMap[&MBB]; 578 579 MInfo.Preds.insert(MBB.pred_begin(), MBB.pred_end()); 580 if (MInfo.Preds.size() != MBB.pred_size()) 581 report("MBB has duplicate entries in its predecessor list.", &MBB); 582 583 MInfo.Succs.insert(MBB.succ_begin(), MBB.succ_end()); 584 if (MInfo.Succs.size() != MBB.succ_size()) 585 report("MBB has duplicate entries in its successor list.", &MBB); 586 } 587 588 // Check that the register use lists are sane. 589 MRI->verifyUseLists(); 590 591 if (!MF->empty()) 592 verifyStackFrame(); 593} 594 595void 596MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { 597 FirstTerminator = nullptr; 598 FirstNonPHI = nullptr; 599 600 if (!MF->getProperties().hasProperty( 601 MachineFunctionProperties::Property::NoPHIs) && MRI->tracksLiveness()) { 602 // If this block has allocatable physical registers live-in, check that 603 // it is an entry block or landing pad. 604 for (const auto &LI : MBB->liveins()) { 605 if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() && 606 MBB->getIterator() != MBB->getParent()->begin()) { 607 report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB); 608 report_context(LI.PhysReg); 609 } 610 } 611 } 612 613 // Count the number of landing pad successors. 614 SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs; 615 for (const auto *succ : MBB->successors()) { 616 if (succ->isEHPad()) 617 LandingPadSuccs.insert(succ); 618 if (!FunctionBlocks.count(succ)) 619 report("MBB has successor that isn't part of the function.", MBB); 620 if (!MBBInfoMap[succ].Preds.count(MBB)) { 621 report("Inconsistent CFG", MBB); 622 errs() << "MBB is not in the predecessor list of the successor " 623 << printMBBReference(*succ) << ".\n"; 624 } 625 } 626 627 // Check the predecessor list. 628 for (const MachineBasicBlock *Pred : MBB->predecessors()) { 629 if (!FunctionBlocks.count(Pred)) 630 report("MBB has predecessor that isn't part of the function.", MBB); 631 if (!MBBInfoMap[Pred].Succs.count(MBB)) { 632 report("Inconsistent CFG", MBB); 633 errs() << "MBB is not in the successor list of the predecessor " 634 << printMBBReference(*Pred) << ".\n"; 635 } 636 } 637 638 const MCAsmInfo *AsmInfo = TM->getMCAsmInfo(); 639 const BasicBlock *BB = MBB->getBasicBlock(); 640 const Function &F = MF->getFunction(); 641 if (LandingPadSuccs.size() > 1 && 642 !(AsmInfo && 643 AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj && 644 BB && isa<SwitchInst>(BB->getTerminator())) && 645 !isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) 646 report("MBB has more than one landing pad successor", MBB); 647 648 // Call analyzeBranch. If it succeeds, there several more conditions to check. 649 MachineBasicBlock *TBB = nullptr, *FBB = nullptr; 650 SmallVector<MachineOperand, 4> Cond; 651 if (!TII->analyzeBranch(*const_cast<MachineBasicBlock *>(MBB), TBB, FBB, 652 Cond)) { 653 // Ok, analyzeBranch thinks it knows what's going on with this block. Let's 654 // check whether its answers match up with reality. 655 if (!TBB && !FBB) { 656 // Block falls through to its successor. 657 if (!MBB->empty() && MBB->back().isBarrier() && 658 !TII->isPredicated(MBB->back())) { 659 report("MBB exits via unconditional fall-through but ends with a " 660 "barrier instruction!", MBB); 661 } 662 if (!Cond.empty()) { 663 report("MBB exits via unconditional fall-through but has a condition!", 664 MBB); 665 } 666 } else if (TBB && !FBB && Cond.empty()) { 667 // Block unconditionally branches somewhere. 668 if (MBB->empty()) { 669 report("MBB exits via unconditional branch but doesn't contain " 670 "any instructions!", MBB); 671 } else if (!MBB->back().isBarrier()) { 672 report("MBB exits via unconditional branch but doesn't end with a " 673 "barrier instruction!", MBB); 674 } else if (!MBB->back().isTerminator()) { 675 report("MBB exits via unconditional branch but the branch isn't a " 676 "terminator instruction!", MBB); 677 } 678 } else if (TBB && !FBB && !Cond.empty()) { 679 // Block conditionally branches somewhere, otherwise falls through. 680 if (MBB->empty()) { 681 report("MBB exits via conditional branch/fall-through but doesn't " 682 "contain any instructions!", MBB); 683 } else if (MBB->back().isBarrier()) { 684 report("MBB exits via conditional branch/fall-through but ends with a " 685 "barrier instruction!", MBB); 686 } else if (!MBB->back().isTerminator()) { 687 report("MBB exits via conditional branch/fall-through but the branch " 688 "isn't a terminator instruction!", MBB); 689 } 690 } else if (TBB && FBB) { 691 // Block conditionally branches somewhere, otherwise branches 692 // somewhere else. 693 if (MBB->empty()) { 694 report("MBB exits via conditional branch/branch but doesn't " 695 "contain any instructions!", MBB); 696 } else if (!MBB->back().isBarrier()) { 697 report("MBB exits via conditional branch/branch but doesn't end with a " 698 "barrier instruction!", MBB); 699 } else if (!MBB->back().isTerminator()) { 700 report("MBB exits via conditional branch/branch but the branch " 701 "isn't a terminator instruction!", MBB); 702 } 703 if (Cond.empty()) { 704 report("MBB exits via conditional branch/branch but there's no " 705 "condition!", MBB); 706 } 707 } else { 708 report("analyzeBranch returned invalid data!", MBB); 709 } 710 711 // Now check that the successors match up with the answers reported by 712 // analyzeBranch. 713 if (TBB && !MBB->isSuccessor(TBB)) 714 report("MBB exits via jump or conditional branch, but its target isn't a " 715 "CFG successor!", 716 MBB); 717 if (FBB && !MBB->isSuccessor(FBB)) 718 report("MBB exits via conditional branch, but its target isn't a CFG " 719 "successor!", 720 MBB); 721 722 // There might be a fallthrough to the next block if there's either no 723 // unconditional true branch, or if there's a condition, and one of the 724 // branches is missing. 725 bool Fallthrough = !TBB || (!Cond.empty() && !FBB); 726 727 // A conditional fallthrough must be an actual CFG successor, not 728 // unreachable. (Conversely, an unconditional fallthrough might not really 729 // be a successor, because the block might end in unreachable.) 730 if (!Cond.empty() && !FBB) { 731 MachineFunction::const_iterator MBBI = std::next(MBB->getIterator()); 732 if (MBBI == MF->end()) { 733 report("MBB conditionally falls through out of function!", MBB); 734 } else if (!MBB->isSuccessor(&*MBBI)) 735 report("MBB exits via conditional branch/fall-through but the CFG " 736 "successors don't match the actual successors!", 737 MBB); 738 } 739 740 // Verify that there aren't any extra un-accounted-for successors. 741 for (const MachineBasicBlock *SuccMBB : MBB->successors()) { 742 // If this successor is one of the branch targets, it's okay. 743 if (SuccMBB == TBB || SuccMBB == FBB) 744 continue; 745 // If we might have a fallthrough, and the successor is the fallthrough 746 // block, that's also ok. 747 if (Fallthrough && SuccMBB == MBB->getNextNode()) 748 continue; 749 // Also accept successors which are for exception-handling or might be 750 // inlineasm_br targets. 751 if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget()) 752 continue; 753 report("MBB has unexpected successors which are not branch targets, " 754 "fallthrough, EHPads, or inlineasm_br targets.", 755 MBB); 756 } 757 } 758 759 regsLive.clear(); 760 if (MRI->tracksLiveness()) { 761 for (const auto &LI : MBB->liveins()) { 762 if (!Register::isPhysicalRegister(LI.PhysReg)) { 763 report("MBB live-in list contains non-physical register", MBB); 764 continue; 765 } 766 for (const MCPhysReg &SubReg : TRI->subregs_inclusive(LI.PhysReg)) 767 regsLive.insert(SubReg); 768 } 769 } 770 771 const MachineFrameInfo &MFI = MF->getFrameInfo(); 772 BitVector PR = MFI.getPristineRegs(*MF); 773 for (unsigned I : PR.set_bits()) { 774 for (const MCPhysReg &SubReg : TRI->subregs_inclusive(I)) 775 regsLive.insert(SubReg); 776 } 777 778 regsKilled.clear(); 779 regsDefined.clear(); 780 781 if (Indexes) 782 lastIndex = Indexes->getMBBStartIdx(MBB); 783} 784 785// This function gets called for all bundle headers, including normal 786// stand-alone unbundled instructions. 787void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) { 788 if (Indexes && Indexes->hasIndex(*MI)) { 789 SlotIndex idx = Indexes->getInstructionIndex(*MI); 790 if (!(idx > lastIndex)) { 791 report("Instruction index out of order", MI); 792 errs() << "Last instruction was at " << lastIndex << '\n'; 793 } 794 lastIndex = idx; 795 } 796 797 // Ensure non-terminators don't follow terminators. 798 if (MI->isTerminator()) { 799 if (!FirstTerminator) 800 FirstTerminator = MI; 801 } else if (FirstTerminator) { 802 report("Non-terminator instruction after the first terminator", MI); 803 errs() << "First terminator was:\t" << *FirstTerminator; 804 } 805} 806 807// The operands on an INLINEASM instruction must follow a template. 808// Verify that the flag operands make sense. 809void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) { 810 // The first two operands on INLINEASM are the asm string and global flags. 811 if (MI->getNumOperands() < 2) { 812 report("Too few operands on inline asm", MI); 813 return; 814 } 815 if (!MI->getOperand(0).isSymbol()) 816 report("Asm string must be an external symbol", MI); 817 if (!MI->getOperand(1).isImm()) 818 report("Asm flags must be an immediate", MI); 819 // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2, 820 // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16, 821 // and Extra_IsConvergent = 32. 822 if (!isUInt<6>(MI->getOperand(1).getImm())) 823 report("Unknown asm flags", &MI->getOperand(1), 1); 824 825 static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed"); 826 827 unsigned OpNo = InlineAsm::MIOp_FirstOperand; 828 unsigned NumOps; 829 for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) { 830 const MachineOperand &MO = MI->getOperand(OpNo); 831 // There may be implicit ops after the fixed operands. 832 if (!MO.isImm()) 833 break; 834 NumOps = 1 + InlineAsm::getNumOperandRegisters(MO.getImm()); 835 } 836 837 if (OpNo > MI->getNumOperands()) 838 report("Missing operands in last group", MI); 839 840 // An optional MDNode follows the groups. 841 if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata()) 842 ++OpNo; 843 844 // All trailing operands must be implicit registers. 845 for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) { 846 const MachineOperand &MO = MI->getOperand(OpNo); 847 if (!MO.isReg() || !MO.isImplicit()) 848 report("Expected implicit register after groups", &MO, OpNo); 849 } 850} 851 852/// Check that types are consistent when two operands need to have the same 853/// number of vector elements. 854/// \return true if the types are valid. 855bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1, 856 const MachineInstr *MI) { 857 if (Ty0.isVector() != Ty1.isVector()) { 858 report("operand types must be all-vector or all-scalar", MI); 859 // Generally we try to report as many issues as possible at once, but in 860 // this case it's not clear what should we be comparing the size of the 861 // scalar with: the size of the whole vector or its lane. Instead of 862 // making an arbitrary choice and emitting not so helpful message, let's 863 // avoid the extra noise and stop here. 864 return false; 865 } 866 867 if (Ty0.isVector() && Ty0.getNumElements() != Ty1.getNumElements()) { 868 report("operand types must preserve number of vector elements", MI); 869 return false; 870 } 871 872 return true; 873} 874 875void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { 876 if (isFunctionSelected) 877 report("Unexpected generic instruction in a Selected function", MI); 878 879 const MCInstrDesc &MCID = MI->getDesc(); 880 unsigned NumOps = MI->getNumOperands(); 881 882 // Branches must reference a basic block if they are not indirect 883 if (MI->isBranch() && !MI->isIndirectBranch()) { 884 bool HasMBB = false; 885 for (const MachineOperand &Op : MI->operands()) { 886 if (Op.isMBB()) { 887 HasMBB = true; 888 break; 889 } 890 } 891 892 if (!HasMBB) { 893 report("Branch instruction is missing a basic block operand or " 894 "isIndirectBranch property", 895 MI); 896 } 897 } 898 899 // Check types. 900 SmallVector<LLT, 4> Types; 901 for (unsigned I = 0, E = std::min(MCID.getNumOperands(), NumOps); 902 I != E; ++I) { 903 if (!MCID.OpInfo[I].isGenericType()) 904 continue; 905 // Generic instructions specify type equality constraints between some of 906 // their operands. Make sure these are consistent. 907 size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex(); 908 Types.resize(std::max(TypeIdx + 1, Types.size())); 909 910 const MachineOperand *MO = &MI->getOperand(I); 911 if (!MO->isReg()) { 912 report("generic instruction must use register operands", MI); 913 continue; 914 } 915 916 LLT OpTy = MRI->getType(MO->getReg()); 917 // Don't report a type mismatch if there is no actual mismatch, only a 918 // type missing, to reduce noise: 919 if (OpTy.isValid()) { 920 // Only the first valid type for a type index will be printed: don't 921 // overwrite it later so it's always clear which type was expected: 922 if (!Types[TypeIdx].isValid()) 923 Types[TypeIdx] = OpTy; 924 else if (Types[TypeIdx] != OpTy) 925 report("Type mismatch in generic instruction", MO, I, OpTy); 926 } else { 927 // Generic instructions must have types attached to their operands. 928 report("Generic instruction is missing a virtual register type", MO, I); 929 } 930 } 931 932 // Generic opcodes must not have physical register operands. 933 for (unsigned I = 0; I < MI->getNumOperands(); ++I) { 934 const MachineOperand *MO = &MI->getOperand(I); 935 if (MO->isReg() && Register::isPhysicalRegister(MO->getReg())) 936 report("Generic instruction cannot have physical register", MO, I); 937 } 938 939 // Avoid out of bounds in checks below. This was already reported earlier. 940 if (MI->getNumOperands() < MCID.getNumOperands()) 941 return; 942 943 StringRef ErrorInfo; 944 if (!TII->verifyInstruction(*MI, ErrorInfo)) 945 report(ErrorInfo.data(), MI); 946 947 // Verify properties of various specific instruction types 948 unsigned Opc = MI->getOpcode(); 949 switch (Opc) { 950 case TargetOpcode::G_ASSERT_SEXT: 951 case TargetOpcode::G_ASSERT_ZEXT: { 952 std::string OpcName = 953 Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT"; 954 if (!MI->getOperand(2).isImm()) { 955 report(Twine(OpcName, " expects an immediate operand #2"), MI); 956 break; 957 } 958 959 Register Dst = MI->getOperand(0).getReg(); 960 Register Src = MI->getOperand(1).getReg(); 961 LLT SrcTy = MRI->getType(Src); 962 int64_t Imm = MI->getOperand(2).getImm(); 963 if (Imm <= 0) { 964 report(Twine(OpcName, " size must be >= 1"), MI); 965 break; 966 } 967 968 if (Imm >= SrcTy.getScalarSizeInBits()) { 969 report(Twine(OpcName, " size must be less than source bit width"), MI); 970 break; 971 } 972 973 if (MRI->getRegBankOrNull(Src) != MRI->getRegBankOrNull(Dst)) { 974 report( 975 Twine(OpcName, " source and destination register banks must match"), 976 MI); 977 break; 978 } 979 980 if (MRI->getRegClassOrNull(Src) != MRI->getRegClassOrNull(Dst)) 981 report( 982 Twine(OpcName, " source and destination register classes must match"), 983 MI); 984 985 break; 986 } 987 988 case TargetOpcode::G_CONSTANT: 989 case TargetOpcode::G_FCONSTANT: { 990 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 991 if (DstTy.isVector()) 992 report("Instruction cannot use a vector result type", MI); 993 994 if (MI->getOpcode() == TargetOpcode::G_CONSTANT) { 995 if (!MI->getOperand(1).isCImm()) { 996 report("G_CONSTANT operand must be cimm", MI); 997 break; 998 } 999 1000 const ConstantInt *CI = MI->getOperand(1).getCImm(); 1001 if (CI->getBitWidth() != DstTy.getSizeInBits()) 1002 report("inconsistent constant size", MI); 1003 } else { 1004 if (!MI->getOperand(1).isFPImm()) { 1005 report("G_FCONSTANT operand must be fpimm", MI); 1006 break; 1007 } 1008 const ConstantFP *CF = MI->getOperand(1).getFPImm(); 1009 1010 if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) != 1011 DstTy.getSizeInBits()) { 1012 report("inconsistent constant size", MI); 1013 } 1014 } 1015 1016 break; 1017 } 1018 case TargetOpcode::G_LOAD: 1019 case TargetOpcode::G_STORE: 1020 case TargetOpcode::G_ZEXTLOAD: 1021 case TargetOpcode::G_SEXTLOAD: { 1022 LLT ValTy = MRI->getType(MI->getOperand(0).getReg()); 1023 LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); 1024 if (!PtrTy.isPointer()) 1025 report("Generic memory instruction must access a pointer", MI); 1026 1027 // Generic loads and stores must have a single MachineMemOperand 1028 // describing that access. 1029 if (!MI->hasOneMemOperand()) { 1030 report("Generic instruction accessing memory must have one mem operand", 1031 MI); 1032 } else { 1033 const MachineMemOperand &MMO = **MI->memoperands_begin(); 1034 if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD || 1035 MI->getOpcode() == TargetOpcode::G_SEXTLOAD) { 1036 if (MMO.getSizeInBits() >= ValTy.getSizeInBits()) 1037 report("Generic extload must have a narrower memory type", MI); 1038 } else if (MI->getOpcode() == TargetOpcode::G_LOAD) { 1039 if (MMO.getSize() > ValTy.getSizeInBytes()) 1040 report("load memory size cannot exceed result size", MI); 1041 } else if (MI->getOpcode() == TargetOpcode::G_STORE) { 1042 if (ValTy.getSizeInBytes() < MMO.getSize()) 1043 report("store memory size cannot exceed value size", MI); 1044 } 1045 } 1046 1047 break; 1048 } 1049 case TargetOpcode::G_PHI: { 1050 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1051 if (!DstTy.isValid() || !all_of(drop_begin(MI->operands()), 1052 [this, &DstTy](const MachineOperand &MO) { 1053 if (!MO.isReg()) 1054 return true; 1055 LLT Ty = MRI->getType(MO.getReg()); 1056 if (!Ty.isValid() || (Ty != DstTy)) 1057 return false; 1058 return true; 1059 })) 1060 report("Generic Instruction G_PHI has operands with incompatible/missing " 1061 "types", 1062 MI); 1063 break; 1064 } 1065 case TargetOpcode::G_BITCAST: { 1066 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1067 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1068 if (!DstTy.isValid() || !SrcTy.isValid()) 1069 break; 1070 1071 if (SrcTy.isPointer() != DstTy.isPointer()) 1072 report("bitcast cannot convert between pointers and other types", MI); 1073 1074 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) 1075 report("bitcast sizes must match", MI); 1076 1077 if (SrcTy == DstTy) 1078 report("bitcast must change the type", MI); 1079 1080 break; 1081 } 1082 case TargetOpcode::G_INTTOPTR: 1083 case TargetOpcode::G_PTRTOINT: 1084 case TargetOpcode::G_ADDRSPACE_CAST: { 1085 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1086 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1087 if (!DstTy.isValid() || !SrcTy.isValid()) 1088 break; 1089 1090 verifyVectorElementMatch(DstTy, SrcTy, MI); 1091 1092 DstTy = DstTy.getScalarType(); 1093 SrcTy = SrcTy.getScalarType(); 1094 1095 if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) { 1096 if (!DstTy.isPointer()) 1097 report("inttoptr result type must be a pointer", MI); 1098 if (SrcTy.isPointer()) 1099 report("inttoptr source type must not be a pointer", MI); 1100 } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) { 1101 if (!SrcTy.isPointer()) 1102 report("ptrtoint source type must be a pointer", MI); 1103 if (DstTy.isPointer()) 1104 report("ptrtoint result type must not be a pointer", MI); 1105 } else { 1106 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST); 1107 if (!SrcTy.isPointer() || !DstTy.isPointer()) 1108 report("addrspacecast types must be pointers", MI); 1109 else { 1110 if (SrcTy.getAddressSpace() == DstTy.getAddressSpace()) 1111 report("addrspacecast must convert different address spaces", MI); 1112 } 1113 } 1114 1115 break; 1116 } 1117 case TargetOpcode::G_PTR_ADD: { 1118 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1119 LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); 1120 LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg()); 1121 if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid()) 1122 break; 1123 1124 if (!PtrTy.getScalarType().isPointer()) 1125 report("gep first operand must be a pointer", MI); 1126 1127 if (OffsetTy.getScalarType().isPointer()) 1128 report("gep offset operand must not be a pointer", MI); 1129 1130 // TODO: Is the offset allowed to be a scalar with a vector? 1131 break; 1132 } 1133 case TargetOpcode::G_PTRMASK: { 1134 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1135 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1136 LLT MaskTy = MRI->getType(MI->getOperand(2).getReg()); 1137 if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid()) 1138 break; 1139 1140 if (!DstTy.getScalarType().isPointer()) 1141 report("ptrmask result type must be a pointer", MI); 1142 1143 if (!MaskTy.getScalarType().isScalar()) 1144 report("ptrmask mask type must be an integer", MI); 1145 1146 verifyVectorElementMatch(DstTy, MaskTy, MI); 1147 break; 1148 } 1149 case TargetOpcode::G_SEXT: 1150 case TargetOpcode::G_ZEXT: 1151 case TargetOpcode::G_ANYEXT: 1152 case TargetOpcode::G_TRUNC: 1153 case TargetOpcode::G_FPEXT: 1154 case TargetOpcode::G_FPTRUNC: { 1155 // Number of operands and presense of types is already checked (and 1156 // reported in case of any issues), so no need to report them again. As 1157 // we're trying to report as many issues as possible at once, however, the 1158 // instructions aren't guaranteed to have the right number of operands or 1159 // types attached to them at this point 1160 assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}"); 1161 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1162 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1163 if (!DstTy.isValid() || !SrcTy.isValid()) 1164 break; 1165 1166 LLT DstElTy = DstTy.getScalarType(); 1167 LLT SrcElTy = SrcTy.getScalarType(); 1168 if (DstElTy.isPointer() || SrcElTy.isPointer()) 1169 report("Generic extend/truncate can not operate on pointers", MI); 1170 1171 verifyVectorElementMatch(DstTy, SrcTy, MI); 1172 1173 unsigned DstSize = DstElTy.getSizeInBits(); 1174 unsigned SrcSize = SrcElTy.getSizeInBits(); 1175 switch (MI->getOpcode()) { 1176 default: 1177 if (DstSize <= SrcSize) 1178 report("Generic extend has destination type no larger than source", MI); 1179 break; 1180 case TargetOpcode::G_TRUNC: 1181 case TargetOpcode::G_FPTRUNC: 1182 if (DstSize >= SrcSize) 1183 report("Generic truncate has destination type no smaller than source", 1184 MI); 1185 break; 1186 } 1187 break; 1188 } 1189 case TargetOpcode::G_SELECT: { 1190 LLT SelTy = MRI->getType(MI->getOperand(0).getReg()); 1191 LLT CondTy = MRI->getType(MI->getOperand(1).getReg()); 1192 if (!SelTy.isValid() || !CondTy.isValid()) 1193 break; 1194 1195 // Scalar condition select on a vector is valid. 1196 if (CondTy.isVector()) 1197 verifyVectorElementMatch(SelTy, CondTy, MI); 1198 break; 1199 } 1200 case TargetOpcode::G_MERGE_VALUES: { 1201 // G_MERGE_VALUES should only be used to merge scalars into a larger scalar, 1202 // e.g. s2N = MERGE sN, sN 1203 // Merging multiple scalars into a vector is not allowed, should use 1204 // G_BUILD_VECTOR for that. 1205 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1206 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1207 if (DstTy.isVector() || SrcTy.isVector()) 1208 report("G_MERGE_VALUES cannot operate on vectors", MI); 1209 1210 const unsigned NumOps = MI->getNumOperands(); 1211 if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1)) 1212 report("G_MERGE_VALUES result size is inconsistent", MI); 1213 1214 for (unsigned I = 2; I != NumOps; ++I) { 1215 if (MRI->getType(MI->getOperand(I).getReg()) != SrcTy) 1216 report("G_MERGE_VALUES source types do not match", MI); 1217 } 1218 1219 break; 1220 } 1221 case TargetOpcode::G_UNMERGE_VALUES: { 1222 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1223 LLT SrcTy = MRI->getType(MI->getOperand(MI->getNumOperands()-1).getReg()); 1224 // For now G_UNMERGE can split vectors. 1225 for (unsigned i = 0; i < MI->getNumOperands()-1; ++i) { 1226 if (MRI->getType(MI->getOperand(i).getReg()) != DstTy) 1227 report("G_UNMERGE_VALUES destination types do not match", MI); 1228 } 1229 if (SrcTy.getSizeInBits() != 1230 (DstTy.getSizeInBits() * (MI->getNumOperands() - 1))) { 1231 report("G_UNMERGE_VALUES source operand does not cover dest operands", 1232 MI); 1233 } 1234 break; 1235 } 1236 case TargetOpcode::G_BUILD_VECTOR: { 1237 // Source types must be scalars, dest type a vector. Total size of scalars 1238 // must match the dest vector size. 1239 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1240 LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg()); 1241 if (!DstTy.isVector() || SrcEltTy.isVector()) { 1242 report("G_BUILD_VECTOR must produce a vector from scalar operands", MI); 1243 break; 1244 } 1245 1246 if (DstTy.getElementType() != SrcEltTy) 1247 report("G_BUILD_VECTOR result element type must match source type", MI); 1248 1249 if (DstTy.getNumElements() != MI->getNumOperands() - 1) 1250 report("G_BUILD_VECTOR must have an operand for each elemement", MI); 1251 1252 for (unsigned i = 2; i < MI->getNumOperands(); ++i) { 1253 if (MRI->getType(MI->getOperand(1).getReg()) != 1254 MRI->getType(MI->getOperand(i).getReg())) 1255 report("G_BUILD_VECTOR source operand types are not homogeneous", MI); 1256 } 1257 1258 break; 1259 } 1260 case TargetOpcode::G_BUILD_VECTOR_TRUNC: { 1261 // Source types must be scalars, dest type a vector. Scalar types must be 1262 // larger than the dest vector elt type, as this is a truncating operation. 1263 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1264 LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg()); 1265 if (!DstTy.isVector() || SrcEltTy.isVector()) 1266 report("G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands", 1267 MI); 1268 for (unsigned i = 2; i < MI->getNumOperands(); ++i) { 1269 if (MRI->getType(MI->getOperand(1).getReg()) != 1270 MRI->getType(MI->getOperand(i).getReg())) 1271 report("G_BUILD_VECTOR_TRUNC source operand types are not homogeneous", 1272 MI); 1273 } 1274 if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits()) 1275 report("G_BUILD_VECTOR_TRUNC source operand types are not larger than " 1276 "dest elt type", 1277 MI); 1278 break; 1279 } 1280 case TargetOpcode::G_CONCAT_VECTORS: { 1281 // Source types should be vectors, and total size should match the dest 1282 // vector size. 1283 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1284 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1285 if (!DstTy.isVector() || !SrcTy.isVector()) 1286 report("G_CONCAT_VECTOR requires vector source and destination operands", 1287 MI); 1288 1289 if (MI->getNumOperands() < 3) 1290 report("G_CONCAT_VECTOR requires at least 2 source operands", MI); 1291 1292 for (unsigned i = 2; i < MI->getNumOperands(); ++i) { 1293 if (MRI->getType(MI->getOperand(1).getReg()) != 1294 MRI->getType(MI->getOperand(i).getReg())) 1295 report("G_CONCAT_VECTOR source operand types are not homogeneous", MI); 1296 } 1297 if (DstTy.getNumElements() != 1298 SrcTy.getNumElements() * (MI->getNumOperands() - 1)) 1299 report("G_CONCAT_VECTOR num dest and source elements should match", MI); 1300 break; 1301 } 1302 case TargetOpcode::G_ICMP: 1303 case TargetOpcode::G_FCMP: { 1304 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1305 LLT SrcTy = MRI->getType(MI->getOperand(2).getReg()); 1306 1307 if ((DstTy.isVector() != SrcTy.isVector()) || 1308 (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements())) 1309 report("Generic vector icmp/fcmp must preserve number of lanes", MI); 1310 1311 break; 1312 } 1313 case TargetOpcode::G_EXTRACT: { 1314 const MachineOperand &SrcOp = MI->getOperand(1); 1315 if (!SrcOp.isReg()) { 1316 report("extract source must be a register", MI); 1317 break; 1318 } 1319 1320 const MachineOperand &OffsetOp = MI->getOperand(2); 1321 if (!OffsetOp.isImm()) { 1322 report("extract offset must be a constant", MI); 1323 break; 1324 } 1325 1326 unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); 1327 unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); 1328 if (SrcSize == DstSize) 1329 report("extract source must be larger than result", MI); 1330 1331 if (DstSize + OffsetOp.getImm() > SrcSize) 1332 report("extract reads past end of register", MI); 1333 break; 1334 } 1335 case TargetOpcode::G_INSERT: { 1336 const MachineOperand &SrcOp = MI->getOperand(2); 1337 if (!SrcOp.isReg()) { 1338 report("insert source must be a register", MI); 1339 break; 1340 } 1341 1342 const MachineOperand &OffsetOp = MI->getOperand(3); 1343 if (!OffsetOp.isImm()) { 1344 report("insert offset must be a constant", MI); 1345 break; 1346 } 1347 1348 unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); 1349 unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); 1350 1351 if (DstSize <= SrcSize) 1352 report("inserted size must be smaller than total register", MI); 1353 1354 if (SrcSize + OffsetOp.getImm() > DstSize) 1355 report("insert writes past end of register", MI); 1356 1357 break; 1358 } 1359 case TargetOpcode::G_JUMP_TABLE: { 1360 if (!MI->getOperand(1).isJTI()) 1361 report("G_JUMP_TABLE source operand must be a jump table index", MI); 1362 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1363 if (!DstTy.isPointer()) 1364 report("G_JUMP_TABLE dest operand must have a pointer type", MI); 1365 break; 1366 } 1367 case TargetOpcode::G_BRJT: { 1368 if (!MRI->getType(MI->getOperand(0).getReg()).isPointer()) 1369 report("G_BRJT src operand 0 must be a pointer type", MI); 1370 1371 if (!MI->getOperand(1).isJTI()) 1372 report("G_BRJT src operand 1 must be a jump table index", MI); 1373 1374 const auto &IdxOp = MI->getOperand(2); 1375 if (!IdxOp.isReg() || MRI->getType(IdxOp.getReg()).isPointer()) 1376 report("G_BRJT src operand 2 must be a scalar reg type", MI); 1377 break; 1378 } 1379 case TargetOpcode::G_INTRINSIC: 1380 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: { 1381 // TODO: Should verify number of def and use operands, but the current 1382 // interface requires passing in IR types for mangling. 1383 const MachineOperand &IntrIDOp = MI->getOperand(MI->getNumExplicitDefs()); 1384 if (!IntrIDOp.isIntrinsicID()) { 1385 report("G_INTRINSIC first src operand must be an intrinsic ID", MI); 1386 break; 1387 } 1388 1389 bool NoSideEffects = MI->getOpcode() == TargetOpcode::G_INTRINSIC; 1390 unsigned IntrID = IntrIDOp.getIntrinsicID(); 1391 if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) { 1392 AttributeList Attrs 1393 = Intrinsic::getAttributes(MF->getFunction().getContext(), 1394 static_cast<Intrinsic::ID>(IntrID)); 1395 bool DeclHasSideEffects = !Attrs.hasFnAttribute(Attribute::ReadNone); 1396 if (NoSideEffects && DeclHasSideEffects) { 1397 report("G_INTRINSIC used with intrinsic that accesses memory", MI); 1398 break; 1399 } 1400 if (!NoSideEffects && !DeclHasSideEffects) { 1401 report("G_INTRINSIC_W_SIDE_EFFECTS used with readnone intrinsic", MI); 1402 break; 1403 } 1404 } 1405 1406 break; 1407 } 1408 case TargetOpcode::G_SEXT_INREG: { 1409 if (!MI->getOperand(2).isImm()) { 1410 report("G_SEXT_INREG expects an immediate operand #2", MI); 1411 break; 1412 } 1413 1414 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1415 int64_t Imm = MI->getOperand(2).getImm(); 1416 if (Imm <= 0) 1417 report("G_SEXT_INREG size must be >= 1", MI); 1418 if (Imm >= SrcTy.getScalarSizeInBits()) 1419 report("G_SEXT_INREG size must be less than source bit width", MI); 1420 break; 1421 } 1422 case TargetOpcode::G_SHUFFLE_VECTOR: { 1423 const MachineOperand &MaskOp = MI->getOperand(3); 1424 if (!MaskOp.isShuffleMask()) { 1425 report("Incorrect mask operand type for G_SHUFFLE_VECTOR", MI); 1426 break; 1427 } 1428 1429 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1430 LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg()); 1431 LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg()); 1432 1433 if (Src0Ty != Src1Ty) 1434 report("Source operands must be the same type", MI); 1435 1436 if (Src0Ty.getScalarType() != DstTy.getScalarType()) 1437 report("G_SHUFFLE_VECTOR cannot change element type", MI); 1438 1439 // Don't check that all operands are vector because scalars are used in 1440 // place of 1 element vectors. 1441 int SrcNumElts = Src0Ty.isVector() ? Src0Ty.getNumElements() : 1; 1442 int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1; 1443 1444 ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask(); 1445 1446 if (static_cast<int>(MaskIdxes.size()) != DstNumElts) 1447 report("Wrong result type for shufflemask", MI); 1448 1449 for (int Idx : MaskIdxes) { 1450 if (Idx < 0) 1451 continue; 1452 1453 if (Idx >= 2 * SrcNumElts) 1454 report("Out of bounds shuffle index", MI); 1455 } 1456 1457 break; 1458 } 1459 case TargetOpcode::G_DYN_STACKALLOC: { 1460 const MachineOperand &DstOp = MI->getOperand(0); 1461 const MachineOperand &AllocOp = MI->getOperand(1); 1462 const MachineOperand &AlignOp = MI->getOperand(2); 1463 1464 if (!DstOp.isReg() || !MRI->getType(DstOp.getReg()).isPointer()) { 1465 report("dst operand 0 must be a pointer type", MI); 1466 break; 1467 } 1468 1469 if (!AllocOp.isReg() || !MRI->getType(AllocOp.getReg()).isScalar()) { 1470 report("src operand 1 must be a scalar reg type", MI); 1471 break; 1472 } 1473 1474 if (!AlignOp.isImm()) { 1475 report("src operand 2 must be an immediate type", MI); 1476 break; 1477 } 1478 break; 1479 } 1480 case TargetOpcode::G_MEMCPY: 1481 case TargetOpcode::G_MEMMOVE: { 1482 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands(); 1483 if (MMOs.size() != 2) { 1484 report("memcpy/memmove must have 2 memory operands", MI); 1485 break; 1486 } 1487 1488 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) || 1489 (MMOs[1]->isStore() || !MMOs[1]->isLoad())) { 1490 report("wrong memory operand types", MI); 1491 break; 1492 } 1493 1494 if (MMOs[0]->getSize() != MMOs[1]->getSize()) 1495 report("inconsistent memory operand sizes", MI); 1496 1497 LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg()); 1498 LLT SrcPtrTy = MRI->getType(MI->getOperand(1).getReg()); 1499 1500 if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) { 1501 report("memory instruction operand must be a pointer", MI); 1502 break; 1503 } 1504 1505 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace()) 1506 report("inconsistent store address space", MI); 1507 if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace()) 1508 report("inconsistent load address space", MI); 1509 1510 break; 1511 } 1512 case TargetOpcode::G_BZERO: 1513 case TargetOpcode::G_MEMSET: { 1514 ArrayRef<MachineMemOperand *> MMOs = MI->memoperands(); 1515 std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero"; 1516 if (MMOs.size() != 1) { 1517 report(Twine(Name, " must have 1 memory operand"), MI); 1518 break; 1519 } 1520 1521 if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) { 1522 report(Twine(Name, " memory operand must be a store"), MI); 1523 break; 1524 } 1525 1526 LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg()); 1527 if (!DstPtrTy.isPointer()) { 1528 report(Twine(Name, " operand must be a pointer"), MI); 1529 break; 1530 } 1531 1532 if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace()) 1533 report("inconsistent " + Twine(Name, " address space"), MI); 1534 1535 break; 1536 } 1537 case TargetOpcode::G_VECREDUCE_SEQ_FADD: 1538 case TargetOpcode::G_VECREDUCE_SEQ_FMUL: { 1539 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1540 LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg()); 1541 LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg()); 1542 if (!DstTy.isScalar()) 1543 report("Vector reduction requires a scalar destination type", MI); 1544 if (!Src1Ty.isScalar()) 1545 report("Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI); 1546 if (!Src2Ty.isVector()) 1547 report("Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI); 1548 break; 1549 } 1550 case TargetOpcode::G_VECREDUCE_FADD: 1551 case TargetOpcode::G_VECREDUCE_FMUL: 1552 case TargetOpcode::G_VECREDUCE_FMAX: 1553 case TargetOpcode::G_VECREDUCE_FMIN: 1554 case TargetOpcode::G_VECREDUCE_ADD: 1555 case TargetOpcode::G_VECREDUCE_MUL: 1556 case TargetOpcode::G_VECREDUCE_AND: 1557 case TargetOpcode::G_VECREDUCE_OR: 1558 case TargetOpcode::G_VECREDUCE_XOR: 1559 case TargetOpcode::G_VECREDUCE_SMAX: 1560 case TargetOpcode::G_VECREDUCE_SMIN: 1561 case TargetOpcode::G_VECREDUCE_UMAX: 1562 case TargetOpcode::G_VECREDUCE_UMIN: { 1563 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1564 LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); 1565 if (!DstTy.isScalar()) 1566 report("Vector reduction requires a scalar destination type", MI); 1567 if (!SrcTy.isVector()) 1568 report("Vector reduction requires vector source=", MI); 1569 break; 1570 } 1571 1572 case TargetOpcode::G_SBFX: 1573 case TargetOpcode::G_UBFX: { 1574 LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); 1575 if (DstTy.isVector()) { 1576 report("Bitfield extraction is not supported on vectors", MI); 1577 break; 1578 } 1579 break; 1580 } 1581 case TargetOpcode::G_ROTR: 1582 case TargetOpcode::G_ROTL: { 1583 LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg()); 1584 LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg()); 1585 if (Src1Ty.isVector() != Src2Ty.isVector()) { 1586 report("Rotate requires operands to be either all scalars or all vectors", 1587 MI); 1588 break; 1589 } 1590 break; 1591 } 1592 1593 default: 1594 break; 1595 } 1596} 1597 1598void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { 1599 const MCInstrDesc &MCID = MI->getDesc(); 1600 if (MI->getNumOperands() < MCID.getNumOperands()) { 1601 report("Too few operands", MI); 1602 errs() << MCID.getNumOperands() << " operands expected, but " 1603 << MI->getNumOperands() << " given.\n"; 1604 } 1605 1606 if (MI->isPHI()) { 1607 if (MF->getProperties().hasProperty( 1608 MachineFunctionProperties::Property::NoPHIs)) 1609 report("Found PHI instruction with NoPHIs property set", MI); 1610 1611 if (FirstNonPHI) 1612 report("Found PHI instruction after non-PHI", MI); 1613 } else if (FirstNonPHI == nullptr) 1614 FirstNonPHI = MI; 1615 1616 // Check the tied operands. 1617 if (MI->isInlineAsm()) 1618 verifyInlineAsm(MI); 1619 1620 // Check that unspillable terminators define a reg and have at most one use. 1621 if (TII->isUnspillableTerminator(MI)) { 1622 if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef()) 1623 report("Unspillable Terminator does not define a reg", MI); 1624 Register Def = MI->getOperand(0).getReg(); 1625 if (Def.isVirtual() && 1626 std::distance(MRI->use_nodbg_begin(Def), MRI->use_nodbg_end()) > 1) 1627 report("Unspillable Terminator expected to have at most one use!", MI); 1628 } 1629 1630 // A fully-formed DBG_VALUE must have a location. Ignore partially formed 1631 // DBG_VALUEs: these are convenient to use in tests, but should never get 1632 // generated. 1633 if (MI->isDebugValue() && MI->getNumOperands() == 4) 1634 if (!MI->getDebugLoc()) 1635 report("Missing DebugLoc for debug instruction", MI); 1636 1637 // Meta instructions should never be the subject of debug value tracking, 1638 // they don't create a value in the output program at all. 1639 if (MI->isMetaInstruction() && MI->peekDebugInstrNum()) 1640 report("Metadata instruction should not have a value tracking number", MI); 1641 1642 // Check the MachineMemOperands for basic consistency. 1643 for (MachineMemOperand *Op : MI->memoperands()) { 1644 if (Op->isLoad() && !MI->mayLoad()) 1645 report("Missing mayLoad flag", MI); 1646 if (Op->isStore() && !MI->mayStore()) 1647 report("Missing mayStore flag", MI); 1648 } 1649 1650 // Debug values must not have a slot index. 1651 // Other instructions must have one, unless they are inside a bundle. 1652 if (LiveInts) { 1653 bool mapped = !LiveInts->isNotInMIMap(*MI); 1654 if (MI->isDebugOrPseudoInstr()) { 1655 if (mapped) 1656 report("Debug instruction has a slot index", MI); 1657 } else if (MI->isInsideBundle()) { 1658 if (mapped) 1659 report("Instruction inside bundle has a slot index", MI); 1660 } else { 1661 if (!mapped) 1662 report("Missing slot index", MI); 1663 } 1664 } 1665 1666 unsigned Opc = MCID.getOpcode(); 1667 if (isPreISelGenericOpcode(Opc) || isPreISelGenericOptimizationHint(Opc)) { 1668 verifyPreISelGenericInstruction(MI); 1669 return; 1670 } 1671 1672 StringRef ErrorInfo; 1673 if (!TII->verifyInstruction(*MI, ErrorInfo)) 1674 report(ErrorInfo.data(), MI); 1675 1676 // Verify properties of various specific instruction types 1677 switch (MI->getOpcode()) { 1678 case TargetOpcode::COPY: { 1679 const MachineOperand &DstOp = MI->getOperand(0); 1680 const MachineOperand &SrcOp = MI->getOperand(1); 1681 const Register SrcReg = SrcOp.getReg(); 1682 const Register DstReg = DstOp.getReg(); 1683 1684 LLT DstTy = MRI->getType(DstReg); 1685 LLT SrcTy = MRI->getType(SrcReg); 1686 if (SrcTy.isValid() && DstTy.isValid()) { 1687 // If both types are valid, check that the types are the same. 1688 if (SrcTy != DstTy) { 1689 report("Copy Instruction is illegal with mismatching types", MI); 1690 errs() << "Def = " << DstTy << ", Src = " << SrcTy << "\n"; 1691 } 1692 1693 break; 1694 } 1695 1696 if (!SrcTy.isValid() && !DstTy.isValid()) 1697 break; 1698 1699 // If we have only one valid type, this is likely a copy between a virtual 1700 // and physical register. 1701 unsigned SrcSize = 0; 1702 unsigned DstSize = 0; 1703 if (SrcReg.isPhysical() && DstTy.isValid()) { 1704 const TargetRegisterClass *SrcRC = 1705 TRI->getMinimalPhysRegClassLLT(SrcReg, DstTy); 1706 if (SrcRC) 1707 SrcSize = TRI->getRegSizeInBits(*SrcRC); 1708 } 1709 1710 if (SrcSize == 0) 1711 SrcSize = TRI->getRegSizeInBits(SrcReg, *MRI); 1712 1713 if (DstReg.isPhysical() && SrcTy.isValid()) { 1714 const TargetRegisterClass *DstRC = 1715 TRI->getMinimalPhysRegClassLLT(DstReg, SrcTy); 1716 if (DstRC) 1717 DstSize = TRI->getRegSizeInBits(*DstRC); 1718 } 1719 1720 if (DstSize == 0) 1721 DstSize = TRI->getRegSizeInBits(DstReg, *MRI); 1722 1723 if (SrcSize != 0 && DstSize != 0 && SrcSize != DstSize) { 1724 if (!DstOp.getSubReg() && !SrcOp.getSubReg()) { 1725 report("Copy Instruction is illegal with mismatching sizes", MI); 1726 errs() << "Def Size = " << DstSize << ", Src Size = " << SrcSize 1727 << "\n"; 1728 } 1729 } 1730 break; 1731 } 1732 case TargetOpcode::STATEPOINT: { 1733 StatepointOpers SO(MI); 1734 if (!MI->getOperand(SO.getIDPos()).isImm() || 1735 !MI->getOperand(SO.getNBytesPos()).isImm() || 1736 !MI->getOperand(SO.getNCallArgsPos()).isImm()) { 1737 report("meta operands to STATEPOINT not constant!", MI); 1738 break; 1739 } 1740 1741 auto VerifyStackMapConstant = [&](unsigned Offset) { 1742 if (Offset >= MI->getNumOperands()) { 1743 report("stack map constant to STATEPOINT is out of range!", MI); 1744 return; 1745 } 1746 if (!MI->getOperand(Offset - 1).isImm() || 1747 MI->getOperand(Offset - 1).getImm() != StackMaps::ConstantOp || 1748 !MI->getOperand(Offset).isImm()) 1749 report("stack map constant to STATEPOINT not well formed!", MI); 1750 }; 1751 VerifyStackMapConstant(SO.getCCIdx()); 1752 VerifyStackMapConstant(SO.getFlagsIdx()); 1753 VerifyStackMapConstant(SO.getNumDeoptArgsIdx()); 1754 VerifyStackMapConstant(SO.getNumGCPtrIdx()); 1755 VerifyStackMapConstant(SO.getNumAllocaIdx()); 1756 VerifyStackMapConstant(SO.getNumGcMapEntriesIdx()); 1757 1758 // Verify that all explicit statepoint defs are tied to gc operands as 1759 // they are expected to be a relocation of gc operands. 1760 unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx(); 1761 unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2; 1762 for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) { 1763 unsigned UseOpIdx; 1764 if (!MI->isRegTiedToUseOperand(Idx, &UseOpIdx)) { 1765 report("STATEPOINT defs expected to be tied", MI); 1766 break; 1767 } 1768 if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) { 1769 report("STATEPOINT def tied to non-gc operand", MI); 1770 break; 1771 } 1772 } 1773 1774 // TODO: verify we have properly encoded deopt arguments 1775 } break; 1776 } 1777} 1778 1779void 1780MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { 1781 const MachineInstr *MI = MO->getParent(); 1782 const MCInstrDesc &MCID = MI->getDesc(); 1783 unsigned NumDefs = MCID.getNumDefs(); 1784 if (MCID.getOpcode() == TargetOpcode::PATCHPOINT) 1785 NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0; 1786 1787 // The first MCID.NumDefs operands must be explicit register defines 1788 if (MONum < NumDefs) { 1789 const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; 1790 if (!MO->isReg()) 1791 report("Explicit definition must be a register", MO, MONum); 1792 else if (!MO->isDef() && !MCOI.isOptionalDef()) 1793 report("Explicit definition marked as use", MO, MONum); 1794 else if (MO->isImplicit()) 1795 report("Explicit definition marked as implicit", MO, MONum); 1796 } else if (MONum < MCID.getNumOperands()) { 1797 const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; 1798 // Don't check if it's the last operand in a variadic instruction. See, 1799 // e.g., LDM_RET in the arm back end. Check non-variadic operands only. 1800 bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1; 1801 if (!IsOptional) { 1802 if (MO->isReg()) { 1803 if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs()) 1804 report("Explicit operand marked as def", MO, MONum); 1805 if (MO->isImplicit()) 1806 report("Explicit operand marked as implicit", MO, MONum); 1807 } 1808 1809 // Check that an instruction has register operands only as expected. 1810 if (MCOI.OperandType == MCOI::OPERAND_REGISTER && 1811 !MO->isReg() && !MO->isFI()) 1812 report("Expected a register operand.", MO, MONum); 1813 if (MO->isReg()) { 1814 if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE || 1815 (MCOI.OperandType == MCOI::OPERAND_PCREL && 1816 !TII->isPCRelRegisterOperandLegal(*MO))) 1817 report("Expected a non-register operand.", MO, MONum); 1818 } 1819 } 1820 1821 int TiedTo = MCID.getOperandConstraint(MONum, MCOI::TIED_TO); 1822 if (TiedTo != -1) { 1823 if (!MO->isReg()) 1824 report("Tied use must be a register", MO, MONum); 1825 else if (!MO->isTied()) 1826 report("Operand should be tied", MO, MONum); 1827 else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum)) 1828 report("Tied def doesn't match MCInstrDesc", MO, MONum); 1829 else if (Register::isPhysicalRegister(MO->getReg())) { 1830 const MachineOperand &MOTied = MI->getOperand(TiedTo); 1831 if (!MOTied.isReg()) 1832 report("Tied counterpart must be a register", &MOTied, TiedTo); 1833 else if (Register::isPhysicalRegister(MOTied.getReg()) && 1834 MO->getReg() != MOTied.getReg()) 1835 report("Tied physical registers must match.", &MOTied, TiedTo); 1836 } 1837 } else if (MO->isReg() && MO->isTied()) 1838 report("Explicit operand should not be tied", MO, MONum); 1839 } else { 1840 // ARM adds %reg0 operands to indicate predicates. We'll allow that. 1841 if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg()) 1842 report("Extra explicit operand on non-variadic instruction", MO, MONum); 1843 } 1844 1845 switch (MO->getType()) { 1846 case MachineOperand::MO_Register: { 1847 const Register Reg = MO->getReg(); 1848 if (!Reg) 1849 return; 1850 if (MRI->tracksLiveness() && !MI->isDebugValue()) 1851 checkLiveness(MO, MONum); 1852 1853 // Verify the consistency of tied operands. 1854 if (MO->isTied()) { 1855 unsigned OtherIdx = MI->findTiedOperandIdx(MONum); 1856 const MachineOperand &OtherMO = MI->getOperand(OtherIdx); 1857 if (!OtherMO.isReg()) 1858 report("Must be tied to a register", MO, MONum); 1859 if (!OtherMO.isTied()) 1860 report("Missing tie flags on tied operand", MO, MONum); 1861 if (MI->findTiedOperandIdx(OtherIdx) != MONum) 1862 report("Inconsistent tie links", MO, MONum); 1863 if (MONum < MCID.getNumDefs()) { 1864 if (OtherIdx < MCID.getNumOperands()) { 1865 if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO)) 1866 report("Explicit def tied to explicit use without tie constraint", 1867 MO, MONum); 1868 } else { 1869 if (!OtherMO.isImplicit()) 1870 report("Explicit def should be tied to implicit use", MO, MONum); 1871 } 1872 } 1873 } 1874 1875 // Verify two-address constraints after the twoaddressinstruction pass. 1876 // Both twoaddressinstruction pass and phi-node-elimination pass call 1877 // MRI->leaveSSA() to set MF as NoSSA, we should do the verification after 1878 // twoaddressinstruction pass not after phi-node-elimination pass. So we 1879 // shouldn't use the NoSSA as the condition, we should based on 1880 // TiedOpsRewritten property to verify two-address constraints, this 1881 // property will be set in twoaddressinstruction pass. 1882 unsigned DefIdx; 1883 if (MF->getProperties().hasProperty( 1884 MachineFunctionProperties::Property::TiedOpsRewritten) && 1885 MO->isUse() && MI->isRegTiedToDefOperand(MONum, &DefIdx) && 1886 Reg != MI->getOperand(DefIdx).getReg()) 1887 report("Two-address instruction operands must be identical", MO, MONum); 1888 1889 // Check register classes. 1890 unsigned SubIdx = MO->getSubReg(); 1891 1892 if (Register::isPhysicalRegister(Reg)) { 1893 if (SubIdx) { 1894 report("Illegal subregister index for physical register", MO, MONum); 1895 return; 1896 } 1897 if (MONum < MCID.getNumOperands()) { 1898 if (const TargetRegisterClass *DRC = 1899 TII->getRegClass(MCID, MONum, TRI, *MF)) { 1900 if (!DRC->contains(Reg)) { 1901 report("Illegal physical register for instruction", MO, MONum); 1902 errs() << printReg(Reg, TRI) << " is not a " 1903 << TRI->getRegClassName(DRC) << " register.\n"; 1904 } 1905 } 1906 } 1907 if (MO->isRenamable()) { 1908 if (MRI->isReserved(Reg)) { 1909 report("isRenamable set on reserved register", MO, MONum); 1910 return; 1911 } 1912 } 1913 if (MI->isDebugValue() && MO->isUse() && !MO->isDebug()) { 1914 report("Use-reg is not IsDebug in a DBG_VALUE", MO, MONum); 1915 return; 1916 } 1917 } else { 1918 // Virtual register. 1919 const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg); 1920 if (!RC) { 1921 // This is a generic virtual register. 1922 1923 // Do not allow undef uses for generic virtual registers. This ensures 1924 // getVRegDef can never fail and return null on a generic register. 1925 // 1926 // FIXME: This restriction should probably be broadened to all SSA 1927 // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still 1928 // run on the SSA function just before phi elimination. 1929 if (MO->isUndef()) 1930 report("Generic virtual register use cannot be undef", MO, MONum); 1931 1932 // If we're post-Select, we can't have gvregs anymore. 1933 if (isFunctionSelected) { 1934 report("Generic virtual register invalid in a Selected function", 1935 MO, MONum); 1936 return; 1937 } 1938 1939 // The gvreg must have a type and it must not have a SubIdx. 1940 LLT Ty = MRI->getType(Reg); 1941 if (!Ty.isValid()) { 1942 report("Generic virtual register must have a valid type", MO, 1943 MONum); 1944 return; 1945 } 1946 1947 const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg); 1948 1949 // If we're post-RegBankSelect, the gvreg must have a bank. 1950 if (!RegBank && isFunctionRegBankSelected) { 1951 report("Generic virtual register must have a bank in a " 1952 "RegBankSelected function", 1953 MO, MONum); 1954 return; 1955 } 1956 1957 // Make sure the register fits into its register bank if any. 1958 if (RegBank && Ty.isValid() && 1959 RegBank->getSize() < Ty.getSizeInBits()) { 1960 report("Register bank is too small for virtual register", MO, 1961 MONum); 1962 errs() << "Register bank " << RegBank->getName() << " too small(" 1963 << RegBank->getSize() << ") to fit " << Ty.getSizeInBits() 1964 << "-bits\n"; 1965 return; 1966 } 1967 if (SubIdx) { 1968 report("Generic virtual register does not allow subregister index", MO, 1969 MONum); 1970 return; 1971 } 1972 1973 // If this is a target specific instruction and this operand 1974 // has register class constraint, the virtual register must 1975 // comply to it. 1976 if (!isPreISelGenericOpcode(MCID.getOpcode()) && 1977 MONum < MCID.getNumOperands() && 1978 TII->getRegClass(MCID, MONum, TRI, *MF)) { 1979 report("Virtual register does not match instruction constraint", MO, 1980 MONum); 1981 errs() << "Expect register class " 1982 << TRI->getRegClassName( 1983 TII->getRegClass(MCID, MONum, TRI, *MF)) 1984 << " but got nothing\n"; 1985 return; 1986 } 1987 1988 break; 1989 } 1990 if (SubIdx) { 1991 const TargetRegisterClass *SRC = 1992 TRI->getSubClassWithSubReg(RC, SubIdx); 1993 if (!SRC) { 1994 report("Invalid subregister index for virtual register", MO, MONum); 1995 errs() << "Register class " << TRI->getRegClassName(RC) 1996 << " does not support subreg index " << SubIdx << "\n"; 1997 return; 1998 } 1999 if (RC != SRC) { 2000 report("Invalid register class for subregister index", MO, MONum); 2001 errs() << "Register class " << TRI->getRegClassName(RC) 2002 << " does not fully support subreg index " << SubIdx << "\n"; 2003 return; 2004 } 2005 } 2006 if (MONum < MCID.getNumOperands()) { 2007 if (const TargetRegisterClass *DRC = 2008 TII->getRegClass(MCID, MONum, TRI, *MF)) { 2009 if (SubIdx) { 2010 const TargetRegisterClass *SuperRC = 2011 TRI->getLargestLegalSuperClass(RC, *MF); 2012 if (!SuperRC) { 2013 report("No largest legal super class exists.", MO, MONum); 2014 return; 2015 } 2016 DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); 2017 if (!DRC) { 2018 report("No matching super-reg register class.", MO, MONum); 2019 return; 2020 } 2021 } 2022 if (!RC->hasSuperClassEq(DRC)) { 2023 report("Illegal virtual register for instruction", MO, MONum); 2024 errs() << "Expected a " << TRI->getRegClassName(DRC) 2025 << " register, but got a " << TRI->getRegClassName(RC) 2026 << " register\n"; 2027 } 2028 } 2029 } 2030 } 2031 break; 2032 } 2033 2034 case MachineOperand::MO_RegisterMask: 2035 regMasks.push_back(MO->getRegMask()); 2036 break; 2037 2038 case MachineOperand::MO_MachineBasicBlock: 2039 if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) 2040 report("PHI operand is not in the CFG", MO, MONum); 2041 break; 2042 2043 case MachineOperand::MO_FrameIndex: 2044 if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && 2045 LiveInts && !LiveInts->isNotInMIMap(*MI)) { 2046 int FI = MO->getIndex(); 2047 LiveInterval &LI = LiveStks->getInterval(FI); 2048 SlotIndex Idx = LiveInts->getInstructionIndex(*MI); 2049 2050 bool stores = MI->mayStore(); 2051 bool loads = MI->mayLoad(); 2052 // For a memory-to-memory move, we need to check if the frame 2053 // index is used for storing or loading, by inspecting the 2054 // memory operands. 2055 if (stores && loads) { 2056 for (auto *MMO : MI->memoperands()) { 2057 const PseudoSourceValue *PSV = MMO->getPseudoValue(); 2058 if (PSV == nullptr) continue; 2059 const FixedStackPseudoSourceValue *Value = 2060 dyn_cast<FixedStackPseudoSourceValue>(PSV); 2061 if (Value == nullptr) continue; 2062 if (Value->getFrameIndex() != FI) continue; 2063 2064 if (MMO->isStore()) 2065 loads = false; 2066 else 2067 stores = false; 2068 break; 2069 } 2070 if (loads == stores) 2071 report("Missing fixed stack memoperand.", MI); 2072 } 2073 if (loads && !LI.liveAt(Idx.getRegSlot(true))) { 2074 report("Instruction loads from dead spill slot", MO, MONum); 2075 errs() << "Live stack: " << LI << '\n'; 2076 } 2077 if (stores && !LI.liveAt(Idx.getRegSlot())) { 2078 report("Instruction stores to dead spill slot", MO, MONum); 2079 errs() << "Live stack: " << LI << '\n'; 2080 } 2081 } 2082 break; 2083 2084 default: 2085 break; 2086 } 2087} 2088 2089void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO, 2090 unsigned MONum, SlotIndex UseIdx, 2091 const LiveRange &LR, 2092 Register VRegOrUnit, 2093 LaneBitmask LaneMask) { 2094 LiveQueryResult LRQ = LR.Query(UseIdx); 2095 // Check if we have a segment at the use, note however that we only need one 2096 // live subregister range, the others may be dead. 2097 if (!LRQ.valueIn() && LaneMask.none()) { 2098 report("No live segment at use", MO, MONum); 2099 report_context_liverange(LR); 2100 report_context_vreg_regunit(VRegOrUnit); 2101 report_context(UseIdx); 2102 } 2103 if (MO->isKill() && !LRQ.isKill()) { 2104 report("Live range continues after kill flag", MO, MONum); 2105 report_context_liverange(LR); 2106 report_context_vreg_regunit(VRegOrUnit); 2107 if (LaneMask.any()) 2108 report_context_lanemask(LaneMask); 2109 report_context(UseIdx); 2110 } 2111} 2112 2113void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO, 2114 unsigned MONum, SlotIndex DefIdx, 2115 const LiveRange &LR, 2116 Register VRegOrUnit, 2117 bool SubRangeCheck, 2118 LaneBitmask LaneMask) { 2119 if (const VNInfo *VNI = LR.getVNInfoAt(DefIdx)) { 2120 assert(VNI && "NULL valno is not allowed"); 2121 if (VNI->def != DefIdx) { 2122 report("Inconsistent valno->def", MO, MONum); 2123 report_context_liverange(LR); 2124 report_context_vreg_regunit(VRegOrUnit); 2125 if (LaneMask.any()) 2126 report_context_lanemask(LaneMask); 2127 report_context(*VNI); 2128 report_context(DefIdx); 2129 } 2130 } else { 2131 report("No live segment at def", MO, MONum); 2132 report_context_liverange(LR); 2133 report_context_vreg_regunit(VRegOrUnit); 2134 if (LaneMask.any()) 2135 report_context_lanemask(LaneMask); 2136 report_context(DefIdx); 2137 } 2138 // Check that, if the dead def flag is present, LiveInts agree. 2139 if (MO->isDead()) { 2140 LiveQueryResult LRQ = LR.Query(DefIdx); 2141 if (!LRQ.isDeadDef()) { 2142 assert(Register::isVirtualRegister(VRegOrUnit) && 2143 "Expecting a virtual register."); 2144 // A dead subreg def only tells us that the specific subreg is dead. There 2145 // could be other non-dead defs of other subregs, or we could have other 2146 // parts of the register being live through the instruction. So unless we 2147 // are checking liveness for a subrange it is ok for the live range to 2148 // continue, given that we have a dead def of a subregister. 2149 if (SubRangeCheck || MO->getSubReg() == 0) { 2150 report("Live range continues after dead def flag", MO, MONum); 2151 report_context_liverange(LR); 2152 report_context_vreg_regunit(VRegOrUnit); 2153 if (LaneMask.any()) 2154 report_context_lanemask(LaneMask); 2155 } 2156 } 2157 } 2158} 2159 2160void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) { 2161 const MachineInstr *MI = MO->getParent(); 2162 const Register Reg = MO->getReg(); 2163 2164 // Both use and def operands can read a register. 2165 if (MO->readsReg()) { 2166 if (MO->isKill()) 2167 addRegWithSubRegs(regsKilled, Reg); 2168 2169 // Check that LiveVars knows this kill. 2170 if (LiveVars && Register::isVirtualRegister(Reg) && MO->isKill()) { 2171 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); 2172 if (!is_contained(VI.Kills, MI)) 2173 report("Kill missing from LiveVariables", MO, MONum); 2174 } 2175 2176 // Check LiveInts liveness and kill. 2177 if (LiveInts && !LiveInts->isNotInMIMap(*MI)) { 2178 SlotIndex UseIdx = LiveInts->getInstructionIndex(*MI); 2179 // Check the cached regunit intervals. 2180 if (Reg.isPhysical() && !isReserved(Reg)) { 2181 for (MCRegUnitIterator Units(Reg.asMCReg(), TRI); Units.isValid(); 2182 ++Units) { 2183 if (MRI->isReservedRegUnit(*Units)) 2184 continue; 2185 if (const LiveRange *LR = LiveInts->getCachedRegUnit(*Units)) 2186 checkLivenessAtUse(MO, MONum, UseIdx, *LR, *Units); 2187 } 2188 } 2189 2190 if (Register::isVirtualRegister(Reg)) { 2191 if (LiveInts->hasInterval(Reg)) { 2192 // This is a virtual register interval. 2193 const LiveInterval &LI = LiveInts->getInterval(Reg); 2194 checkLivenessAtUse(MO, MONum, UseIdx, LI, Reg); 2195 2196 if (LI.hasSubRanges() && !MO->isDef()) { 2197 unsigned SubRegIdx = MO->getSubReg(); 2198 LaneBitmask MOMask = SubRegIdx != 0 2199 ? TRI->getSubRegIndexLaneMask(SubRegIdx) 2200 : MRI->getMaxLaneMaskForVReg(Reg); 2201 LaneBitmask LiveInMask; 2202 for (const LiveInterval::SubRange &SR : LI.subranges()) { 2203 if ((MOMask & SR.LaneMask).none()) 2204 continue; 2205 checkLivenessAtUse(MO, MONum, UseIdx, SR, Reg, SR.LaneMask); 2206 LiveQueryResult LRQ = SR.Query(UseIdx); 2207 if (LRQ.valueIn()) 2208 LiveInMask |= SR.LaneMask; 2209 } 2210 // At least parts of the register has to be live at the use. 2211 if ((LiveInMask & MOMask).none()) { 2212 report("No live subrange at use", MO, MONum); 2213 report_context(LI); 2214 report_context(UseIdx); 2215 } 2216 } 2217 } else { 2218 report("Virtual register has no live interval", MO, MONum); 2219 } 2220 } 2221 } 2222 2223 // Use of a dead register. 2224 if (!regsLive.count(Reg)) { 2225 if (Register::isPhysicalRegister(Reg)) { 2226 // Reserved registers may be used even when 'dead'. 2227 bool Bad = !isReserved(Reg); 2228 // We are fine if just any subregister has a defined value. 2229 if (Bad) { 2230 2231 for (const MCPhysReg &SubReg : TRI->subregs(Reg)) { 2232 if (regsLive.count(SubReg)) { 2233 Bad = false; 2234 break; 2235 } 2236 } 2237 } 2238 // If there is an additional implicit-use of a super register we stop 2239 // here. By definition we are fine if the super register is not 2240 // (completely) dead, if the complete super register is dead we will 2241 // get a report for its operand. 2242 if (Bad) { 2243 for (const MachineOperand &MOP : MI->uses()) { 2244 if (!MOP.isReg() || !MOP.isImplicit()) 2245 continue; 2246 2247 if (!Register::isPhysicalRegister(MOP.getReg())) 2248 continue; 2249 2250 if (llvm::is_contained(TRI->subregs(MOP.getReg()), Reg)) 2251 Bad = false; 2252 } 2253 } 2254 if (Bad) 2255 report("Using an undefined physical register", MO, MONum); 2256 } else if (MRI->def_empty(Reg)) { 2257 report("Reading virtual register without a def", MO, MONum); 2258 } else { 2259 BBInfo &MInfo = MBBInfoMap[MI->getParent()]; 2260 // We don't know which virtual registers are live in, so only complain 2261 // if vreg was killed in this MBB. Otherwise keep track of vregs that 2262 // must be live in. PHI instructions are handled separately. 2263 if (MInfo.regsKilled.count(Reg)) 2264 report("Using a killed virtual register", MO, MONum); 2265 else if (!MI->isPHI()) 2266 MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI)); 2267 } 2268 } 2269 } 2270 2271 if (MO->isDef()) { 2272 // Register defined. 2273 // TODO: verify that earlyclobber ops are not used. 2274 if (MO->isDead()) 2275 addRegWithSubRegs(regsDead, Reg); 2276 else 2277 addRegWithSubRegs(regsDefined, Reg); 2278 2279 // Verify SSA form. 2280 if (MRI->isSSA() && Register::isVirtualRegister(Reg) && 2281 std::next(MRI->def_begin(Reg)) != MRI->def_end()) 2282 report("Multiple virtual register defs in SSA form", MO, MONum); 2283 2284 // Check LiveInts for a live segment, but only for virtual registers. 2285 if (LiveInts && !LiveInts->isNotInMIMap(*MI)) { 2286 SlotIndex DefIdx = LiveInts->getInstructionIndex(*MI); 2287 DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber()); 2288 2289 if (Register::isVirtualRegister(Reg)) { 2290 if (LiveInts->hasInterval(Reg)) { 2291 const LiveInterval &LI = LiveInts->getInterval(Reg); 2292 checkLivenessAtDef(MO, MONum, DefIdx, LI, Reg); 2293 2294 if (LI.hasSubRanges()) { 2295 unsigned SubRegIdx = MO->getSubReg(); 2296 LaneBitmask MOMask = SubRegIdx != 0 2297 ? TRI->getSubRegIndexLaneMask(SubRegIdx) 2298 : MRI->getMaxLaneMaskForVReg(Reg); 2299 for (const LiveInterval::SubRange &SR : LI.subranges()) { 2300 if ((SR.LaneMask & MOMask).none()) 2301 continue; 2302 checkLivenessAtDef(MO, MONum, DefIdx, SR, Reg, true, SR.LaneMask); 2303 } 2304 } 2305 } else { 2306 report("Virtual register has no Live interval", MO, MONum); 2307 } 2308 } 2309 } 2310 } 2311} 2312 2313// This function gets called after visiting all instructions in a bundle. The 2314// argument points to the bundle header. 2315// Normal stand-alone instructions are also considered 'bundles', and this 2316// function is called for all of them. 2317void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) { 2318 BBInfo &MInfo = MBBInfoMap[MI->getParent()]; 2319 set_union(MInfo.regsKilled, regsKilled); 2320 set_subtract(regsLive, regsKilled); regsKilled.clear(); 2321 // Kill any masked registers. 2322 while (!regMasks.empty()) { 2323 const uint32_t *Mask = regMasks.pop_back_val(); 2324 for (Register Reg : regsLive) 2325 if (Reg.isPhysical() && 2326 MachineOperand::clobbersPhysReg(Mask, Reg.asMCReg())) 2327 regsDead.push_back(Reg); 2328 } 2329 set_subtract(regsLive, regsDead); regsDead.clear(); 2330 set_union(regsLive, regsDefined); regsDefined.clear(); 2331} 2332 2333void 2334MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) { 2335 MBBInfoMap[MBB].regsLiveOut = regsLive; 2336 regsLive.clear(); 2337 2338 if (Indexes) { 2339 SlotIndex stop = Indexes->getMBBEndIdx(MBB); 2340 if (!(stop > lastIndex)) { 2341 report("Block ends before last instruction index", MBB); 2342 errs() << "Block ends at " << stop 2343 << " last instruction was at " << lastIndex << '\n'; 2344 } 2345 lastIndex = stop; 2346 } 2347} 2348 2349namespace { 2350// This implements a set of registers that serves as a filter: can filter other 2351// sets by passing through elements not in the filter and blocking those that 2352// are. Any filter implicitly includes the full set of physical registers upon 2353// creation, thus filtering them all out. The filter itself as a set only grows, 2354// and needs to be as efficient as possible. 2355struct VRegFilter { 2356 // Add elements to the filter itself. \pre Input set \p FromRegSet must have 2357 // no duplicates. Both virtual and physical registers are fine. 2358 template <typename RegSetT> void add(const RegSetT &FromRegSet) { 2359 SmallVector<Register, 0> VRegsBuffer; 2360 filterAndAdd(FromRegSet, VRegsBuffer); 2361 } 2362 // Filter \p FromRegSet through the filter and append passed elements into \p 2363 // ToVRegs. All elements appended are then added to the filter itself. 2364 // \returns true if anything changed. 2365 template <typename RegSetT> 2366 bool filterAndAdd(const RegSetT &FromRegSet, 2367 SmallVectorImpl<Register> &ToVRegs) { 2368 unsigned SparseUniverse = Sparse.size(); 2369 unsigned NewSparseUniverse = SparseUniverse; 2370 unsigned NewDenseSize = Dense.size(); 2371 size_t Begin = ToVRegs.size(); 2372 for (Register Reg : FromRegSet) { 2373 if (!Reg.isVirtual()) 2374 continue; 2375 unsigned Index = Register::virtReg2Index(Reg); 2376 if (Index < SparseUniverseMax) { 2377 if (Index < SparseUniverse && Sparse.test(Index)) 2378 continue; 2379 NewSparseUniverse = std::max(NewSparseUniverse, Index + 1); 2380 } else { 2381 if (Dense.count(Reg)) 2382 continue; 2383 ++NewDenseSize; 2384 } 2385 ToVRegs.push_back(Reg); 2386 } 2387 size_t End = ToVRegs.size(); 2388 if (Begin == End) 2389 return false; 2390 // Reserving space in sets once performs better than doing so continuously 2391 // and pays easily for double look-ups (even in Dense with SparseUniverseMax 2392 // tuned all the way down) and double iteration (the second one is over a 2393 // SmallVector, which is a lot cheaper compared to DenseSet or BitVector). 2394 Sparse.resize(NewSparseUniverse); 2395 Dense.reserve(NewDenseSize); 2396 for (unsigned I = Begin; I < End; ++I) { 2397 Register Reg = ToVRegs[I]; 2398 unsigned Index = Register::virtReg2Index(Reg); 2399 if (Index < SparseUniverseMax) 2400 Sparse.set(Index); 2401 else 2402 Dense.insert(Reg); 2403 } 2404 return true; 2405 } 2406 2407private: 2408 static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8; 2409 // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyound 2410 // are tracked by Dense. The only purpose of the threashold and the Dense set 2411 // is to have a reasonably growing memory usage in pathological cases (large 2412 // number of very sparse VRegFilter instances live at the same time). In 2413 // practice even in the worst-by-execution time cases having all elements 2414 // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more 2415 // space efficient than if tracked by Dense. The threashold is set to keep the 2416 // worst-case memory usage within 2x of figures determined empirically for 2417 // "all Dense" scenario in such worst-by-execution-time cases. 2418 BitVector Sparse; 2419 DenseSet<unsigned> Dense; 2420}; 2421 2422// Implements both a transfer function and a (binary, in-place) join operator 2423// for a dataflow over register sets with set union join and filtering transfer 2424// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time. 2425// Maintains out_b as its state, allowing for O(n) iteration over it at any 2426// time, where n is the size of the set (as opposed to O(U) where U is the 2427// universe). filter_b implicitly contains all physical registers at all times. 2428class FilteringVRegSet { 2429 VRegFilter Filter; 2430 SmallVector<Register, 0> VRegs; 2431 2432public: 2433 // Set-up the filter_b. \pre Input register set \p RS must have no duplicates. 2434 // Both virtual and physical registers are fine. 2435 template <typename RegSetT> void addToFilter(const RegSetT &RS) { 2436 Filter.add(RS); 2437 } 2438 // Passes \p RS through the filter_b (transfer function) and adds what's left 2439 // to itself (out_b). 2440 template <typename RegSetT> bool add(const RegSetT &RS) { 2441 // Double-duty the Filter: to maintain VRegs a set (and the join operation 2442 // a set union) just add everything being added here to the Filter as well. 2443 return Filter.filterAndAdd(RS, VRegs); 2444 } 2445 using const_iterator = decltype(VRegs)::const_iterator; 2446 const_iterator begin() const { return VRegs.begin(); } 2447 const_iterator end() const { return VRegs.end(); } 2448 size_t size() const { return VRegs.size(); } 2449}; 2450} // namespace 2451 2452// Calculate the largest possible vregsPassed sets. These are the registers that 2453// can pass through an MBB live, but may not be live every time. It is assumed 2454// that all vregsPassed sets are empty before the call. 2455void MachineVerifier::calcRegsPassed() { 2456 if (MF->empty()) 2457 // ReversePostOrderTraversal doesn't handle empty functions. 2458 return; 2459 2460 for (const MachineBasicBlock *MB : 2461 ReversePostOrderTraversal<const MachineFunction *>(MF)) { 2462 FilteringVRegSet VRegs; 2463 BBInfo &Info = MBBInfoMap[MB]; 2464 assert(Info.reachable); 2465 2466 VRegs.addToFilter(Info.regsKilled); 2467 VRegs.addToFilter(Info.regsLiveOut); 2468 for (const MachineBasicBlock *Pred : MB->predecessors()) { 2469 const BBInfo &PredInfo = MBBInfoMap[Pred]; 2470 if (!PredInfo.reachable) 2471 continue; 2472 2473 VRegs.add(PredInfo.regsLiveOut); 2474 VRegs.add(PredInfo.vregsPassed); 2475 } 2476 Info.vregsPassed.reserve(VRegs.size()); 2477 Info.vregsPassed.insert(VRegs.begin(), VRegs.end()); 2478 } 2479} 2480 2481// Calculate the set of virtual registers that must be passed through each basic 2482// block in order to satisfy the requirements of successor blocks. This is very 2483// similar to calcRegsPassed, only backwards. 2484void MachineVerifier::calcRegsRequired() { 2485 // First push live-in regs to predecessors' vregsRequired. 2486 SmallPtrSet<const MachineBasicBlock*, 8> todo; 2487 for (const auto &MBB : *MF) { 2488 BBInfo &MInfo = MBBInfoMap[&MBB]; 2489 for (const MachineBasicBlock *Pred : MBB.predecessors()) { 2490 BBInfo &PInfo = MBBInfoMap[Pred]; 2491 if (PInfo.addRequired(MInfo.vregsLiveIn)) 2492 todo.insert(Pred); 2493 } 2494 2495 // Handle the PHI node. 2496 for (const MachineInstr &MI : MBB.phis()) { 2497 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 2498 // Skip those Operands which are undef regs or not regs. 2499 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg()) 2500 continue; 2501 2502 // Get register and predecessor for one PHI edge. 2503 Register Reg = MI.getOperand(i).getReg(); 2504 const MachineBasicBlock *Pred = MI.getOperand(i + 1).getMBB(); 2505 2506 BBInfo &PInfo = MBBInfoMap[Pred]; 2507 if (PInfo.addRequired(Reg)) 2508 todo.insert(Pred); 2509 } 2510 } 2511 } 2512 2513 // Iteratively push vregsRequired to predecessors. This will converge to the 2514 // same final state regardless of DenseSet iteration order. 2515 while (!todo.empty()) { 2516 const MachineBasicBlock *MBB = *todo.begin(); 2517 todo.erase(MBB); 2518 BBInfo &MInfo = MBBInfoMap[MBB]; 2519 for (const MachineBasicBlock *Pred : MBB->predecessors()) { 2520 if (Pred == MBB) 2521 continue; 2522 BBInfo &SInfo = MBBInfoMap[Pred]; 2523 if (SInfo.addRequired(MInfo.vregsRequired)) 2524 todo.insert(Pred); 2525 } 2526 } 2527} 2528 2529// Check PHI instructions at the beginning of MBB. It is assumed that 2530// calcRegsPassed has been run so BBInfo::isLiveOut is valid. 2531void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) { 2532 BBInfo &MInfo = MBBInfoMap[&MBB]; 2533 2534 SmallPtrSet<const MachineBasicBlock*, 8> seen; 2535 for (const MachineInstr &Phi : MBB) { 2536 if (!Phi.isPHI()) 2537 break; 2538 seen.clear(); 2539 2540 const MachineOperand &MODef = Phi.getOperand(0); 2541 if (!MODef.isReg() || !MODef.isDef()) { 2542 report("Expected first PHI operand to be a register def", &MODef, 0); 2543 continue; 2544 } 2545 if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() || 2546 MODef.isEarlyClobber() || MODef.isDebug()) 2547 report("Unexpected flag on PHI operand", &MODef, 0); 2548 Register DefReg = MODef.getReg(); 2549 if (!Register::isVirtualRegister(DefReg)) 2550 report("Expected first PHI operand to be a virtual register", &MODef, 0); 2551 2552 for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) { 2553 const MachineOperand &MO0 = Phi.getOperand(I); 2554 if (!MO0.isReg()) { 2555 report("Expected PHI operand to be a register", &MO0, I); 2556 continue; 2557 } 2558 if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() || 2559 MO0.isDebug() || MO0.isTied()) 2560 report("Unexpected flag on PHI operand", &MO0, I); 2561 2562 const MachineOperand &MO1 = Phi.getOperand(I + 1); 2563 if (!MO1.isMBB()) { 2564 report("Expected PHI operand to be a basic block", &MO1, I + 1); 2565 continue; 2566 } 2567 2568 const MachineBasicBlock &Pre = *MO1.getMBB(); 2569 if (!Pre.isSuccessor(&MBB)) { 2570 report("PHI input is not a predecessor block", &MO1, I + 1); 2571 continue; 2572 } 2573 2574 if (MInfo.reachable) { 2575 seen.insert(&Pre); 2576 BBInfo &PrInfo = MBBInfoMap[&Pre]; 2577 if (!MO0.isUndef() && PrInfo.reachable && 2578 !PrInfo.isLiveOut(MO0.getReg())) 2579 report("PHI operand is not live-out from predecessor", &MO0, I); 2580 } 2581 } 2582 2583 // Did we see all predecessors? 2584 if (MInfo.reachable) { 2585 for (MachineBasicBlock *Pred : MBB.predecessors()) { 2586 if (!seen.count(Pred)) { 2587 report("Missing PHI operand", &Phi); 2588 errs() << printMBBReference(*Pred) 2589 << " is a predecessor according to the CFG.\n"; 2590 } 2591 } 2592 } 2593 } 2594} 2595 2596void MachineVerifier::visitMachineFunctionAfter() { 2597 calcRegsPassed(); 2598 2599 for (const MachineBasicBlock &MBB : *MF) 2600 checkPHIOps(MBB); 2601 2602 // Now check liveness info if available 2603 calcRegsRequired(); 2604 2605 // Check for killed virtual registers that should be live out. 2606 for (const auto &MBB : *MF) { 2607 BBInfo &MInfo = MBBInfoMap[&MBB]; 2608 for (Register VReg : MInfo.vregsRequired) 2609 if (MInfo.regsKilled.count(VReg)) { 2610 report("Virtual register killed in block, but needed live out.", &MBB); 2611 errs() << "Virtual register " << printReg(VReg) 2612 << " is used after the block.\n"; 2613 } 2614 } 2615 2616 if (!MF->empty()) { 2617 BBInfo &MInfo = MBBInfoMap[&MF->front()]; 2618 for (Register VReg : MInfo.vregsRequired) { 2619 report("Virtual register defs don't dominate all uses.", MF); 2620 report_context_vreg(VReg); 2621 } 2622 } 2623 2624 if (LiveVars) 2625 verifyLiveVariables(); 2626 if (LiveInts) 2627 verifyLiveIntervals(); 2628 2629 // Check live-in list of each MBB. If a register is live into MBB, check 2630 // that the register is in regsLiveOut of each predecessor block. Since 2631 // this must come from a definition in the predecesssor or its live-in 2632 // list, this will catch a live-through case where the predecessor does not 2633 // have the register in its live-in list. This currently only checks 2634 // registers that have no aliases, are not allocatable and are not 2635 // reserved, which could mean a condition code register for instance. 2636 if (MRI->tracksLiveness()) 2637 for (const auto &MBB : *MF) 2638 for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) { 2639 MCPhysReg LiveInReg = P.PhysReg; 2640 bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid(); 2641 if (hasAliases || isAllocatable(LiveInReg) || isReserved(LiveInReg)) 2642 continue; 2643 for (const MachineBasicBlock *Pred : MBB.predecessors()) { 2644 BBInfo &PInfo = MBBInfoMap[Pred]; 2645 if (!PInfo.regsLiveOut.count(LiveInReg)) { 2646 report("Live in register not found to be live out from predecessor.", 2647 &MBB); 2648 errs() << TRI->getName(LiveInReg) 2649 << " not found to be live out from " 2650 << printMBBReference(*Pred) << "\n"; 2651 } 2652 } 2653 } 2654 2655 for (auto CSInfo : MF->getCallSitesInfo()) 2656 if (!CSInfo.first->isCall()) 2657 report("Call site info referencing instruction that is not call", MF); 2658 2659 // If there's debug-info, check that we don't have any duplicate value 2660 // tracking numbers. 2661 if (MF->getFunction().getSubprogram()) { 2662 DenseSet<unsigned> SeenNumbers; 2663 for (auto &MBB : *MF) { 2664 for (auto &MI : MBB) { 2665 if (auto Num = MI.peekDebugInstrNum()) { 2666 auto Result = SeenNumbers.insert((unsigned)Num); 2667 if (!Result.second) 2668 report("Instruction has a duplicated value tracking number", &MI); 2669 } 2670 } 2671 } 2672 } 2673} 2674 2675void MachineVerifier::verifyLiveVariables() { 2676 assert(LiveVars && "Don't call verifyLiveVariables without LiveVars"); 2677 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) { 2678 Register Reg = Register::index2VirtReg(I); 2679 LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); 2680 for (const auto &MBB : *MF) { 2681 BBInfo &MInfo = MBBInfoMap[&MBB]; 2682 2683 // Our vregsRequired should be identical to LiveVariables' AliveBlocks 2684 if (MInfo.vregsRequired.count(Reg)) { 2685 if (!VI.AliveBlocks.test(MBB.getNumber())) { 2686 report("LiveVariables: Block missing from AliveBlocks", &MBB); 2687 errs() << "Virtual register " << printReg(Reg) 2688 << " must be live through the block.\n"; 2689 } 2690 } else { 2691 if (VI.AliveBlocks.test(MBB.getNumber())) { 2692 report("LiveVariables: Block should not be in AliveBlocks", &MBB); 2693 errs() << "Virtual register " << printReg(Reg) 2694 << " is not needed live through the block.\n"; 2695 } 2696 } 2697 } 2698 } 2699} 2700 2701void MachineVerifier::verifyLiveIntervals() { 2702 assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts"); 2703 for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) { 2704 Register Reg = Register::index2VirtReg(I); 2705 2706 // Spilling and splitting may leave unused registers around. Skip them. 2707 if (MRI->reg_nodbg_empty(Reg)) 2708 continue; 2709 2710 if (!LiveInts->hasInterval(Reg)) { 2711 report("Missing live interval for virtual register", MF); 2712 errs() << printReg(Reg, TRI) << " still has defs or uses\n"; 2713 continue; 2714 } 2715 2716 const LiveInterval &LI = LiveInts->getInterval(Reg); 2717 assert(Reg == LI.reg() && "Invalid reg to interval mapping"); 2718 verifyLiveInterval(LI); 2719 } 2720 2721 // Verify all the cached regunit intervals. 2722 for (unsigned i = 0, e = TRI->getNumRegUnits(); i != e; ++i) 2723 if (const LiveRange *LR = LiveInts->getCachedRegUnit(i)) 2724 verifyLiveRange(*LR, i); 2725} 2726 2727void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, 2728 const VNInfo *VNI, Register Reg, 2729 LaneBitmask LaneMask) { 2730 if (VNI->isUnused()) 2731 return; 2732 2733 const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def); 2734 2735 if (!DefVNI) { 2736 report("Value not live at VNInfo def and not marked unused", MF); 2737 report_context(LR, Reg, LaneMask); 2738 report_context(*VNI); 2739 return; 2740 } 2741 2742 if (DefVNI != VNI) { 2743 report("Live segment at def has different VNInfo", MF); 2744 report_context(LR, Reg, LaneMask); 2745 report_context(*VNI); 2746 return; 2747 } 2748 2749 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def); 2750 if (!MBB) { 2751 report("Invalid VNInfo definition index", MF); 2752 report_context(LR, Reg, LaneMask); 2753 report_context(*VNI); 2754 return; 2755 } 2756 2757 if (VNI->isPHIDef()) { 2758 if (VNI->def != LiveInts->getMBBStartIdx(MBB)) { 2759 report("PHIDef VNInfo is not defined at MBB start", MBB); 2760 report_context(LR, Reg, LaneMask); 2761 report_context(*VNI); 2762 } 2763 return; 2764 } 2765 2766 // Non-PHI def. 2767 const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def); 2768 if (!MI) { 2769 report("No instruction at VNInfo def index", MBB); 2770 report_context(LR, Reg, LaneMask); 2771 report_context(*VNI); 2772 return; 2773 } 2774 2775 if (Reg != 0) { 2776 bool hasDef = false; 2777 bool isEarlyClobber = false; 2778 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) { 2779 if (!MOI->isReg() || !MOI->isDef()) 2780 continue; 2781 if (Register::isVirtualRegister(Reg)) { 2782 if (MOI->getReg() != Reg) 2783 continue; 2784 } else { 2785 if (!Register::isPhysicalRegister(MOI->getReg()) || 2786 !TRI->hasRegUnit(MOI->getReg(), Reg)) 2787 continue; 2788 } 2789 if (LaneMask.any() && 2790 (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none()) 2791 continue; 2792 hasDef = true; 2793 if (MOI->isEarlyClobber()) 2794 isEarlyClobber = true; 2795 } 2796 2797 if (!hasDef) { 2798 report("Defining instruction does not modify register", MI); 2799 report_context(LR, Reg, LaneMask); 2800 report_context(*VNI); 2801 } 2802 2803 // Early clobber defs begin at USE slots, but other defs must begin at 2804 // DEF slots. 2805 if (isEarlyClobber) { 2806 if (!VNI->def.isEarlyClobber()) { 2807 report("Early clobber def must be at an early-clobber slot", MBB); 2808 report_context(LR, Reg, LaneMask); 2809 report_context(*VNI); 2810 } 2811 } else if (!VNI->def.isRegister()) { 2812 report("Non-PHI, non-early clobber def must be at a register slot", MBB); 2813 report_context(LR, Reg, LaneMask); 2814 report_context(*VNI); 2815 } 2816 } 2817} 2818 2819void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, 2820 const LiveRange::const_iterator I, 2821 Register Reg, 2822 LaneBitmask LaneMask) { 2823 const LiveRange::Segment &S = *I; 2824 const VNInfo *VNI = S.valno; 2825 assert(VNI && "Live segment has no valno"); 2826 2827 if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) { 2828 report("Foreign valno in live segment", MF); 2829 report_context(LR, Reg, LaneMask); 2830 report_context(S); 2831 report_context(*VNI); 2832 } 2833 2834 if (VNI->isUnused()) { 2835 report("Live segment valno is marked unused", MF); 2836 report_context(LR, Reg, LaneMask); 2837 report_context(S); 2838 } 2839 2840 const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start); 2841 if (!MBB) { 2842 report("Bad start of live segment, no basic block", MF); 2843 report_context(LR, Reg, LaneMask); 2844 report_context(S); 2845 return; 2846 } 2847 SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB); 2848 if (S.start != MBBStartIdx && S.start != VNI->def) { 2849 report("Live segment must begin at MBB entry or valno def", MBB); 2850 report_context(LR, Reg, LaneMask); 2851 report_context(S); 2852 } 2853 2854 const MachineBasicBlock *EndMBB = 2855 LiveInts->getMBBFromIndex(S.end.getPrevSlot()); 2856 if (!EndMBB) { 2857 report("Bad end of live segment, no basic block", MF); 2858 report_context(LR, Reg, LaneMask); 2859 report_context(S); 2860 return; 2861 } 2862 2863 // No more checks for live-out segments. 2864 if (S.end == LiveInts->getMBBEndIdx(EndMBB)) 2865 return; 2866 2867 // RegUnit intervals are allowed dead phis. 2868 if (!Register::isVirtualRegister(Reg) && VNI->isPHIDef() && 2869 S.start == VNI->def && S.end == VNI->def.getDeadSlot()) 2870 return; 2871 2872 // The live segment is ending inside EndMBB 2873 const MachineInstr *MI = 2874 LiveInts->getInstructionFromIndex(S.end.getPrevSlot()); 2875 if (!MI) { 2876 report("Live segment doesn't end at a valid instruction", EndMBB); 2877 report_context(LR, Reg, LaneMask); 2878 report_context(S); 2879 return; 2880 } 2881 2882 // The block slot must refer to a basic block boundary. 2883 if (S.end.isBlock()) { 2884 report("Live segment ends at B slot of an instruction", EndMBB); 2885 report_context(LR, Reg, LaneMask); 2886 report_context(S); 2887 } 2888 2889 if (S.end.isDead()) { 2890 // Segment ends on the dead slot. 2891 // That means there must be a dead def. 2892 if (!SlotIndex::isSameInstr(S.start, S.end)) { 2893 report("Live segment ending at dead slot spans instructions", EndMBB); 2894 report_context(LR, Reg, LaneMask); 2895 report_context(S); 2896 } 2897 } 2898 2899 // A live segment can only end at an early-clobber slot if it is being 2900 // redefined by an early-clobber def. 2901 if (S.end.isEarlyClobber()) { 2902 if (I+1 == LR.end() || (I+1)->start != S.end) { 2903 report("Live segment ending at early clobber slot must be " 2904 "redefined by an EC def in the same instruction", EndMBB); 2905 report_context(LR, Reg, LaneMask); 2906 report_context(S); 2907 } 2908 } 2909 2910 // The following checks only apply to virtual registers. Physreg liveness 2911 // is too weird to check. 2912 if (Register::isVirtualRegister(Reg)) { 2913 // A live segment can end with either a redefinition, a kill flag on a 2914 // use, or a dead flag on a def. 2915 bool hasRead = false; 2916 bool hasSubRegDef = false; 2917 bool hasDeadDef = false; 2918 for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) { 2919 if (!MOI->isReg() || MOI->getReg() != Reg) 2920 continue; 2921 unsigned Sub = MOI->getSubReg(); 2922 LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) 2923 : LaneBitmask::getAll(); 2924 if (MOI->isDef()) { 2925 if (Sub != 0) { 2926 hasSubRegDef = true; 2927 // An operand %0:sub0 reads %0:sub1..n. Invert the lane 2928 // mask for subregister defs. Read-undef defs will be handled by 2929 // readsReg below. 2930 SLM = ~SLM; 2931 } 2932 if (MOI->isDead()) 2933 hasDeadDef = true; 2934 } 2935 if (LaneMask.any() && (LaneMask & SLM).none()) 2936 continue; 2937 if (MOI->readsReg()) 2938 hasRead = true; 2939 } 2940 if (S.end.isDead()) { 2941 // Make sure that the corresponding machine operand for a "dead" live 2942 // range has the dead flag. We cannot perform this check for subregister 2943 // liveranges as partially dead values are allowed. 2944 if (LaneMask.none() && !hasDeadDef) { 2945 report("Instruction ending live segment on dead slot has no dead flag", 2946 MI); 2947 report_context(LR, Reg, LaneMask); 2948 report_context(S); 2949 } 2950 } else { 2951 if (!hasRead) { 2952 // When tracking subregister liveness, the main range must start new 2953 // values on partial register writes, even if there is no read. 2954 if (!MRI->shouldTrackSubRegLiveness(Reg) || LaneMask.any() || 2955 !hasSubRegDef) { 2956 report("Instruction ending live segment doesn't read the register", 2957 MI); 2958 report_context(LR, Reg, LaneMask); 2959 report_context(S); 2960 } 2961 } 2962 } 2963 } 2964 2965 // Now check all the basic blocks in this live segment. 2966 MachineFunction::const_iterator MFI = MBB->getIterator(); 2967 // Is this live segment the beginning of a non-PHIDef VN? 2968 if (S.start == VNI->def && !VNI->isPHIDef()) { 2969 // Not live-in to any blocks. 2970 if (MBB == EndMBB) 2971 return; 2972 // Skip this block. 2973 ++MFI; 2974 } 2975 2976 SmallVector<SlotIndex, 4> Undefs; 2977 if (LaneMask.any()) { 2978 LiveInterval &OwnerLI = LiveInts->getInterval(Reg); 2979 OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, *MRI, *Indexes); 2980 } 2981 2982 while (true) { 2983 assert(LiveInts->isLiveInToMBB(LR, &*MFI)); 2984 // We don't know how to track physregs into a landing pad. 2985 if (!Register::isVirtualRegister(Reg) && MFI->isEHPad()) { 2986 if (&*MFI == EndMBB) 2987 break; 2988 ++MFI; 2989 continue; 2990 } 2991 2992 // Is VNI a PHI-def in the current block? 2993 bool IsPHI = VNI->isPHIDef() && 2994 VNI->def == LiveInts->getMBBStartIdx(&*MFI); 2995 2996 // Check that VNI is live-out of all predecessors. 2997 for (const MachineBasicBlock *Pred : MFI->predecessors()) { 2998 SlotIndex PEnd = LiveInts->getMBBEndIdx(Pred); 2999 // Predecessor of landing pad live-out on last call. 3000 if (MFI->isEHPad()) { 3001 for (auto I = Pred->rbegin(), E = Pred->rend(); I != E; ++I) { 3002 if (I->isCall()) { 3003 PEnd = Indexes->getInstructionIndex(*I).getBoundaryIndex(); 3004 break; 3005 } 3006 } 3007 } 3008 const VNInfo *PVNI = LR.getVNInfoBefore(PEnd); 3009 3010 // All predecessors must have a live-out value. However for a phi 3011 // instruction with subregister intervals 3012 // only one of the subregisters (not necessarily the current one) needs to 3013 // be defined. 3014 if (!PVNI && (LaneMask.none() || !IsPHI)) { 3015 if (LiveRangeCalc::isJointlyDominated(Pred, Undefs, *Indexes)) 3016 continue; 3017 report("Register not marked live out of predecessor", Pred); 3018 report_context(LR, Reg, LaneMask); 3019 report_context(*VNI); 3020 errs() << " live into " << printMBBReference(*MFI) << '@' 3021 << LiveInts->getMBBStartIdx(&*MFI) << ", not live before " 3022 << PEnd << '\n'; 3023 continue; 3024 } 3025 3026 // Only PHI-defs can take different predecessor values. 3027 if (!IsPHI && PVNI != VNI) { 3028 report("Different value live out of predecessor", Pred); 3029 report_context(LR, Reg, LaneMask); 3030 errs() << "Valno #" << PVNI->id << " live out of " 3031 << printMBBReference(*Pred) << '@' << PEnd << "\nValno #" 3032 << VNI->id << " live into " << printMBBReference(*MFI) << '@' 3033 << LiveInts->getMBBStartIdx(&*MFI) << '\n'; 3034 } 3035 } 3036 if (&*MFI == EndMBB) 3037 break; 3038 ++MFI; 3039 } 3040} 3041 3042void MachineVerifier::verifyLiveRange(const LiveRange &LR, Register Reg, 3043 LaneBitmask LaneMask) { 3044 for (const VNInfo *VNI : LR.valnos) 3045 verifyLiveRangeValue(LR, VNI, Reg, LaneMask); 3046 3047 for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I) 3048 verifyLiveRangeSegment(LR, I, Reg, LaneMask); 3049} 3050 3051void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) { 3052 Register Reg = LI.reg(); 3053 assert(Register::isVirtualRegister(Reg)); 3054 verifyLiveRange(LI, Reg); 3055 3056 LaneBitmask Mask; 3057 LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg); 3058 for (const LiveInterval::SubRange &SR : LI.subranges()) { 3059 if ((Mask & SR.LaneMask).any()) { 3060 report("Lane masks of sub ranges overlap in live interval", MF); 3061 report_context(LI); 3062 } 3063 if ((SR.LaneMask & ~MaxMask).any()) { 3064 report("Subrange lanemask is invalid", MF); 3065 report_context(LI); 3066 } 3067 if (SR.empty()) { 3068 report("Subrange must not be empty", MF); 3069 report_context(SR, LI.reg(), SR.LaneMask); 3070 } 3071 Mask |= SR.LaneMask; 3072 verifyLiveRange(SR, LI.reg(), SR.LaneMask); 3073 if (!LI.covers(SR)) { 3074 report("A Subrange is not covered by the main range", MF); 3075 report_context(LI); 3076 } 3077 } 3078 3079 // Check the LI only has one connected component. 3080 ConnectedVNInfoEqClasses ConEQ(*LiveInts); 3081 unsigned NumComp = ConEQ.Classify(LI); 3082 if (NumComp > 1) { 3083 report("Multiple connected components in live interval", MF); 3084 report_context(LI); 3085 for (unsigned comp = 0; comp != NumComp; ++comp) { 3086 errs() << comp << ": valnos"; 3087 for (const VNInfo *I : LI.valnos) 3088 if (comp == ConEQ.getEqClass(I)) 3089 errs() << ' ' << I->id; 3090 errs() << '\n'; 3091 } 3092 } 3093} 3094 3095namespace { 3096 3097 // FrameSetup and FrameDestroy can have zero adjustment, so using a single 3098 // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the 3099 // value is zero. 3100 // We use a bool plus an integer to capture the stack state. 3101 struct StackStateOfBB { 3102 StackStateOfBB() = default; 3103 StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup) : 3104 EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup), 3105 ExitIsSetup(ExitSetup) {} 3106 3107 // Can be negative, which means we are setting up a frame. 3108 int EntryValue = 0; 3109 int ExitValue = 0; 3110 bool EntryIsSetup = false; 3111 bool ExitIsSetup = false; 3112 }; 3113 3114} // end anonymous namespace 3115 3116/// Make sure on every path through the CFG, a FrameSetup <n> is always followed 3117/// by a FrameDestroy <n>, stack adjustments are identical on all 3118/// CFG edges to a merge point, and frame is destroyed at end of a return block. 3119void MachineVerifier::verifyStackFrame() { 3120 unsigned FrameSetupOpcode = TII->getCallFrameSetupOpcode(); 3121 unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode(); 3122 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u) 3123 return; 3124 3125 SmallVector<StackStateOfBB, 8> SPState; 3126 SPState.resize(MF->getNumBlockIDs()); 3127 df_iterator_default_set<const MachineBasicBlock*> Reachable; 3128 3129 // Visit the MBBs in DFS order. 3130 for (df_ext_iterator<const MachineFunction *, 3131 df_iterator_default_set<const MachineBasicBlock *>> 3132 DFI = df_ext_begin(MF, Reachable), DFE = df_ext_end(MF, Reachable); 3133 DFI != DFE; ++DFI) { 3134 const MachineBasicBlock *MBB = *DFI; 3135 3136 StackStateOfBB BBState; 3137 // Check the exit state of the DFS stack predecessor. 3138 if (DFI.getPathLength() >= 2) { 3139 const MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2); 3140 assert(Reachable.count(StackPred) && 3141 "DFS stack predecessor is already visited.\n"); 3142 BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue; 3143 BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup; 3144 BBState.ExitValue = BBState.EntryValue; 3145 BBState.ExitIsSetup = BBState.EntryIsSetup; 3146 } 3147 3148 // Update stack state by checking contents of MBB. 3149 for (const auto &I : *MBB) { 3150 if (I.getOpcode() == FrameSetupOpcode) { 3151 if (BBState.ExitIsSetup) 3152 report("FrameSetup is after another FrameSetup", &I); 3153 BBState.ExitValue -= TII->getFrameTotalSize(I); 3154 BBState.ExitIsSetup = true; 3155 } 3156 3157 if (I.getOpcode() == FrameDestroyOpcode) { 3158 int Size = TII->getFrameTotalSize(I); 3159 if (!BBState.ExitIsSetup) 3160 report("FrameDestroy is not after a FrameSetup", &I); 3161 int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue : 3162 BBState.ExitValue; 3163 if (BBState.ExitIsSetup && AbsSPAdj != Size) { 3164 report("FrameDestroy <n> is after FrameSetup <m>", &I); 3165 errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" 3166 << AbsSPAdj << ">.\n"; 3167 } 3168 BBState.ExitValue += Size; 3169 BBState.ExitIsSetup = false; 3170 } 3171 } 3172 SPState[MBB->getNumber()] = BBState; 3173 3174 // Make sure the exit state of any predecessor is consistent with the entry 3175 // state. 3176 for (const MachineBasicBlock *Pred : MBB->predecessors()) { 3177 if (Reachable.count(Pred) && 3178 (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue || 3179 SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) { 3180 report("The exit stack state of a predecessor is inconsistent.", MBB); 3181 errs() << "Predecessor " << printMBBReference(*Pred) 3182 << " has exit state (" << SPState[Pred->getNumber()].ExitValue 3183 << ", " << SPState[Pred->getNumber()].ExitIsSetup << "), while " 3184 << printMBBReference(*MBB) << " has entry state (" 3185 << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n"; 3186 } 3187 } 3188 3189 // Make sure the entry state of any successor is consistent with the exit 3190 // state. 3191 for (const MachineBasicBlock *Succ : MBB->successors()) { 3192 if (Reachable.count(Succ) && 3193 (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue || 3194 SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) { 3195 report("The entry stack state of a successor is inconsistent.", MBB); 3196 errs() << "Successor " << printMBBReference(*Succ) 3197 << " has entry state (" << SPState[Succ->getNumber()].EntryValue 3198 << ", " << SPState[Succ->getNumber()].EntryIsSetup << "), while " 3199 << printMBBReference(*MBB) << " has exit state (" 3200 << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n"; 3201 } 3202 } 3203 3204 // Make sure a basic block with return ends with zero stack adjustment. 3205 if (!MBB->empty() && MBB->back().isReturn()) { 3206 if (BBState.ExitIsSetup) 3207 report("A return block ends with a FrameSetup.", MBB); 3208 if (BBState.ExitValue) 3209 report("A return block ends with a nonzero stack adjustment.", MBB); 3210 } 3211 } 3212} 3213