lib/CodeGen/ComplexDeinterleavingPass.cpp

48 // Each node maintains a reference  to its Real and Imaginary instructions,
56 // Note: If the operation of a Node is Shuffle, only the Real, Imaginary, and
142       : Operation(Op), Real(R), Imag(I) {}
151   Value *Real;
184     OS << "  Real: ";
185     PrintValue(Real);
290            "Reduction related nodes must have Real and Imaginary parts");
297     if (Node->Real && Node->Imag)
298       CachedResult[{Node->Real, Node->Imag}] = Node;
313   NodePtr identifyPartialMul(Instruction *Real, Instruction *Imag);
329   NodePtr identifyAdd(Instruction *Real, Instruction *Imag);
330   NodePtr identifySymmetricOperation(Instruction *Real, Instruction *Imag);
354   /// Go through pairs of multiplication (one Real and one Imag) and find all
378   NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
384   NodePtr identifySplat(Value *Real, Value *Imag);
386   NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
390   NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
550     Instruction *Real, Instruction *Imag,
552   LLVM_DEBUG(dbgs() << "identifyNodeWithImplicitAdd " << *Real << " / " << *Imag
555   if (!Real->hasOneUse() || !Imag->hasOneUse()) {
560   if ((Real->getOpcode() != Instruction::FMul &&
561        Real->getOpcode() != Instruction::Mul) ||
565         dbgs() << "  - Real or imaginary instruction is not fmul or mul\n");
569   Value *R0 = Real->getOperand(0);
570   Value *R1 = Real->getOperand(1);
644       ComplexDeinterleavingOperation::CMulPartial, Real, Imag);
652 ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real,
654   LLVM_DEBUG(dbgs() << "identifyPartialMul " << *Real << " / " << *Imag
664   if (IsAdd(Real->getOpcode()) && IsAdd(Imag->getOpcode()))
666   else if (IsSub(Real->getOpcode()) && IsAdd(Imag->getOpcode()))
668   else if (IsSub(Real->getOpcode()) && IsSub(Imag->getOpcode()))
670   else if (IsAdd(Real->getOpcode()) && IsSub(Imag->getOpcode()))
677   if (isa<FPMathOperator>(Real) &&
678       (!Real->getFastMathFlags().allowContract() ||
684   Value *CR = Real->getOperand(0);
685   Instruction *RealMulI = dyn_cast<Instruction>(Real->getOperand(1));
761       ComplexDeinterleavingOperation::CMulPartial, Real, Imag);
770 ComplexDeinterleavingGraph::identifyAdd(Instruction *Real, Instruction *Imag) {
771   LLVM_DEBUG(dbgs() << "identifyAdd " << *Real << " / " << *Imag << "\n");
775   if ((Real->getOpcode() == Instruction::FSub &&
777       (Real->getOpcode() == Instruction::Sub &&
780   else if ((Real->getOpcode() == Instruction::FAdd &&
782            (Real->getOpcode() == Instruction::Add &&
790   auto *AR = dyn_cast<Instruction>(Real->getOperand(0));
791   auto *BI = dyn_cast<Instruction>(Real->getOperand(1));
812       prepareCompositeNode(ComplexDeinterleavingOperation::CAdd, Real, Imag);
852 ComplexDeinterleavingGraph::identifySymmetricOperation(Instruction *Real,
854   if (Real->getOpcode() != Imag->getOpcode())
857   if (!isInstructionPotentiallySymmetric(Real) ||
861   auto *R0 = Real->getOperand(0);
869   if (Real->isBinaryOp()) {
870     auto *R1 = Real->getOperand(1);
877   if (isa<FPMathOperator>(Real) &&
878       Real->getFastMathFlags() != Imag->getFastMathFlags())
882                                    Real, Imag);
883   Node->Opcode = Real->getOpcode();
884   if (isa<FPMathOperator>(Real))
885     Node->Flags = Real->getFastMathFlags();
888   if (Real->isBinaryOp())
898          "Real and imaginary parts should not have different types");
909   auto *Real = dyn_cast<Instruction>(R);
911   if (!Real || !Imag)
914   if (NodePtr CN = identifyDeinterleave(Real, Imag))
917   if (NodePtr CN = identifyPHINode(Real, Imag))
920   if (NodePtr CN = identifySelectNode(Real, Imag))
923   auto *VTy = cast<VectorType>(Real->getType());
931   if (HasCMulSupport && isInstructionPairMul(Real, Imag)) {
932     if (NodePtr CN = identifyPartialMul(Real, Imag))
936   if (HasCAddSupport && isInstructionPairAdd(Real, Imag)) {
937     if (NodePtr CN = identifyAdd(Real, Imag))
942     if (NodePtr CN = identifyReassocNodes(Real, Imag))
946   if (NodePtr CN = identifySymmetricOperation(Real, Imag))
955 ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
963   if (!IsOperationSupported(Real->getOpcode()) ||
968   if (isa<FPMathOperator>(Real)) {
969     if (Real->getFastMathFlags() != Imag->getFastMathFlags()) {
970       LLVM_DEBUG(dbgs() << "The flags in Real and Imaginary instructions are "
975     Flags = Real->getFastMathFlags();
1071   if (!Collect(Real, RealMuls, RealAddends) ||
1095   // Set the Real and Imag fields of the final node and submit it
1096   FinalNode->Real = Real;
1106   auto FindCommonInstruction = [](const Product &Real,
1108     if (Real.Multiplicand == Imag.Multiplicand ||
1109         Real.Multiplicand == Imag.Multiplier)
1110       return Real.Multiplicand;
1112     if (Real.Multiplier == Imag.Multiplicand ||
1113         Real.Multiplier == Imag.Multiplier)
1114       return Real.Multiplier;
1220     auto IsMultiplicandReal = PMI.Common == NodeA->Real;
1225     // Rotation |   Real |   Imag |
1262       dbgs().indent(4) << "X: " << *NodeA->Real << "\n";
1264       dbgs().indent(4) << "U: " << *NodeB->Real << "\n";
1289       dbgs() << "Unprocessed products (Real):\n";
1424   // reduction. Because RootToNode maps both Real and Imaginary parts to
1425   // CompositeNode we should choose only one either Real or Imag instruction to
1432     // Find out which part, Real or Imag, comes later, and only if we come to
1434     auto *R = cast<Instruction>(RootNode->Real);
1528       auto *Real = OperationInstruction[i];
1530       if (Real->getType() != Imag->getType())
1533       RealPHI = ReductionInfo[Real].first;
1536       auto Node = identifyNode(Real, Imag);
1538         std::swap(Real, Imag);
1540         Node = identifyNode(Real, Imag);
1548                           << *Real << " / " << *Imag << "\n");
1552             ComplexDeinterleavingOperation::ReductionOperation, Real, Imag);
1554         RootToNode[Real] = RootNode;
1646     auto *Real = dyn_cast<Instruction>(Intrinsic->getOperand(0));
1648     if (!Real || !Imag)
1651     return identifyNode(Real, Imag);
1663   Instruction *Real;
1665   if (!match(RootI, m_Shuffle(m_Instruction(Real), m_Instruction(Imag))))
1668   return identifyNode(Real, Imag);
1672 ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real,
1676   if (match(Real, m_ExtractValue<0>(m_Instruction(I))) &&
1681         llvm::ComplexDeinterleavingOperation::Deinterleave, Real, Imag);
1683     FinalInstructions.insert(Real);
1688   auto *RealShuffle = dyn_cast<ShuffleVectorInst>(Real);
1815   auto *Real = dyn_cast<Instruction>(R);
1817   if ((!Real && Imag) || (Real && !Imag))
1820   if (Real && Imag) {
1822     if (Real->getParent() != Imag->getParent())
1825     FinalInstructions.insert(Real);
1834 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real,
1836   if (Real != RealPHI || Imag != ImagPHI)
1841       ComplexDeinterleavingOperation::ReductionPHI, Real, Imag);
1846 ComplexDeinterleavingGraph::identifySelectNode(Instruction *Real,
1848   auto *SelectReal = dyn_cast<SelectInst>(Real);
1855   if (!match(Real, m_Select(m_Instruction(MaskA), m_Instruction(AR),
1876       ComplexDeinterleavingOperation::ReductionSelect, Real, Imag);
1956         cast<VectorType>(Node->Real->getType()));
1957     auto *R = dyn_cast<Instruction>(Node->Real);
1965                               {Node->Real, Node->Imag});
1969                                   NewTy, {Node->Real, Node->Imag});
1976     auto *VTy = cast<VectorType>(Node->Real->getType());
1979     OldToNewPHI[dyn_cast<PHINode>(Node->Real)] = NewPHI;
1988     auto *MaskReal = cast<Instruction>(Node->Real)->getOperand(0);
2010   auto *Real = cast<Instruction>(Node->Real);
2012   auto *OldPHIReal = ReductionInfo[Real].first;
2016   auto *VTy = cast<VectorType>(Real->getType());
2032   auto *FinalReductionReal = ReductionInfo[Real].second;
2042   FinalReductionReal->replaceUsesOfWith(Real, NewReal);
2063       auto *RootReal = cast<Instruction>(RootNode->Real);