Target/AMDGPU/SIInsertWaitcnts.cpp

1 //===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
169 void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) {
172     Wait.VmCnt = std::min(Wait.VmCnt, Count);
175     Wait.ExpCnt = std::min(Wait.ExpCnt, Count);
178     Wait.LgkmCnt = std::min(Wait.LgkmCnt, Count);
181     Wait.VsCnt = std::min(Wait.VsCnt, Count);
274   bool simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
277                      AMDGPU::Waitcnt &Wait) const;
278   void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
364   // Wait cnt scores for every sgpr, only lgkmcnt is relevant.
721 bool WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
722   return simplifyWaitcnt(VM_CNT, Wait.VmCnt) |
723          simplifyWaitcnt(EXP_CNT, Wait.ExpCnt) |
724          simplifyWaitcnt(LGKM_CNT, Wait.LgkmCnt) |
725          simplifyWaitcnt(VS_CNT, Wait.VsCnt);
740                                     AMDGPU::Waitcnt &Wait) const {
752       addWait(Wait, T, 0);
757       addWait(Wait, T, 0);
762       addWait(Wait, T, NeededWait);
767 void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
768   applyWaitcnt(VM_CNT, Wait.VmCnt);
769   applyWaitcnt(EXP_CNT, Wait.ExpCnt);
770   applyWaitcnt(LGKM_CNT, Wait.LgkmCnt);
771   applyWaitcnt(VS_CNT, Wait.VsCnt);
852   AMDGPU::Waitcnt Wait;
861     Wait.VmCnt = 0;
870     Wait = Wait.combined(AMDGPU::Waitcnt::allZero(IV));
877     Wait.VmCnt = 0;
945         Wait.ExpCnt = 0;
953       Wait = AMDGPU::Waitcnt();
963             LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
974               LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
992             VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1004                 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1007               LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
1023             addWait(Wait, LGKM_CNT, 0);
1033               VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1035               EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
1046                 VM_CNT, ScoreBrackets.getRegScore(RegNo, VM_CNT), Wait);
1048                 EXP_CNT, ScoreBrackets.getRegScore(RegNo, EXP_CNT), Wait);
1051               LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait);
1063     Wait = Wait.combined(AMDGPU::Waitcnt::allZero(IV));
1073       Wait.LgkmCnt = 0;
1078   if (!ScoreBrackets.simplifyWaitcnt(Wait) && !IsForceEmitWaitcnt) {
1105     Wait = AMDGPU::Waitcnt::allZero(IV);
1108     Wait.VmCnt = 0;
1110     Wait.ExpCnt = 0;
1112     Wait.LgkmCnt = 0;
1114     Wait.VsCnt = 0;
1116   ScoreBrackets.applyWaitcnt(Wait);
1132           Wait = Wait.combined(IWait);
1133         unsigned NewEnc = AMDGPU::encodeWaitcnt(IV, Wait);
1138         Wait.VmCnt = ~0u;
1139         Wait.LgkmCnt = ~0u;
1140         Wait.ExpCnt = ~0u;
1148           Wait.VsCnt = std::min(Wait.VsCnt, ICnt);
1149         if (Wait.VsCnt != ICnt) {
1150           II->getOperand(1).setImm(Wait.VsCnt);
1153         Wait.VsCnt = ~0u;
1160       if (!Wait.hasWait())
1165   if (Wait.VmCnt != ~0u || Wait.LgkmCnt != ~0u || Wait.ExpCnt != ~0u) {
1166     unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
1178   if (Wait.VsCnt != ~0u) {
1185             .addImm(Wait.VsCnt);
1608     // Wait for any outstanding memory operations that the input registers may