AMDGPUAsmParser.cpp revision 285181 
1//===-- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
11#include "MCTargetDesc/AMDGPUTargetStreamer.h"
12#include "Utils/AMDGPUBaseInfo.h"
13#include "AMDKernelCodeT.h"
14#include "SIDefines.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/SmallString.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/MC/MCContext.h"
22#include "llvm/MC/MCExpr.h"
23#include "llvm/MC/MCInst.h"
24#include "llvm/MC/MCInstrInfo.h"
25#include "llvm/MC/MCParser/MCAsmLexer.h"
26#include "llvm/MC/MCParser/MCAsmParser.h"
27#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
28#include "llvm/MC/MCRegisterInfo.h"
29#include "llvm/MC/MCStreamer.h"
30#include "llvm/MC/MCSubtargetInfo.h"
31#include "llvm/MC/MCTargetAsmParser.h"
32#include "llvm/Support/SourceMgr.h"
33#include "llvm/Support/TargetRegistry.h"
34#include "llvm/Support/raw_ostream.h"
35#include "llvm/Support/Debug.h"
36
37using namespace llvm;
38
39namespace {
40
41struct OptionalOperand;
42
43class AMDGPUOperand : public MCParsedAsmOperand {
44  enum KindTy {
45    Token,
46    Immediate,
47    Register,
48    Expression
49  } Kind;
50
51  SMLoc StartLoc, EndLoc;
52
53public:
54  AMDGPUOperand(enum KindTy K) : MCParsedAsmOperand(), Kind(K) {}
55
56  MCContext *Ctx;
57
58  enum ImmTy {
59    ImmTyNone,
60    ImmTyDSOffset0,
61    ImmTyDSOffset1,
62    ImmTyGDS,
63    ImmTyOffset,
64    ImmTyGLC,
65    ImmTySLC,
66    ImmTyTFE,
67    ImmTyClamp,
68    ImmTyOMod
69  };
70
71  struct TokOp {
72    const char *Data;
73    unsigned Length;
74  };
75
76  struct ImmOp {
77    bool IsFPImm;
78    ImmTy Type;
79    int64_t Val;
80  };
81
82  struct RegOp {
83    unsigned RegNo;
84    int Modifiers;
85    const MCRegisterInfo *TRI;
86    bool IsForcedVOP3;
87  };
88
89  union {
90    TokOp Tok;
91    ImmOp Imm;
92    RegOp Reg;
93    const MCExpr *Expr;
94  };
95
96  void addImmOperands(MCInst &Inst, unsigned N) const {
97    Inst.addOperand(MCOperand::createImm(getImm()));
98  }
99
100  StringRef getToken() const {
101    return StringRef(Tok.Data, Tok.Length);
102  }
103
104  void addRegOperands(MCInst &Inst, unsigned N) const {
105    Inst.addOperand(MCOperand::createReg(getReg()));
106  }
107
108  void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
109    if (isReg())
110      addRegOperands(Inst, N);
111    else
112      addImmOperands(Inst, N);
113  }
114
115  void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
116    Inst.addOperand(MCOperand::createImm(
117        Reg.Modifiers == -1 ? 0 : Reg.Modifiers));
118    addRegOperands(Inst, N);
119  }
120
121  void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
122    if (isImm())
123      addImmOperands(Inst, N);
124    else {
125      assert(isExpr());
126      Inst.addOperand(MCOperand::createExpr(Expr));
127    }
128  }
129
130  bool defaultTokenHasSuffix() const {
131    StringRef Token(Tok.Data, Tok.Length);
132
133    return Token.endswith("_e32") || Token.endswith("_e64");
134  }
135
136  bool isToken() const override {
137    return Kind == Token;
138  }
139
140  bool isImm() const override {
141    return Kind == Immediate;
142  }
143
144  bool isInlineImm() const {
145    float F = BitsToFloat(Imm.Val);
146    // TODO: Add 0.5pi for VI
147    return isImm() && ((Imm.Val <= 64 && Imm.Val >= -16) ||
148           (F == 0.0 || F == 0.5 || F == -0.5 || F == 1.0 || F == -1.0 ||
149           F == 2.0 || F == -2.0 || F == 4.0 || F == -4.0));
150  }
151
152  bool isDSOffset0() const {
153    assert(isImm());
154    return Imm.Type == ImmTyDSOffset0;
155  }
156
157  bool isDSOffset1() const {
158    assert(isImm());
159    return Imm.Type == ImmTyDSOffset1;
160  }
161
162  int64_t getImm() const {
163    return Imm.Val;
164  }
165
166  enum ImmTy getImmTy() const {
167    assert(isImm());
168    return Imm.Type;
169  }
170
171  bool isRegKind() const {
172    return Kind == Register;
173  }
174
175  bool isReg() const override {
176    return Kind == Register && Reg.Modifiers == -1;
177  }
178
179  bool isRegWithInputMods() const {
180    return Kind == Register && (Reg.IsForcedVOP3 || Reg.Modifiers != -1);
181  }
182
183  void setModifiers(unsigned Mods) {
184    assert(isReg());
185    Reg.Modifiers = Mods;
186  }
187
188  bool hasModifiers() const {
189    assert(isRegKind());
190    return Reg.Modifiers != -1;
191  }
192
193  unsigned getReg() const override {
194    return Reg.RegNo;
195  }
196
197  bool isRegOrImm() const {
198    return isReg() || isImm();
199  }
200
201  bool isRegClass(unsigned RCID) const {
202    return Reg.TRI->getRegClass(RCID).contains(getReg());
203  }
204
205  bool isSCSrc32() const {
206    return isInlineImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
207  }
208
209  bool isSSrc32() const {
210    return isImm() || (isReg() && isRegClass(AMDGPU::SReg_32RegClassID));
211  }
212
213  bool isSSrc64() const {
214    return isImm() || isInlineImm() ||
215           (isReg() && isRegClass(AMDGPU::SReg_64RegClassID));
216  }
217
218  bool isVCSrc32() const {
219    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
220  }
221
222  bool isVCSrc64() const {
223    return isInlineImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
224  }
225
226  bool isVSrc32() const {
227    return isImm() || (isReg() && isRegClass(AMDGPU::VS_32RegClassID));
228  }
229
230  bool isVSrc64() const {
231    return isImm() || (isReg() && isRegClass(AMDGPU::VS_64RegClassID));
232  }
233
234  bool isMem() const override {
235    return false;
236  }
237
238  bool isExpr() const {
239    return Kind == Expression;
240  }
241
242  bool isSoppBrTarget() const {
243    return isExpr() || isImm();
244  }
245
246  SMLoc getStartLoc() const override {
247    return StartLoc;
248  }
249
250  SMLoc getEndLoc() const override {
251    return EndLoc;
252  }
253
254  void print(raw_ostream &OS) const override { }
255
256  static std::unique_ptr<AMDGPUOperand> CreateImm(int64_t Val, SMLoc Loc,
257                                                  enum ImmTy Type = ImmTyNone,
258                                                  bool IsFPImm = false) {
259    auto Op = llvm::make_unique<AMDGPUOperand>(Immediate);
260    Op->Imm.Val = Val;
261    Op->Imm.IsFPImm = IsFPImm;
262    Op->Imm.Type = Type;
263    Op->StartLoc = Loc;
264    Op->EndLoc = Loc;
265    return Op;
266  }
267
268  static std::unique_ptr<AMDGPUOperand> CreateToken(StringRef Str, SMLoc Loc,
269                                           bool HasExplicitEncodingSize = true) {
270    auto Res = llvm::make_unique<AMDGPUOperand>(Token);
271    Res->Tok.Data = Str.data();
272    Res->Tok.Length = Str.size();
273    Res->StartLoc = Loc;
274    Res->EndLoc = Loc;
275    return Res;
276  }
277
278  static std::unique_ptr<AMDGPUOperand> CreateReg(unsigned RegNo, SMLoc S,
279                                                  SMLoc E,
280                                                  const MCRegisterInfo *TRI,
281                                                  bool ForceVOP3) {
282    auto Op = llvm::make_unique<AMDGPUOperand>(Register);
283    Op->Reg.RegNo = RegNo;
284    Op->Reg.TRI = TRI;
285    Op->Reg.Modifiers = -1;
286    Op->Reg.IsForcedVOP3 = ForceVOP3;
287    Op->StartLoc = S;
288    Op->EndLoc = E;
289    return Op;
290  }
291
292  static std::unique_ptr<AMDGPUOperand> CreateExpr(const class MCExpr *Expr, SMLoc S) {
293    auto Op = llvm::make_unique<AMDGPUOperand>(Expression);
294    Op->Expr = Expr;
295    Op->StartLoc = S;
296    Op->EndLoc = S;
297    return Op;
298  }
299
300  bool isDSOffset() const;
301  bool isDSOffset01() const;
302  bool isSWaitCnt() const;
303  bool isMubufOffset() const;
304};
305
306class AMDGPUAsmParser : public MCTargetAsmParser {
307  MCSubtargetInfo &STI;
308  const MCInstrInfo &MII;
309  MCAsmParser &Parser;
310
311  unsigned ForcedEncodingSize;
312  /// @name Auto-generated Match Functions
313  /// {
314
315#define GET_ASSEMBLER_HEADER
316#include "AMDGPUGenAsmMatcher.inc"
317
318  /// }
319
320private:
321  bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
322  bool ParseDirectiveHSACodeObjectVersion();
323  bool ParseDirectiveHSACodeObjectISA();
324  bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
325  bool ParseDirectiveAMDKernelCodeT();
326
327public:
328  AMDGPUAsmParser(MCSubtargetInfo &STI, MCAsmParser &_Parser,
329               const MCInstrInfo &MII,
330               const MCTargetOptions &Options)
331      : MCTargetAsmParser(), STI(STI), MII(MII), Parser(_Parser),
332        ForcedEncodingSize(0){
333
334    if (STI.getFeatureBits().none()) {
335      // Set default features.
336      STI.ToggleFeature("SOUTHERN_ISLANDS");
337    }
338
339    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
340  }
341
342  AMDGPUTargetStreamer &getTargetStreamer() {
343    MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
344    return static_cast<AMDGPUTargetStreamer &>(TS);
345  }
346
347  unsigned getForcedEncodingSize() const {
348    return ForcedEncodingSize;
349  }
350
351  void setForcedEncodingSize(unsigned Size) {
352    ForcedEncodingSize = Size;
353  }
354
355  bool isForcedVOP3() const {
356    return ForcedEncodingSize == 64;
357  }
358
359  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
360  unsigned checkTargetMatchPredicate(MCInst &Inst) override;
361  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
362                               OperandVector &Operands, MCStreamer &Out,
363                               uint64_t &ErrorInfo,
364                               bool MatchingInlineAsm) override;
365  bool ParseDirective(AsmToken DirectiveID) override;
366  OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic);
367  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
368                        SMLoc NameLoc, OperandVector &Operands) override;
369
370  OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int,
371                                          int64_t Default = 0);
372  OperandMatchResultTy parseIntWithPrefix(const char *Prefix,
373                                          OperandVector &Operands,
374                                          enum AMDGPUOperand::ImmTy ImmTy =
375                                                      AMDGPUOperand::ImmTyNone);
376  OperandMatchResultTy parseNamedBit(const char *Name, OperandVector &Operands,
377                                     enum AMDGPUOperand::ImmTy ImmTy =
378                                                      AMDGPUOperand::ImmTyNone);
379  OperandMatchResultTy parseOptionalOps(
380                                   const ArrayRef<OptionalOperand> &OptionalOps,
381                                   OperandVector &Operands);
382
383
384  void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
385  void cvtDS(MCInst &Inst, const OperandVector &Operands);
386  OperandMatchResultTy parseDSOptionalOps(OperandVector &Operands);
387  OperandMatchResultTy parseDSOff01OptionalOps(OperandVector &Operands);
388  OperandMatchResultTy parseDSOffsetOptional(OperandVector &Operands);
389
390  bool parseCnt(int64_t &IntVal);
391  OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
392  OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
393
394  OperandMatchResultTy parseFlatOptionalOps(OperandVector &Operands);
395  OperandMatchResultTy parseFlatAtomicOptionalOps(OperandVector &Operands);
396  void cvtFlat(MCInst &Inst, const OperandVector &Operands);
397
398  void cvtMubuf(MCInst &Inst, const OperandVector &Operands);
399  OperandMatchResultTy parseOffset(OperandVector &Operands);
400  OperandMatchResultTy parseMubufOptionalOps(OperandVector &Operands);
401  OperandMatchResultTy parseGLC(OperandVector &Operands);
402  OperandMatchResultTy parseSLC(OperandVector &Operands);
403  OperandMatchResultTy parseTFE(OperandVector &Operands);
404
405  OperandMatchResultTy parseDMask(OperandVector &Operands);
406  OperandMatchResultTy parseUNorm(OperandVector &Operands);
407  OperandMatchResultTy parseR128(OperandVector &Operands);
408
409  void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
410  OperandMatchResultTy parseVOP3OptionalOps(OperandVector &Operands);
411};
412
413struct OptionalOperand {
414  const char *Name;
415  AMDGPUOperand::ImmTy Type;
416  bool IsBit;
417  int64_t Default;
418  bool (*ConvertResult)(int64_t&);
419};
420
421}
422
423static unsigned getRegClass(bool IsVgpr, unsigned RegWidth) {
424  if (IsVgpr) {
425    switch (RegWidth) {
426      default: llvm_unreachable("Unknown register width");
427      case 1: return AMDGPU::VGPR_32RegClassID;
428      case 2: return AMDGPU::VReg_64RegClassID;
429      case 3: return AMDGPU::VReg_96RegClassID;
430      case 4: return AMDGPU::VReg_128RegClassID;
431      case 8: return AMDGPU::VReg_256RegClassID;
432      case 16: return AMDGPU::VReg_512RegClassID;
433    }
434  }
435
436  switch (RegWidth) {
437    default: llvm_unreachable("Unknown register width");
438    case 1: return AMDGPU::SGPR_32RegClassID;
439    case 2: return AMDGPU::SGPR_64RegClassID;
440    case 4: return AMDGPU::SReg_128RegClassID;
441    case 8: return AMDGPU::SReg_256RegClassID;
442    case 16: return AMDGPU::SReg_512RegClassID;
443  }
444}
445
446static unsigned getRegForName(const StringRef &RegName) {
447
448  return StringSwitch<unsigned>(RegName)
449    .Case("exec", AMDGPU::EXEC)
450    .Case("vcc", AMDGPU::VCC)
451    .Case("flat_scr", AMDGPU::FLAT_SCR)
452    .Case("m0", AMDGPU::M0)
453    .Case("scc", AMDGPU::SCC)
454    .Case("flat_scr_lo", AMDGPU::FLAT_SCR_LO)
455    .Case("flat_scr_hi", AMDGPU::FLAT_SCR_HI)
456    .Case("vcc_lo", AMDGPU::VCC_LO)
457    .Case("vcc_hi", AMDGPU::VCC_HI)
458    .Case("exec_lo", AMDGPU::EXEC_LO)
459    .Case("exec_hi", AMDGPU::EXEC_HI)
460    .Default(0);
461}
462
463bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
464  const AsmToken Tok = Parser.getTok();
465  StartLoc = Tok.getLoc();
466  EndLoc = Tok.getEndLoc();
467  const StringRef &RegName = Tok.getString();
468  RegNo = getRegForName(RegName);
469
470  if (RegNo) {
471    Parser.Lex();
472    return false;
473  }
474
475  // Match vgprs and sgprs
476  if (RegName[0] != 's' && RegName[0] != 'v')
477    return true;
478
479  bool IsVgpr = RegName[0] == 'v';
480  unsigned RegWidth;
481  unsigned RegIndexInClass;
482  if (RegName.size() > 1) {
483    // We have a 32-bit register
484    RegWidth = 1;
485    if (RegName.substr(1).getAsInteger(10, RegIndexInClass))
486      return true;
487    Parser.Lex();
488  } else {
489    // We have a register greater than 32-bits.
490
491    int64_t RegLo, RegHi;
492    Parser.Lex();
493    if (getLexer().isNot(AsmToken::LBrac))
494      return true;
495
496    Parser.Lex();
497    if (getParser().parseAbsoluteExpression(RegLo))
498      return true;
499
500    if (getLexer().isNot(AsmToken::Colon))
501      return true;
502
503    Parser.Lex();
504    if (getParser().parseAbsoluteExpression(RegHi))
505      return true;
506
507    if (getLexer().isNot(AsmToken::RBrac))
508      return true;
509
510    Parser.Lex();
511    RegWidth = (RegHi - RegLo) + 1;
512    if (IsVgpr) {
513      // VGPR registers aren't aligned.
514      RegIndexInClass = RegLo;
515    } else {
516      // SGPR registers are aligned.  Max alignment is 4 dwords.
517      RegIndexInClass = RegLo / std::min(RegWidth, 4u);
518    }
519  }
520
521  const MCRegisterInfo *TRC = getContext().getRegisterInfo();
522  unsigned RC = getRegClass(IsVgpr, RegWidth);
523  if (RegIndexInClass > TRC->getRegClass(RC).getNumRegs())
524    return true;
525  RegNo = TRC->getRegClass(RC).getRegister(RegIndexInClass);
526  return false;
527}
528
529unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
530
531  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
532
533  if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
534      (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)))
535    return Match_InvalidOperand;
536
537  return Match_Success;
538}
539
540
541bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
542                                              OperandVector &Operands,
543                                              MCStreamer &Out,
544                                              uint64_t &ErrorInfo,
545                                              bool MatchingInlineAsm) {
546  MCInst Inst;
547
548  switch (MatchInstructionImpl(Operands, Inst, ErrorInfo, MatchingInlineAsm)) {
549    default: break;
550    case Match_Success:
551      Inst.setLoc(IDLoc);
552      Out.EmitInstruction(Inst, STI);
553      return false;
554    case Match_MissingFeature:
555      return Error(IDLoc, "instruction not supported on this GPU");
556
557    case Match_MnemonicFail:
558      return Error(IDLoc, "unrecognized instruction mnemonic");
559
560    case Match_InvalidOperand: {
561      SMLoc ErrorLoc = IDLoc;
562      if (ErrorInfo != ~0ULL) {
563        if (ErrorInfo >= Operands.size()) {
564          if (isForcedVOP3()) {
565            // If 64-bit encoding has been forced we can end up with no
566            // clamp or omod operands if none of the registers have modifiers,
567            // so we need to add these to the operand list.
568            AMDGPUOperand &LastOp =
569                ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
570            if (LastOp.isRegKind() ||
571               (LastOp.isImm() &&
572                LastOp.getImmTy() != AMDGPUOperand::ImmTyNone)) {
573              SMLoc S = Parser.getTok().getLoc();
574              Operands.push_back(AMDGPUOperand::CreateImm(0, S,
575                                 AMDGPUOperand::ImmTyClamp));
576              Operands.push_back(AMDGPUOperand::CreateImm(0, S,
577                                 AMDGPUOperand::ImmTyOMod));
578              bool Res = MatchAndEmitInstruction(IDLoc, Opcode, Operands,
579                                                 Out, ErrorInfo,
580                                                 MatchingInlineAsm);
581              if (!Res)
582                return Res;
583            }
584
585          }
586          return Error(IDLoc, "too few operands for instruction");
587        }
588
589        ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
590        if (ErrorLoc == SMLoc())
591          ErrorLoc = IDLoc;
592      }
593      return Error(ErrorLoc, "invalid operand for instruction");
594    }
595  }
596  llvm_unreachable("Implement any new match types added!");
597}
598
599bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
600                                               uint32_t &Minor) {
601  if (getLexer().isNot(AsmToken::Integer))
602    return TokError("invalid major version");
603
604  Major = getLexer().getTok().getIntVal();
605  Lex();
606
607  if (getLexer().isNot(AsmToken::Comma))
608    return TokError("minor version number required, comma expected");
609  Lex();
610
611  if (getLexer().isNot(AsmToken::Integer))
612    return TokError("invalid minor version");
613
614  Minor = getLexer().getTok().getIntVal();
615  Lex();
616
617  return false;
618}
619
620bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
621
622  uint32_t Major;
623  uint32_t Minor;
624
625  if (ParseDirectiveMajorMinor(Major, Minor))
626    return true;
627
628  getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
629  return false;
630}
631
632bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
633
634  uint32_t Major;
635  uint32_t Minor;
636  uint32_t Stepping;
637  StringRef VendorName;
638  StringRef ArchName;
639
640  // If this directive has no arguments, then use the ISA version for the
641  // targeted GPU.
642  if (getLexer().is(AsmToken::EndOfStatement)) {
643    AMDGPU::IsaVersion Isa = AMDGPU::getIsaVersion(STI.getFeatureBits());
644    getTargetStreamer().EmitDirectiveHSACodeObjectISA(Isa.Major, Isa.Minor,
645                                                      Isa.Stepping,
646                                                      "AMD", "AMDGPU");
647    return false;
648  }
649
650
651  if (ParseDirectiveMajorMinor(Major, Minor))
652    return true;
653
654  if (getLexer().isNot(AsmToken::Comma))
655    return TokError("stepping version number required, comma expected");
656  Lex();
657
658  if (getLexer().isNot(AsmToken::Integer))
659    return TokError("invalid stepping version");
660
661  Stepping = getLexer().getTok().getIntVal();
662  Lex();
663
664  if (getLexer().isNot(AsmToken::Comma))
665    return TokError("vendor name required, comma expected");
666  Lex();
667
668  if (getLexer().isNot(AsmToken::String))
669    return TokError("invalid vendor name");
670
671  VendorName = getLexer().getTok().getStringContents();
672  Lex();
673
674  if (getLexer().isNot(AsmToken::Comma))
675    return TokError("arch name required, comma expected");
676  Lex();
677
678  if (getLexer().isNot(AsmToken::String))
679    return TokError("invalid arch name");
680
681  ArchName = getLexer().getTok().getStringContents();
682  Lex();
683
684  getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
685                                                    VendorName, ArchName);
686  return false;
687}
688
689bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
690                                               amd_kernel_code_t &Header) {
691
692  if (getLexer().isNot(AsmToken::Equal))
693    return TokError("expected '='");
694  Lex();
695
696  if (getLexer().isNot(AsmToken::Integer))
697    return TokError("amd_kernel_code_t values must be integers");
698
699  uint64_t Value = getLexer().getTok().getIntVal();
700  Lex();
701
702  if (ID == "kernel_code_version_major")
703    Header.amd_kernel_code_version_major = Value;
704  else if (ID == "kernel_code_version_minor")
705    Header.amd_kernel_code_version_minor = Value;
706  else if (ID == "machine_kind")
707    Header.amd_machine_kind = Value;
708  else if (ID == "machine_version_major")
709    Header.amd_machine_version_major = Value;
710  else if (ID == "machine_version_minor")
711    Header.amd_machine_version_minor = Value;
712  else if (ID == "machine_version_stepping")
713    Header.amd_machine_version_stepping = Value;
714  else if (ID == "kernel_code_entry_byte_offset")
715    Header.kernel_code_entry_byte_offset = Value;
716  else if (ID == "kernel_code_prefetch_byte_size")
717    Header.kernel_code_prefetch_byte_size = Value;
718  else if (ID == "max_scratch_backing_memory_byte_size")
719    Header.max_scratch_backing_memory_byte_size = Value;
720  else if (ID == "compute_pgm_rsrc1_vgprs")
721    Header.compute_pgm_resource_registers |= S_00B848_VGPRS(Value);
722  else if (ID == "compute_pgm_rsrc1_sgprs")
723    Header.compute_pgm_resource_registers |= S_00B848_SGPRS(Value);
724  else if (ID == "compute_pgm_rsrc1_priority")
725    Header.compute_pgm_resource_registers |= S_00B848_PRIORITY(Value);
726  else if (ID == "compute_pgm_rsrc1_float_mode")
727    Header.compute_pgm_resource_registers |= S_00B848_FLOAT_MODE(Value);
728  else if (ID == "compute_pgm_rsrc1_priv")
729    Header.compute_pgm_resource_registers |= S_00B848_PRIV(Value);
730  else if (ID == "compute_pgm_rsrc1_dx10_clamp")
731    Header.compute_pgm_resource_registers |= S_00B848_DX10_CLAMP(Value);
732  else if (ID == "compute_pgm_rsrc1_debug_mode")
733    Header.compute_pgm_resource_registers |= S_00B848_DEBUG_MODE(Value);
734  else if (ID == "compute_pgm_rsrc1_ieee_mode")
735    Header.compute_pgm_resource_registers |= S_00B848_IEEE_MODE(Value);
736  else if (ID == "compute_pgm_rsrc2_scratch_en")
737    Header.compute_pgm_resource_registers |= (S_00B84C_SCRATCH_EN(Value) << 32);
738  else if (ID == "compute_pgm_rsrc2_user_sgpr")
739    Header.compute_pgm_resource_registers |= (S_00B84C_USER_SGPR(Value) << 32);
740  else if (ID == "compute_pgm_rsrc2_tgid_x_en")
741    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_X_EN(Value) << 32);
742  else if (ID == "compute_pgm_rsrc2_tgid_y_en")
743    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Y_EN(Value) << 32);
744  else if (ID == "compute_pgm_rsrc2_tgid_z_en")
745    Header.compute_pgm_resource_registers |= (S_00B84C_TGID_Z_EN(Value) << 32);
746  else if (ID == "compute_pgm_rsrc2_tg_size_en")
747    Header.compute_pgm_resource_registers |= (S_00B84C_TG_SIZE_EN(Value) << 32);
748  else if (ID == "compute_pgm_rsrc2_tidig_comp_cnt")
749    Header.compute_pgm_resource_registers |=
750        (S_00B84C_TIDIG_COMP_CNT(Value) << 32);
751  else if (ID == "compute_pgm_rsrc2_excp_en_msb")
752    Header.compute_pgm_resource_registers |=
753        (S_00B84C_EXCP_EN_MSB(Value) << 32);
754  else if (ID == "compute_pgm_rsrc2_lds_size")
755    Header.compute_pgm_resource_registers |= (S_00B84C_LDS_SIZE(Value) << 32);
756  else if (ID == "compute_pgm_rsrc2_excp_en")
757    Header.compute_pgm_resource_registers |= (S_00B84C_EXCP_EN(Value) << 32);
758  else if (ID == "compute_pgm_resource_registers")
759    Header.compute_pgm_resource_registers = Value;
760  else if (ID == "enable_sgpr_private_segment_buffer")
761    Header.code_properties |=
762        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT);
763  else if (ID == "enable_sgpr_dispatch_ptr")
764    Header.code_properties |=
765        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT);
766  else if (ID == "enable_sgpr_queue_ptr")
767    Header.code_properties |=
768        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT);
769  else if (ID == "enable_sgpr_kernarg_segment_ptr")
770    Header.code_properties |=
771        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT);
772  else if (ID == "enable_sgpr_dispatch_id")
773    Header.code_properties |=
774        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT);
775  else if (ID == "enable_sgpr_flat_scratch_init")
776    Header.code_properties |=
777        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT);
778  else if (ID == "enable_sgpr_private_segment_size")
779    Header.code_properties |=
780        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT);
781  else if (ID == "enable_sgpr_grid_workgroup_count_x")
782    Header.code_properties |=
783        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_X_SHIFT);
784  else if (ID == "enable_sgpr_grid_workgroup_count_y")
785    Header.code_properties |=
786        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Y_SHIFT);
787  else if (ID == "enable_sgpr_grid_workgroup_count_z")
788    Header.code_properties |=
789        (Value << AMD_CODE_PROPERTY_ENABLE_SGPR_GRID_WORKGROUP_COUNT_Z_SHIFT);
790  else if (ID == "enable_ordered_append_gds")
791    Header.code_properties |=
792        (Value << AMD_CODE_PROPERTY_ENABLE_ORDERED_APPEND_GDS_SHIFT);
793  else if (ID == "private_element_size")
794    Header.code_properties |=
795        (Value << AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE_SHIFT);
796  else if (ID == "is_ptr64")
797    Header.code_properties |=
798        (Value << AMD_CODE_PROPERTY_IS_PTR64_SHIFT);
799  else if (ID == "is_dynamic_callstack")
800    Header.code_properties |=
801        (Value << AMD_CODE_PROPERTY_IS_DYNAMIC_CALLSTACK_SHIFT);
802  else if (ID == "is_debug_enabled")
803    Header.code_properties |=
804        (Value << AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED_SHIFT);
805  else if (ID == "is_xnack_enabled")
806    Header.code_properties |=
807        (Value << AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED_SHIFT);
808  else if (ID == "workitem_private_segment_byte_size")
809    Header.workitem_private_segment_byte_size = Value;
810  else if (ID == "workgroup_group_segment_byte_size")
811    Header.workgroup_group_segment_byte_size = Value;
812  else if (ID == "gds_segment_byte_size")
813    Header.gds_segment_byte_size = Value;
814  else if (ID == "kernarg_segment_byte_size")
815    Header.kernarg_segment_byte_size = Value;
816  else if (ID == "workgroup_fbarrier_count")
817    Header.workgroup_fbarrier_count = Value;
818  else if (ID == "wavefront_sgpr_count")
819    Header.wavefront_sgpr_count = Value;
820  else if (ID == "workitem_vgpr_count")
821    Header.workitem_vgpr_count = Value;
822  else if (ID == "reserved_vgpr_first")
823    Header.reserved_vgpr_first = Value;
824  else if (ID == "reserved_vgpr_count")
825    Header.reserved_vgpr_count = Value;
826  else if (ID == "reserved_sgpr_first")
827    Header.reserved_sgpr_first = Value;
828  else if (ID == "reserved_sgpr_count")
829    Header.reserved_sgpr_count = Value;
830  else if (ID == "debug_wavefront_private_segment_offset_sgpr")
831    Header.debug_wavefront_private_segment_offset_sgpr = Value;
832  else if (ID == "debug_private_segment_buffer_sgpr")
833    Header.debug_private_segment_buffer_sgpr = Value;
834  else if (ID == "kernarg_segment_alignment")
835    Header.kernarg_segment_alignment = Value;
836  else if (ID == "group_segment_alignment")
837    Header.group_segment_alignment = Value;
838  else if (ID == "private_segment_alignment")
839    Header.private_segment_alignment = Value;
840  else if (ID == "wavefront_size")
841    Header.wavefront_size = Value;
842  else if (ID == "call_convention")
843    Header.call_convention = Value;
844  else if (ID == "runtime_loader_kernel_symbol")
845    Header.runtime_loader_kernel_symbol = Value;
846  else
847    return TokError("amd_kernel_code_t value not recognized.");
848
849  return false;
850}
851
852bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
853
854  amd_kernel_code_t Header;
855  AMDGPU::initDefaultAMDKernelCodeT(Header, STI.getFeatureBits());
856
857  while (true) {
858
859    if (getLexer().isNot(AsmToken::EndOfStatement))
860      return TokError("amd_kernel_code_t values must begin on a new line");
861
862    // Lex EndOfStatement.  This is in a while loop, because lexing a comment
863    // will set the current token to EndOfStatement.
864    while(getLexer().is(AsmToken::EndOfStatement))
865      Lex();
866
867    if (getLexer().isNot(AsmToken::Identifier))
868      return TokError("expected value identifier or .end_amd_kernel_code_t");
869
870    StringRef ID = getLexer().getTok().getIdentifier();
871    Lex();
872
873    if (ID == ".end_amd_kernel_code_t")
874      break;
875
876    if (ParseAMDKernelCodeTValue(ID, Header))
877      return true;
878  }
879
880  getTargetStreamer().EmitAMDKernelCodeT(Header);
881
882  return false;
883}
884
885bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
886  StringRef IDVal = DirectiveID.getString();
887
888  if (IDVal == ".hsa_code_object_version")
889    return ParseDirectiveHSACodeObjectVersion();
890
891  if (IDVal == ".hsa_code_object_isa")
892    return ParseDirectiveHSACodeObjectISA();
893
894  if (IDVal == ".amd_kernel_code_t")
895    return ParseDirectiveAMDKernelCodeT();
896
897  return true;
898}
899
900static bool operandsHaveModifiers(const OperandVector &Operands) {
901
902  for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
903    const AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
904    if (Op.isRegKind() && Op.hasModifiers())
905      return true;
906    if (Op.isImm() && (Op.getImmTy() == AMDGPUOperand::ImmTyOMod ||
907                       Op.getImmTy() == AMDGPUOperand::ImmTyClamp))
908      return true;
909  }
910  return false;
911}
912
913AMDGPUAsmParser::OperandMatchResultTy
914AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
915
916  // Try to parse with a custom parser
917  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
918
919  // If we successfully parsed the operand or if there as an error parsing,
920  // we are done.
921  //
922  // If we are parsing after we reach EndOfStatement then this means we
923  // are appending default values to the Operands list.  This is only done
924  // by custom parser, so we shouldn't continue on to the generic parsing.
925  if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
926      getLexer().is(AsmToken::EndOfStatement))
927    return ResTy;
928
929  bool Negate = false, Abs = false;
930  if (getLexer().getKind()== AsmToken::Minus) {
931    Parser.Lex();
932    Negate = true;
933  }
934
935  if (getLexer().getKind() == AsmToken::Pipe) {
936    Parser.Lex();
937    Abs = true;
938  }
939
940  switch(getLexer().getKind()) {
941    case AsmToken::Integer: {
942      SMLoc S = Parser.getTok().getLoc();
943      int64_t IntVal;
944      if (getParser().parseAbsoluteExpression(IntVal))
945        return MatchOperand_ParseFail;
946      APInt IntVal32(32, IntVal);
947      if (IntVal32.getSExtValue() != IntVal) {
948        Error(S, "invalid immediate: only 32-bit values are legal");
949        return MatchOperand_ParseFail;
950      }
951
952      IntVal = IntVal32.getSExtValue();
953      if (Negate)
954        IntVal *= -1;
955      Operands.push_back(AMDGPUOperand::CreateImm(IntVal, S));
956      return MatchOperand_Success;
957    }
958    case AsmToken::Real: {
959      // FIXME: We should emit an error if a double precisions floating-point
960      // value is used.  I'm not sure the best way to detect this.
961      SMLoc S = Parser.getTok().getLoc();
962      int64_t IntVal;
963      if (getParser().parseAbsoluteExpression(IntVal))
964        return MatchOperand_ParseFail;
965
966      APFloat F((float)BitsToDouble(IntVal));
967      if (Negate)
968        F.changeSign();
969      Operands.push_back(
970          AMDGPUOperand::CreateImm(F.bitcastToAPInt().getZExtValue(), S));
971      return MatchOperand_Success;
972    }
973    case AsmToken::Identifier: {
974      SMLoc S, E;
975      unsigned RegNo;
976      if (!ParseRegister(RegNo, S, E)) {
977
978        bool HasModifiers = operandsHaveModifiers(Operands);
979        unsigned Modifiers = 0;
980
981        if (Negate)
982          Modifiers |= 0x1;
983
984        if (Abs) {
985          if (getLexer().getKind() != AsmToken::Pipe)
986            return MatchOperand_ParseFail;
987          Parser.Lex();
988          Modifiers |= 0x2;
989        }
990
991        if (Modifiers && !HasModifiers) {
992          // We are adding a modifier to src1 or src2 and previous sources
993          // don't have modifiers, so we need to go back and empty modifers
994          // for each previous source.
995          for (unsigned PrevRegIdx = Operands.size() - 1; PrevRegIdx > 1;
996               --PrevRegIdx) {
997
998            AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[PrevRegIdx]);
999            RegOp.setModifiers(0);
1000          }
1001        }
1002
1003
1004        Operands.push_back(AMDGPUOperand::CreateReg(
1005            RegNo, S, E, getContext().getRegisterInfo(),
1006            isForcedVOP3()));
1007
1008        if (HasModifiers || Modifiers) {
1009          AMDGPUOperand &RegOp = ((AMDGPUOperand&)*Operands[Operands.size() - 1]);
1010          RegOp.setModifiers(Modifiers);
1011
1012        }
1013     }  else {
1014      Operands.push_back(AMDGPUOperand::CreateToken(Parser.getTok().getString(),
1015                                                    S));
1016      Parser.Lex();
1017     }
1018     return MatchOperand_Success;
1019    }
1020    default:
1021      return MatchOperand_NoMatch;
1022  }
1023}
1024
1025bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
1026                                       StringRef Name,
1027                                       SMLoc NameLoc, OperandVector &Operands) {
1028
1029  // Clear any forced encodings from the previous instruction.
1030  setForcedEncodingSize(0);
1031
1032  if (Name.endswith("_e64"))
1033    setForcedEncodingSize(64);
1034  else if (Name.endswith("_e32"))
1035    setForcedEncodingSize(32);
1036
1037  // Add the instruction mnemonic
1038  Operands.push_back(AMDGPUOperand::CreateToken(Name, NameLoc));
1039
1040  while (!getLexer().is(AsmToken::EndOfStatement)) {
1041    AMDGPUAsmParser::OperandMatchResultTy Res = parseOperand(Operands, Name);
1042
1043    // Eat the comma or space if there is one.
1044    if (getLexer().is(AsmToken::Comma))
1045      Parser.Lex();
1046
1047    switch (Res) {
1048      case MatchOperand_Success: break;
1049      case MatchOperand_ParseFail: return Error(getLexer().getLoc(),
1050                                                "failed parsing operand.");
1051      case MatchOperand_NoMatch: return Error(getLexer().getLoc(),
1052                                              "not a valid operand.");
1053    }
1054  }
1055
1056  // Once we reach end of statement, continue parsing so we can add default
1057  // values for optional arguments.
1058  AMDGPUAsmParser::OperandMatchResultTy Res;
1059  while ((Res = parseOperand(Operands, Name)) != MatchOperand_NoMatch) {
1060    if (Res != MatchOperand_Success)
1061      return Error(getLexer().getLoc(), "failed parsing operand.");
1062  }
1063  return false;
1064}
1065
1066//===----------------------------------------------------------------------===//
1067// Utility functions
1068//===----------------------------------------------------------------------===//
1069
1070AMDGPUAsmParser::OperandMatchResultTy
1071AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &Int,
1072                                    int64_t Default) {
1073
1074  // We are at the end of the statement, and this is a default argument, so
1075  // use a default value.
1076  if (getLexer().is(AsmToken::EndOfStatement)) {
1077    Int = Default;
1078    return MatchOperand_Success;
1079  }
1080
1081  switch(getLexer().getKind()) {
1082    default: return MatchOperand_NoMatch;
1083    case AsmToken::Identifier: {
1084      StringRef OffsetName = Parser.getTok().getString();
1085      if (!OffsetName.equals(Prefix))
1086        return MatchOperand_NoMatch;
1087
1088      Parser.Lex();
1089      if (getLexer().isNot(AsmToken::Colon))
1090        return MatchOperand_ParseFail;
1091
1092      Parser.Lex();
1093      if (getLexer().isNot(AsmToken::Integer))
1094        return MatchOperand_ParseFail;
1095
1096      if (getParser().parseAbsoluteExpression(Int))
1097        return MatchOperand_ParseFail;
1098      break;
1099    }
1100  }
1101  return MatchOperand_Success;
1102}
1103
1104AMDGPUAsmParser::OperandMatchResultTy
1105AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1106                                    enum AMDGPUOperand::ImmTy ImmTy) {
1107
1108  SMLoc S = Parser.getTok().getLoc();
1109  int64_t Offset = 0;
1110
1111  AMDGPUAsmParser::OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Offset);
1112  if (Res != MatchOperand_Success)
1113    return Res;
1114
1115  Operands.push_back(AMDGPUOperand::CreateImm(Offset, S, ImmTy));
1116  return MatchOperand_Success;
1117}
1118
1119AMDGPUAsmParser::OperandMatchResultTy
1120AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
1121                               enum AMDGPUOperand::ImmTy ImmTy) {
1122  int64_t Bit = 0;
1123  SMLoc S = Parser.getTok().getLoc();
1124
1125  // We are at the end of the statement, and this is a default argument, so
1126  // use a default value.
1127  if (getLexer().isNot(AsmToken::EndOfStatement)) {
1128    switch(getLexer().getKind()) {
1129      case AsmToken::Identifier: {
1130        StringRef Tok = Parser.getTok().getString();
1131        if (Tok == Name) {
1132          Bit = 1;
1133          Parser.Lex();
1134        } else if (Tok.startswith("no") && Tok.endswith(Name)) {
1135          Bit = 0;
1136          Parser.Lex();
1137        } else {
1138          return MatchOperand_NoMatch;
1139        }
1140        break;
1141      }
1142      default:
1143        return MatchOperand_NoMatch;
1144    }
1145  }
1146
1147  Operands.push_back(AMDGPUOperand::CreateImm(Bit, S, ImmTy));
1148  return MatchOperand_Success;
1149}
1150
1151static bool operandsHasOptionalOp(const OperandVector &Operands,
1152                                  const OptionalOperand &OOp) {
1153  for (unsigned i = 0; i < Operands.size(); i++) {
1154    const AMDGPUOperand &ParsedOp = ((const AMDGPUOperand &)*Operands[i]);
1155    if ((ParsedOp.isImm() && ParsedOp.getImmTy() == OOp.Type) ||
1156        (ParsedOp.isToken() && ParsedOp.getToken() == OOp.Name))
1157      return true;
1158
1159  }
1160  return false;
1161}
1162
1163AMDGPUAsmParser::OperandMatchResultTy
1164AMDGPUAsmParser::parseOptionalOps(const ArrayRef<OptionalOperand> &OptionalOps,
1165                                   OperandVector &Operands) {
1166  SMLoc S = Parser.getTok().getLoc();
1167  for (const OptionalOperand &Op : OptionalOps) {
1168    if (operandsHasOptionalOp(Operands, Op))
1169      continue;
1170    AMDGPUAsmParser::OperandMatchResultTy Res;
1171    int64_t Value;
1172    if (Op.IsBit) {
1173      Res = parseNamedBit(Op.Name, Operands, Op.Type);
1174      if (Res == MatchOperand_NoMatch)
1175        continue;
1176      return Res;
1177    }
1178
1179    Res = parseIntWithPrefix(Op.Name, Value, Op.Default);
1180
1181    if (Res == MatchOperand_NoMatch)
1182      continue;
1183
1184    if (Res != MatchOperand_Success)
1185      return Res;
1186
1187    if (Op.ConvertResult && !Op.ConvertResult(Value)) {
1188      return MatchOperand_ParseFail;
1189    }
1190
1191    Operands.push_back(AMDGPUOperand::CreateImm(Value, S, Op.Type));
1192    return MatchOperand_Success;
1193  }
1194  return MatchOperand_NoMatch;
1195}
1196
1197//===----------------------------------------------------------------------===//
1198// ds
1199//===----------------------------------------------------------------------===//
1200
1201static const OptionalOperand DSOptionalOps [] = {
1202  {"offset",  AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1203  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1204};
1205
1206static const OptionalOperand DSOptionalOpsOff01 [] = {
1207  {"offset0", AMDGPUOperand::ImmTyDSOffset0, false, 0, nullptr},
1208  {"offset1", AMDGPUOperand::ImmTyDSOffset1, false, 0, nullptr},
1209  {"gds",     AMDGPUOperand::ImmTyGDS, true, 0, nullptr}
1210};
1211
1212AMDGPUAsmParser::OperandMatchResultTy
1213AMDGPUAsmParser::parseDSOptionalOps(OperandVector &Operands) {
1214  return parseOptionalOps(DSOptionalOps, Operands);
1215}
1216AMDGPUAsmParser::OperandMatchResultTy
1217AMDGPUAsmParser::parseDSOff01OptionalOps(OperandVector &Operands) {
1218  return parseOptionalOps(DSOptionalOpsOff01, Operands);
1219}
1220
1221AMDGPUAsmParser::OperandMatchResultTy
1222AMDGPUAsmParser::parseDSOffsetOptional(OperandVector &Operands) {
1223  SMLoc S = Parser.getTok().getLoc();
1224  AMDGPUAsmParser::OperandMatchResultTy Res =
1225    parseIntWithPrefix("offset", Operands, AMDGPUOperand::ImmTyOffset);
1226  if (Res == MatchOperand_NoMatch) {
1227    Operands.push_back(AMDGPUOperand::CreateImm(0, S,
1228                       AMDGPUOperand::ImmTyOffset));
1229    Res = MatchOperand_Success;
1230  }
1231  return Res;
1232}
1233
1234bool AMDGPUOperand::isDSOffset() const {
1235  return isImm() && isUInt<16>(getImm());
1236}
1237
1238bool AMDGPUOperand::isDSOffset01() const {
1239  return isImm() && isUInt<8>(getImm());
1240}
1241
1242void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
1243                                    const OperandVector &Operands) {
1244
1245  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1246
1247  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1248    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1249
1250    // Add the register arguments
1251    if (Op.isReg()) {
1252      Op.addRegOperands(Inst, 1);
1253      continue;
1254    }
1255
1256    // Handle optional arguments
1257    OptionalIdx[Op.getImmTy()] = i;
1258  }
1259
1260  unsigned Offset0Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset0];
1261  unsigned Offset1Idx = OptionalIdx[AMDGPUOperand::ImmTyDSOffset1];
1262  unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
1263
1264  ((AMDGPUOperand &)*Operands[Offset0Idx]).addImmOperands(Inst, 1); // offset0
1265  ((AMDGPUOperand &)*Operands[Offset1Idx]).addImmOperands(Inst, 1); // offset1
1266  ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
1267  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1268}
1269
1270void AMDGPUAsmParser::cvtDS(MCInst &Inst, const OperandVector &Operands) {
1271
1272  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1273  bool GDSOnly = false;
1274
1275  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1276    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1277
1278    // Add the register arguments
1279    if (Op.isReg()) {
1280      Op.addRegOperands(Inst, 1);
1281      continue;
1282    }
1283
1284    if (Op.isToken() && Op.getToken() == "gds") {
1285      GDSOnly = true;
1286      continue;
1287    }
1288
1289    // Handle optional arguments
1290    OptionalIdx[Op.getImmTy()] = i;
1291  }
1292
1293  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
1294  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1); // offset
1295
1296  if (!GDSOnly) {
1297    unsigned GDSIdx = OptionalIdx[AMDGPUOperand::ImmTyGDS];
1298    ((AMDGPUOperand &)*Operands[GDSIdx]).addImmOperands(Inst, 1); // gds
1299  }
1300  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
1301}
1302
1303
1304//===----------------------------------------------------------------------===//
1305// s_waitcnt
1306//===----------------------------------------------------------------------===//
1307
1308bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
1309  StringRef CntName = Parser.getTok().getString();
1310  int64_t CntVal;
1311
1312  Parser.Lex();
1313  if (getLexer().isNot(AsmToken::LParen))
1314    return true;
1315
1316  Parser.Lex();
1317  if (getLexer().isNot(AsmToken::Integer))
1318    return true;
1319
1320  if (getParser().parseAbsoluteExpression(CntVal))
1321    return true;
1322
1323  if (getLexer().isNot(AsmToken::RParen))
1324    return true;
1325
1326  Parser.Lex();
1327  if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma))
1328    Parser.Lex();
1329
1330  int CntShift;
1331  int CntMask;
1332
1333  if (CntName == "vmcnt") {
1334    CntMask = 0xf;
1335    CntShift = 0;
1336  } else if (CntName == "expcnt") {
1337    CntMask = 0x7;
1338    CntShift = 4;
1339  } else if (CntName == "lgkmcnt") {
1340    CntMask = 0x7;
1341    CntShift = 8;
1342  } else {
1343    return true;
1344  }
1345
1346  IntVal &= ~(CntMask << CntShift);
1347  IntVal |= (CntVal << CntShift);
1348  return false;
1349}
1350
1351AMDGPUAsmParser::OperandMatchResultTy
1352AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
1353  // Disable all counters by default.
1354  // vmcnt   [3:0]
1355  // expcnt  [6:4]
1356  // lgkmcnt [10:8]
1357  int64_t CntVal = 0x77f;
1358  SMLoc S = Parser.getTok().getLoc();
1359
1360  switch(getLexer().getKind()) {
1361    default: return MatchOperand_ParseFail;
1362    case AsmToken::Integer:
1363      // The operand can be an integer value.
1364      if (getParser().parseAbsoluteExpression(CntVal))
1365        return MatchOperand_ParseFail;
1366      break;
1367
1368    case AsmToken::Identifier:
1369      do {
1370        if (parseCnt(CntVal))
1371          return MatchOperand_ParseFail;
1372      } while(getLexer().isNot(AsmToken::EndOfStatement));
1373      break;
1374  }
1375  Operands.push_back(AMDGPUOperand::CreateImm(CntVal, S));
1376  return MatchOperand_Success;
1377}
1378
1379bool AMDGPUOperand::isSWaitCnt() const {
1380  return isImm();
1381}
1382
1383//===----------------------------------------------------------------------===//
1384// sopp branch targets
1385//===----------------------------------------------------------------------===//
1386
1387AMDGPUAsmParser::OperandMatchResultTy
1388AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
1389  SMLoc S = Parser.getTok().getLoc();
1390
1391  switch (getLexer().getKind()) {
1392    default: return MatchOperand_ParseFail;
1393    case AsmToken::Integer: {
1394      int64_t Imm;
1395      if (getParser().parseAbsoluteExpression(Imm))
1396        return MatchOperand_ParseFail;
1397      Operands.push_back(AMDGPUOperand::CreateImm(Imm, S));
1398      return MatchOperand_Success;
1399    }
1400
1401    case AsmToken::Identifier:
1402      Operands.push_back(AMDGPUOperand::CreateExpr(
1403          MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
1404                                  Parser.getTok().getString()), getContext()), S));
1405      Parser.Lex();
1406      return MatchOperand_Success;
1407  }
1408}
1409
1410//===----------------------------------------------------------------------===//
1411// flat
1412//===----------------------------------------------------------------------===//
1413
1414static const OptionalOperand FlatOptionalOps [] = {
1415  {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1416  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1417  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1418};
1419
1420static const OptionalOperand FlatAtomicOptionalOps [] = {
1421  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1422  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1423};
1424
1425AMDGPUAsmParser::OperandMatchResultTy
1426AMDGPUAsmParser::parseFlatOptionalOps(OperandVector &Operands) {
1427  return parseOptionalOps(FlatOptionalOps, Operands);
1428}
1429
1430AMDGPUAsmParser::OperandMatchResultTy
1431AMDGPUAsmParser::parseFlatAtomicOptionalOps(OperandVector &Operands) {
1432  return parseOptionalOps(FlatAtomicOptionalOps, Operands);
1433}
1434
1435void AMDGPUAsmParser::cvtFlat(MCInst &Inst,
1436                               const OperandVector &Operands) {
1437  std::map<AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1438
1439  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1440    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1441
1442    // Add the register arguments
1443    if (Op.isReg()) {
1444      Op.addRegOperands(Inst, 1);
1445      continue;
1446    }
1447
1448    // Handle 'glc' token which is sometimes hard-coded into the
1449    // asm string.  There are no MCInst operands for these.
1450    if (Op.isToken())
1451      continue;
1452
1453    // Handle optional arguments
1454    OptionalIdx[Op.getImmTy()] = i;
1455
1456  }
1457
1458  // flat atomic instructions don't have a glc argument.
1459  if (OptionalIdx.count(AMDGPUOperand::ImmTyGLC)) {
1460    unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
1461    ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
1462  }
1463
1464  unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
1465  unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
1466
1467  ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
1468  ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
1469}
1470
1471//===----------------------------------------------------------------------===//
1472// mubuf
1473//===----------------------------------------------------------------------===//
1474
1475static const OptionalOperand MubufOptionalOps [] = {
1476  {"offset", AMDGPUOperand::ImmTyOffset, false, 0, nullptr},
1477  {"glc",    AMDGPUOperand::ImmTyGLC, true, 0, nullptr},
1478  {"slc",    AMDGPUOperand::ImmTySLC, true, 0, nullptr},
1479  {"tfe",    AMDGPUOperand::ImmTyTFE, true, 0, nullptr}
1480};
1481
1482AMDGPUAsmParser::OperandMatchResultTy
1483AMDGPUAsmParser::parseMubufOptionalOps(OperandVector &Operands) {
1484  return parseOptionalOps(MubufOptionalOps, Operands);
1485}
1486
1487AMDGPUAsmParser::OperandMatchResultTy
1488AMDGPUAsmParser::parseOffset(OperandVector &Operands) {
1489  return parseIntWithPrefix("offset", Operands);
1490}
1491
1492AMDGPUAsmParser::OperandMatchResultTy
1493AMDGPUAsmParser::parseGLC(OperandVector &Operands) {
1494  return parseNamedBit("glc", Operands);
1495}
1496
1497AMDGPUAsmParser::OperandMatchResultTy
1498AMDGPUAsmParser::parseSLC(OperandVector &Operands) {
1499  return parseNamedBit("slc", Operands);
1500}
1501
1502AMDGPUAsmParser::OperandMatchResultTy
1503AMDGPUAsmParser::parseTFE(OperandVector &Operands) {
1504  return parseNamedBit("tfe", Operands);
1505}
1506
1507bool AMDGPUOperand::isMubufOffset() const {
1508  return isImm() && isUInt<12>(getImm());
1509}
1510
1511void AMDGPUAsmParser::cvtMubuf(MCInst &Inst,
1512                               const OperandVector &Operands) {
1513  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1514
1515  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1516    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1517
1518    // Add the register arguments
1519    if (Op.isReg()) {
1520      Op.addRegOperands(Inst, 1);
1521      continue;
1522    }
1523
1524    // Handle the case where soffset is an immediate
1525    if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
1526      Op.addImmOperands(Inst, 1);
1527      continue;
1528    }
1529
1530    // Handle tokens like 'offen' which are sometimes hard-coded into the
1531    // asm string.  There are no MCInst operands for these.
1532    if (Op.isToken()) {
1533      continue;
1534    }
1535    assert(Op.isImm());
1536
1537    // Handle optional arguments
1538    OptionalIdx[Op.getImmTy()] = i;
1539  }
1540
1541  assert(OptionalIdx.size() == 4);
1542
1543  unsigned OffsetIdx = OptionalIdx[AMDGPUOperand::ImmTyOffset];
1544  unsigned GLCIdx = OptionalIdx[AMDGPUOperand::ImmTyGLC];
1545  unsigned SLCIdx = OptionalIdx[AMDGPUOperand::ImmTySLC];
1546  unsigned TFEIdx = OptionalIdx[AMDGPUOperand::ImmTyTFE];
1547
1548  ((AMDGPUOperand &)*Operands[OffsetIdx]).addImmOperands(Inst, 1);
1549  ((AMDGPUOperand &)*Operands[GLCIdx]).addImmOperands(Inst, 1);
1550  ((AMDGPUOperand &)*Operands[SLCIdx]).addImmOperands(Inst, 1);
1551  ((AMDGPUOperand &)*Operands[TFEIdx]).addImmOperands(Inst, 1);
1552}
1553
1554//===----------------------------------------------------------------------===//
1555// mimg
1556//===----------------------------------------------------------------------===//
1557
1558AMDGPUAsmParser::OperandMatchResultTy
1559AMDGPUAsmParser::parseDMask(OperandVector &Operands) {
1560  return parseIntWithPrefix("dmask", Operands);
1561}
1562
1563AMDGPUAsmParser::OperandMatchResultTy
1564AMDGPUAsmParser::parseUNorm(OperandVector &Operands) {
1565  return parseNamedBit("unorm", Operands);
1566}
1567
1568AMDGPUAsmParser::OperandMatchResultTy
1569AMDGPUAsmParser::parseR128(OperandVector &Operands) {
1570  return parseNamedBit("r128", Operands);
1571}
1572
1573//===----------------------------------------------------------------------===//
1574// vop3
1575//===----------------------------------------------------------------------===//
1576
1577static bool ConvertOmodMul(int64_t &Mul) {
1578  if (Mul != 1 && Mul != 2 && Mul != 4)
1579    return false;
1580
1581  Mul >>= 1;
1582  return true;
1583}
1584
1585static bool ConvertOmodDiv(int64_t &Div) {
1586  if (Div == 1) {
1587    Div = 0;
1588    return true;
1589  }
1590
1591  if (Div == 2) {
1592    Div = 3;
1593    return true;
1594  }
1595
1596  return false;
1597}
1598
1599static const OptionalOperand VOP3OptionalOps [] = {
1600  {"clamp", AMDGPUOperand::ImmTyClamp, true, 0, nullptr},
1601  {"mul",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodMul},
1602  {"div",   AMDGPUOperand::ImmTyOMod, false, 1, ConvertOmodDiv},
1603};
1604
1605static bool isVOP3(OperandVector &Operands) {
1606  if (operandsHaveModifiers(Operands))
1607    return true;
1608
1609  AMDGPUOperand &DstOp = ((AMDGPUOperand&)*Operands[1]);
1610
1611  if (DstOp.isReg() && DstOp.isRegClass(AMDGPU::SGPR_64RegClassID))
1612    return true;
1613
1614  if (Operands.size() >= 5)
1615    return true;
1616
1617  if (Operands.size() > 3) {
1618    AMDGPUOperand &Src1Op = ((AMDGPUOperand&)*Operands[3]);
1619    if (Src1Op.getReg() && (Src1Op.isRegClass(AMDGPU::SReg_32RegClassID) ||
1620                            Src1Op.isRegClass(AMDGPU::SReg_64RegClassID)))
1621      return true;
1622  }
1623  return false;
1624}
1625
1626AMDGPUAsmParser::OperandMatchResultTy
1627AMDGPUAsmParser::parseVOP3OptionalOps(OperandVector &Operands) {
1628
1629  // The value returned by this function may change after parsing
1630  // an operand so store the original value here.
1631  bool HasModifiers = operandsHaveModifiers(Operands);
1632
1633  bool IsVOP3 = isVOP3(Operands);
1634  if (HasModifiers || IsVOP3 ||
1635      getLexer().isNot(AsmToken::EndOfStatement) ||
1636      getForcedEncodingSize() == 64) {
1637
1638    AMDGPUAsmParser::OperandMatchResultTy Res =
1639        parseOptionalOps(VOP3OptionalOps, Operands);
1640
1641    if (!HasModifiers && Res == MatchOperand_Success) {
1642      // We have added a modifier operation, so we need to make sure all
1643      // previous register operands have modifiers
1644      for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
1645        AMDGPUOperand &Op = ((AMDGPUOperand&)*Operands[i]);
1646        if (Op.isReg())
1647          Op.setModifiers(0);
1648      }
1649    }
1650    return Res;
1651  }
1652  return MatchOperand_NoMatch;
1653}
1654
1655void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
1656  ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, 1);
1657  unsigned i = 2;
1658
1659  std::map<enum AMDGPUOperand::ImmTy, unsigned> OptionalIdx;
1660
1661  if (operandsHaveModifiers(Operands)) {
1662    for (unsigned e = Operands.size(); i != e; ++i) {
1663      AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
1664
1665      if (Op.isRegWithInputMods()) {
1666        ((AMDGPUOperand &)*Operands[i]).addRegWithInputModsOperands(Inst, 2);
1667        continue;
1668      }
1669      OptionalIdx[Op.getImmTy()] = i;
1670    }
1671
1672    unsigned ClampIdx = OptionalIdx[AMDGPUOperand::ImmTyClamp];
1673    unsigned OModIdx = OptionalIdx[AMDGPUOperand::ImmTyOMod];
1674
1675    ((AMDGPUOperand &)*Operands[ClampIdx]).addImmOperands(Inst, 1);
1676    ((AMDGPUOperand &)*Operands[OModIdx]).addImmOperands(Inst, 1);
1677  } else {
1678    for (unsigned e = Operands.size(); i != e; ++i)
1679      ((AMDGPUOperand &)*Operands[i]).addRegOrImmOperands(Inst, 1);
1680  }
1681}
1682
1683/// Force static initialization.
1684extern "C" void LLVMInitializeAMDGPUAsmParser() {
1685  RegisterMCAsmParser<AMDGPUAsmParser> A(TheAMDGPUTarget);
1686  RegisterMCAsmParser<AMDGPUAsmParser> B(TheGCNTarget);
1687}
1688
1689#define GET_REGISTER_MATCHER
1690#define GET_MATCHER_IMPLEMENTATION
1691#include "AMDGPUGenAsmMatcher.inc"
1692
1693