1//===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the RISC-V assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVInstPrinter.h"
16#include "MCTargetDesc/RISCVMCExpr.h"
17#include "MCTargetDesc/RISCVTargetStreamer.h"
18#include "RISCV.h"
19#include "RISCVMachineFunctionInfo.h"
20#include "RISCVTargetMachine.h"
21#include "TargetInfo/RISCVTargetInfo.h"
22#include "llvm/ADT/APInt.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/BinaryFormat/ELF.h"
25#include "llvm/CodeGen/AsmPrinter.h"
26#include "llvm/CodeGen/MachineConstantPool.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/MC/MCAsmInfo.h"
31#include "llvm/MC/MCContext.h"
32#include "llvm/MC/MCInst.h"
33#include "llvm/MC/MCInstBuilder.h"
34#include "llvm/MC/MCObjectFileInfo.h"
35#include "llvm/MC/MCSectionELF.h"
36#include "llvm/MC/MCStreamer.h"
37#include "llvm/MC/MCSymbol.h"
38#include "llvm/MC/TargetRegistry.h"
39#include "llvm/Support/RISCVISAInfo.h"
40#include "llvm/Support/raw_ostream.h"
41#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
42
43using namespace llvm;
44
45#define DEBUG_TYPE "asm-printer"
46
47STATISTIC(RISCVNumInstrsCompressed,
48          "Number of RISC-V Compressed instructions emitted");
49
50namespace llvm {
51extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures];
52} // namespace llvm
53
54namespace {
55class RISCVAsmPrinter : public AsmPrinter {
56  const RISCVSubtarget *STI;
57
58public:
59  explicit RISCVAsmPrinter(TargetMachine &TM,
60                           std::unique_ptr<MCStreamer> Streamer)
61      : AsmPrinter(TM, std::move(Streamer)) {}
62
63  StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
64
65  void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
66                     const MachineInstr &MI);
67
68  void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
69                       const MachineInstr &MI);
70
71  void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
72                       const MachineInstr &MI);
73
74  bool runOnMachineFunction(MachineFunction &MF) override;
75
76  void emitInstruction(const MachineInstr *MI) override;
77
78  bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
79                       const char *ExtraCode, raw_ostream &OS) override;
80  bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
81                             const char *ExtraCode, raw_ostream &OS) override;
82
83  void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
84  bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
85                                   const MachineInstr *MI);
86
87  typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
88  std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
89  void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
90  void LowerKCFI_CHECK(const MachineInstr &MI);
91  void EmitHwasanMemaccessSymbols(Module &M);
92
93  // Wrapper needed for tblgenned pseudo lowering.
94  bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
95
96  void emitStartOfAsmFile(Module &M) override;
97  void emitEndOfAsmFile(Module &M) override;
98
99  void emitFunctionEntryLabel() override;
100  bool emitDirectiveOptionArch();
101
102private:
103  void emitAttributes(const MCSubtargetInfo &SubtargetInfo);
104
105  void emitNTLHint(const MachineInstr *MI);
106
107  bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
108};
109}
110
111void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
112                                    const MachineInstr &MI) {
113  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
114  unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
115
116  auto &Ctx = OutStreamer.getContext();
117  MCSymbol *MILabel = Ctx.createTempSymbol();
118  OutStreamer.emitLabel(MILabel);
119
120  SM.recordStackMap(*MILabel, MI);
121  assert(NumNOPBytes % NOPBytes == 0 &&
122         "Invalid number of NOP bytes requested!");
123
124  // Scan ahead to trim the shadow.
125  const MachineBasicBlock &MBB = *MI.getParent();
126  MachineBasicBlock::const_iterator MII(MI);
127  ++MII;
128  while (NumNOPBytes > 0) {
129    if (MII == MBB.end() || MII->isCall() ||
130        MII->getOpcode() == RISCV::DBG_VALUE ||
131        MII->getOpcode() == TargetOpcode::PATCHPOINT ||
132        MII->getOpcode() == TargetOpcode::STACKMAP)
133      break;
134    ++MII;
135    NumNOPBytes -= 4;
136  }
137
138  // Emit nops.
139  emitNops(NumNOPBytes / NOPBytes);
140}
141
142// Lower a patchpoint of the form:
143// [<def>], <id>, <numBytes>, <target>, <numArgs>
144void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
145                                      const MachineInstr &MI) {
146  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
147
148  auto &Ctx = OutStreamer.getContext();
149  MCSymbol *MILabel = Ctx.createTempSymbol();
150  OutStreamer.emitLabel(MILabel);
151  SM.recordPatchPoint(*MILabel, MI);
152
153  PatchPointOpers Opers(&MI);
154
155  unsigned EncodedBytes = 0;
156
157  // Emit padding.
158  unsigned NumBytes = Opers.getNumPatchBytes();
159  assert(NumBytes >= EncodedBytes &&
160         "Patchpoint can't request size less than the length of a call.");
161  assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
162         "Invalid number of NOP bytes requested!");
163  emitNops((NumBytes - EncodedBytes) / NOPBytes);
164}
165
166void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
167                                      const MachineInstr &MI) {
168  unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
169
170  StatepointOpers SOpers(&MI);
171  if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
172    assert(PatchBytes % NOPBytes == 0 &&
173           "Invalid number of NOP bytes requested!");
174    emitNops(PatchBytes / NOPBytes);
175  }
176
177  auto &Ctx = OutStreamer.getContext();
178  MCSymbol *MILabel = Ctx.createTempSymbol();
179  OutStreamer.emitLabel(MILabel);
180  SM.recordStatepoint(*MILabel, MI);
181}
182
183void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
184  MCInst CInst;
185  bool Res = RISCVRVC::compress(CInst, Inst, *STI);
186  if (Res)
187    ++RISCVNumInstrsCompressed;
188  AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst);
189}
190
191// Simple pseudo-instructions have their lowering (with expansion to real
192// instructions) auto-generated.
193#include "RISCVGenMCPseudoLowering.inc"
194
195// If the target supports Zihintntl and the instruction has a nontemporal
196// MachineMemOperand, emit an NTLH hint instruction before it.
197void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
198  if (!STI->hasStdExtZihintntl())
199    return;
200
201  if (MI->memoperands_empty())
202    return;
203
204  MachineMemOperand *MMO = *(MI->memoperands_begin());
205  if (!MMO->isNonTemporal())
206    return;
207
208  unsigned NontemporalMode = 0;
209  if (MMO->getFlags() & MONontemporalBit0)
210    NontemporalMode += 0b1;
211  if (MMO->getFlags() & MONontemporalBit1)
212    NontemporalMode += 0b10;
213
214  MCInst Hint;
215  if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
216    Hint.setOpcode(RISCV::C_ADD_HINT);
217  else
218    Hint.setOpcode(RISCV::ADD);
219
220  Hint.addOperand(MCOperand::createReg(RISCV::X0));
221  Hint.addOperand(MCOperand::createReg(RISCV::X0));
222  Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode));
223
224  EmitToStreamer(*OutStreamer, Hint);
225}
226
227void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
228  RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
229                                        getSubtargetInfo().getFeatureBits());
230
231  emitNTLHint(MI);
232
233  // Do any auto-generated pseudo lowerings.
234  if (emitPseudoExpansionLowering(*OutStreamer, MI))
235    return;
236
237
238  switch (MI->getOpcode()) {
239  case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
240    LowerHWASAN_CHECK_MEMACCESS(*MI);
241    return;
242  case RISCV::KCFI_CHECK:
243    LowerKCFI_CHECK(*MI);
244    return;
245  case RISCV::PseudoRVVInitUndefM1:
246  case RISCV::PseudoRVVInitUndefM2:
247  case RISCV::PseudoRVVInitUndefM4:
248  case RISCV::PseudoRVVInitUndefM8:
249    return;
250  case TargetOpcode::STACKMAP:
251    return LowerSTACKMAP(*OutStreamer, SM, *MI);
252  case TargetOpcode::PATCHPOINT:
253    return LowerPATCHPOINT(*OutStreamer, SM, *MI);
254  case TargetOpcode::STATEPOINT:
255    return LowerSTATEPOINT(*OutStreamer, SM, *MI);
256  }
257
258  MCInst OutInst;
259  if (!lowerToMCInst(MI, OutInst))
260    EmitToStreamer(*OutStreamer, OutInst);
261}
262
263bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
264                                      const char *ExtraCode, raw_ostream &OS) {
265  // First try the generic code, which knows about modifiers like 'c' and 'n'.
266  if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS))
267    return false;
268
269  const MachineOperand &MO = MI->getOperand(OpNo);
270  if (ExtraCode && ExtraCode[0]) {
271    if (ExtraCode[1] != 0)
272      return true; // Unknown modifier.
273
274    switch (ExtraCode[0]) {
275    default:
276      return true; // Unknown modifier.
277    case 'z':      // Print zero register if zero, regular printing otherwise.
278      if (MO.isImm() && MO.getImm() == 0) {
279        OS << RISCVInstPrinter::getRegisterName(RISCV::X0);
280        return false;
281      }
282      break;
283    case 'i': // Literal 'i' if operand is not a register.
284      if (!MO.isReg())
285        OS << 'i';
286      return false;
287    }
288  }
289
290  switch (MO.getType()) {
291  case MachineOperand::MO_Immediate:
292    OS << MO.getImm();
293    return false;
294  case MachineOperand::MO_Register:
295    OS << RISCVInstPrinter::getRegisterName(MO.getReg());
296    return false;
297  case MachineOperand::MO_GlobalAddress:
298    PrintSymbolOperand(MO, OS);
299    return false;
300  case MachineOperand::MO_BlockAddress: {
301    MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
302    Sym->print(OS, MAI);
303    return false;
304  }
305  default:
306    break;
307  }
308
309  return true;
310}
311
312bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
313                                            unsigned OpNo,
314                                            const char *ExtraCode,
315                                            raw_ostream &OS) {
316  if (ExtraCode)
317    return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS);
318
319  const MachineOperand &AddrReg = MI->getOperand(OpNo);
320  assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand");
321  const MachineOperand &Offset = MI->getOperand(OpNo + 1);
322  // All memory operands should have a register and an immediate operand (see
323  // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand).
324  if (!AddrReg.isReg())
325    return true;
326  if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() &&
327      !Offset.isMCSymbol())
328    return true;
329
330  MCOperand MCO;
331  if (!lowerOperand(Offset, MCO))
332    return true;
333
334  if (Offset.isImm())
335    OS << MCO.getImm();
336  else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
337    OS << *MCO.getExpr();
338  OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")";
339  return false;
340}
341
342bool RISCVAsmPrinter::emitDirectiveOptionArch() {
343  RISCVTargetStreamer &RTS =
344      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
345  SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs;
346  const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
347  for (const auto &Feature : RISCVFeatureKV) {
348    if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value))
349      continue;
350
351    if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
352      continue;
353
354    auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus
355                                                : RISCVOptionArchArgType::Minus;
356    NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key);
357  }
358  if (!NeedEmitStdOptionArgs.empty()) {
359    RTS.emitDirectiveOptionPush();
360    RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs);
361    return true;
362  }
363
364  return false;
365}
366
367bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
368  STI = &MF.getSubtarget<RISCVSubtarget>();
369  RISCVTargetStreamer &RTS =
370      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
371
372  bool EmittedOptionArch = emitDirectiveOptionArch();
373
374  SetupMachineFunction(MF);
375  emitFunctionBody();
376
377  if (EmittedOptionArch)
378    RTS.emitDirectiveOptionPop();
379  return false;
380}
381
382void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
383  RISCVTargetStreamer &RTS =
384      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
385  if (const MDString *ModuleTargetABI =
386          dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi")))
387    RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString()));
388
389  MCSubtargetInfo SubtargetInfo = *TM.getMCSubtargetInfo();
390
391  // Use module flag to update feature bits.
392  if (auto *MD = dyn_cast_or_null<MDNode>(M.getModuleFlag("riscv-isa"))) {
393    for (auto &ISA : MD->operands()) {
394      if (auto *ISAString = dyn_cast_or_null<MDString>(ISA)) {
395        auto ParseResult = llvm::RISCVISAInfo::parseArchString(
396            ISAString->getString(), /*EnableExperimentalExtension=*/true,
397            /*ExperimentalExtensionVersionCheck=*/true);
398        if (!errorToBool(ParseResult.takeError())) {
399          auto &ISAInfo = *ParseResult;
400          for (const auto &Feature : RISCVFeatureKV) {
401            if (ISAInfo->hasExtension(Feature.Key) &&
402                !SubtargetInfo.hasFeature(Feature.Value))
403              SubtargetInfo.ToggleFeature(Feature.Key);
404          }
405        }
406      }
407    }
408
409    RTS.setFlagsFromFeatures(SubtargetInfo);
410  }
411
412  if (TM.getTargetTriple().isOSBinFormatELF())
413    emitAttributes(SubtargetInfo);
414}
415
416void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
417  RISCVTargetStreamer &RTS =
418      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
419
420  if (TM.getTargetTriple().isOSBinFormatELF())
421    RTS.finishAttributeSection();
422  EmitHwasanMemaccessSymbols(M);
423}
424
425void RISCVAsmPrinter::emitAttributes(const MCSubtargetInfo &SubtargetInfo) {
426  RISCVTargetStreamer &RTS =
427      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
428  // Use MCSubtargetInfo from TargetMachine. Individual functions may have
429  // attributes that differ from other functions in the module and we have no
430  // way to know which function is correct.
431  RTS.emitTargetAttributes(SubtargetInfo, /*EmitStackAlign*/ true);
432}
433
434void RISCVAsmPrinter::emitFunctionEntryLabel() {
435  const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>();
436  if (RMFI->isVectorCall()) {
437    auto &RTS =
438        static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
439    RTS.emitDirectiveVariantCC(*CurrentFnSym);
440  }
441  return AsmPrinter::emitFunctionEntryLabel();
442}
443
444// Force static initialization.
445extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
446  RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
447  RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
448}
449
450void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
451  Register Reg = MI.getOperand(0).getReg();
452  uint32_t AccessInfo = MI.getOperand(1).getImm();
453  MCSymbol *&Sym =
454      HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)];
455  if (!Sym) {
456    // FIXME: Make this work on non-ELF.
457    if (!TM.getTargetTriple().isOSBinFormatELF())
458      report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
459
460    std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" +
461                          utostr(AccessInfo) + "_short";
462    Sym = OutContext.getOrCreateSymbol(SymName);
463  }
464  auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext);
465  auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext);
466
467  EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
468}
469
470void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
471  Register AddrReg = MI.getOperand(0).getReg();
472  assert(std::next(MI.getIterator())->isCall() &&
473         "KCFI_CHECK not followed by a call instruction");
474  assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
475         "KCFI_CHECK call target doesn't match call operand");
476
477  // Temporary registers for comparing the hashes. If a register is used
478  // for the call target, or reserved by the user, we can clobber another
479  // temporary register as the check is immediately followed by the
480  // call. The check defaults to X6/X7, but can fall back to X28-X31 if
481  // needed.
482  unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
483  unsigned NextReg = RISCV::X28;
484  auto isRegAvailable = [&](unsigned Reg) {
485    return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg);
486  };
487  for (auto &Reg : ScratchRegs) {
488    if (isRegAvailable(Reg))
489      continue;
490    while (!isRegAvailable(NextReg))
491      ++NextReg;
492    Reg = NextReg++;
493    if (Reg > RISCV::X31)
494      report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
495  }
496
497  if (AddrReg == RISCV::X0) {
498    // Checking X0 makes no sense. Instead of emitting a load, zero
499    // ScratchRegs[0].
500    EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
501                                     .addReg(ScratchRegs[0])
502                                     .addReg(RISCV::X0)
503                                     .addImm(0));
504  } else {
505    // Adjust the offset for patchable-function-prefix. This assumes that
506    // patchable-function-prefix is the same for all functions.
507    int NopSize = STI->hasStdExtCOrZca() ? 2 : 4;
508    int64_t PrefixNops = 0;
509    (void)MI.getMF()
510        ->getFunction()
511        .getFnAttribute("patchable-function-prefix")
512        .getValueAsString()
513        .getAsInteger(10, PrefixNops);
514
515    // Load the target function type hash.
516    EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
517                                     .addReg(ScratchRegs[0])
518                                     .addReg(AddrReg)
519                                     .addImm(-(PrefixNops * NopSize + 4)));
520  }
521
522  // Load the expected 32-bit type hash.
523  const int64_t Type = MI.getOperand(1).getImm();
524  const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
525  const int64_t Lo12 = SignExtend64<12>(Type);
526  if (Hi20) {
527    EmitToStreamer(
528        *OutStreamer,
529        MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
530  }
531  if (Lo12 || Hi20 == 0) {
532    EmitToStreamer(*OutStreamer,
533                   MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
534                                     ? RISCV::ADDIW
535                                     : RISCV::ADDI)
536                       .addReg(ScratchRegs[1])
537                       .addReg(ScratchRegs[1])
538                       .addImm(Lo12));
539  }
540
541  // Compare the hashes and trap if there's a mismatch.
542  MCSymbol *Pass = OutContext.createTempSymbol();
543  EmitToStreamer(*OutStreamer,
544                 MCInstBuilder(RISCV::BEQ)
545                     .addReg(ScratchRegs[0])
546                     .addReg(ScratchRegs[1])
547                     .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
548
549  MCSymbol *Trap = OutContext.createTempSymbol();
550  OutStreamer->emitLabel(Trap);
551  EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
552  emitKCFITrapEntry(*MI.getMF(), Trap);
553  OutStreamer->emitLabel(Pass);
554}
555
556void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
557  if (HwasanMemaccessSymbols.empty())
558    return;
559
560  assert(TM.getTargetTriple().isOSBinFormatELF());
561  // Use MCSubtargetInfo from TargetMachine. Individual functions may have
562  // attributes that differ from other functions in the module and we have no
563  // way to know which function is correct.
564  const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
565
566  MCSymbol *HwasanTagMismatchV2Sym =
567      OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
568  // Annotate symbol as one having incompatible calling convention, so
569  // run-time linkers can instead eagerly bind this function.
570  auto &RTS =
571      static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
572  RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym);
573
574  const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
575      MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
576  auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref,
577                                  RISCVMCExpr::VK_RISCV_CALL, OutContext);
578
579  for (auto &P : HwasanMemaccessSymbols) {
580    unsigned Reg = std::get<0>(P.first);
581    uint32_t AccessInfo = std::get<1>(P.first);
582    MCSymbol *Sym = P.second;
583
584    unsigned Size =
585        1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
586    OutStreamer->switchSection(OutContext.getELFSection(
587        ".text.hot", ELF::SHT_PROGBITS,
588        ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
589        /*IsComdat=*/true));
590
591    OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
592    OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
593    OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
594    OutStreamer->emitLabel(Sym);
595
596    // Extract shadow offset from ptr
597    OutStreamer->emitInstruction(
598        MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8),
599        MCSTI);
600    OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI)
601                                     .addReg(RISCV::X6)
602                                     .addReg(RISCV::X6)
603                                     .addImm(12),
604                                 MCSTI);
605    // load shadow tag in X6, X5 contains shadow base
606    OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD)
607                                     .addReg(RISCV::X6)
608                                     .addReg(RISCV::X5)
609                                     .addReg(RISCV::X6),
610                                 MCSTI);
611    OutStreamer->emitInstruction(
612        MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
613        MCSTI);
614    // Extract tag from X5 and compare it with loaded tag from shadow
615    OutStreamer->emitInstruction(
616        MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56),
617        MCSTI);
618    MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
619    // X7 contains tag from memory, while X6 contains tag from the pointer
620    OutStreamer->emitInstruction(
621        MCInstBuilder(RISCV::BNE)
622            .addReg(RISCV::X7)
623            .addReg(RISCV::X6)
624            .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
625                                             OutContext)),
626        MCSTI);
627    MCSymbol *ReturnSym = OutContext.createTempSymbol();
628    OutStreamer->emitLabel(ReturnSym);
629    OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR)
630                                     .addReg(RISCV::X0)
631                                     .addReg(RISCV::X1)
632                                     .addImm(0),
633                                 MCSTI);
634    OutStreamer->emitLabel(HandleMismatchOrPartialSym);
635
636    OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
637                                     .addReg(RISCV::X28)
638                                     .addReg(RISCV::X0)
639                                     .addImm(16),
640                                 MCSTI);
641    MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
642    OutStreamer->emitInstruction(
643        MCInstBuilder(RISCV::BGEU)
644            .addReg(RISCV::X6)
645            .addReg(RISCV::X28)
646            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
647        MCSTI);
648
649    OutStreamer->emitInstruction(
650        MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF),
651        MCSTI);
652
653    if (Size != 1)
654      OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
655                                       .addReg(RISCV::X28)
656                                       .addReg(RISCV::X28)
657                                       .addImm(Size - 1),
658                                   MCSTI);
659    OutStreamer->emitInstruction(
660        MCInstBuilder(RISCV::BGE)
661            .addReg(RISCV::X28)
662            .addReg(RISCV::X6)
663            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
664        MCSTI);
665
666    OutStreamer->emitInstruction(
667        MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF),
668        MCSTI);
669    OutStreamer->emitInstruction(
670        MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
671        MCSTI);
672    OutStreamer->emitInstruction(
673        MCInstBuilder(RISCV::BEQ)
674            .addReg(RISCV::X6)
675            .addReg(RISCV::X7)
676            .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
677        MCSTI);
678
679    OutStreamer->emitLabel(HandleMismatchSym);
680
681    // | Previous stack frames...        |
682    // +=================================+ <-- [SP + 256]
683    // |              ...                |
684    // |                                 |
685    // | Stack frame space for x12 - x31.|
686    // |                                 |
687    // |              ...                |
688    // +---------------------------------+ <-- [SP + 96]
689    // | Saved x11(arg1), as             |
690    // | __hwasan_check_* clobbers it.   |
691    // +---------------------------------+ <-- [SP + 88]
692    // | Saved x10(arg0), as             |
693    // | __hwasan_check_* clobbers it.   |
694    // +---------------------------------+ <-- [SP + 80]
695    // |                                 |
696    // | Stack frame space for x9.       |
697    // +---------------------------------+ <-- [SP + 72]
698    // |                                 |
699    // | Saved x8(fp), as                |
700    // | __hwasan_check_* clobbers it.   |
701    // +---------------------------------+ <-- [SP + 64]
702    // |              ...                |
703    // |                                 |
704    // | Stack frame space for x2 - x7.  |
705    // |                                 |
706    // |              ...                |
707    // +---------------------------------+ <-- [SP + 16]
708    // | Return address (x1) for caller  |
709    // | of __hwasan_check_*.            |
710    // +---------------------------------+ <-- [SP + 8]
711    // | Reserved place for x0, possibly |
712    // | junk, since we don't save it.   |
713    // +---------------------------------+ <-- [x2 / SP]
714
715    // Adjust sp
716    OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
717                                     .addReg(RISCV::X2)
718                                     .addReg(RISCV::X2)
719                                     .addImm(-256),
720                                 MCSTI);
721
722    // store x10(arg0) by new sp
723    OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
724                                     .addReg(RISCV::X10)
725                                     .addReg(RISCV::X2)
726                                     .addImm(8 * 10),
727                                 MCSTI);
728    // store x11(arg1) by new sp
729    OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
730                                     .addReg(RISCV::X11)
731                                     .addReg(RISCV::X2)
732                                     .addImm(8 * 11),
733                                 MCSTI);
734
735    // store x8(fp) by new sp
736    OutStreamer->emitInstruction(
737        MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 *
738                                                                            8),
739        MCSTI);
740    // store x1(ra) by new sp
741    OutStreamer->emitInstruction(
742        MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 *
743                                                                            8),
744        MCSTI);
745    if (Reg != RISCV::X10)
746      OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
747                                       .addReg(RISCV::X10)
748                                       .addReg(Reg)
749                                       .addImm(0),
750                                   MCSTI);
751    OutStreamer->emitInstruction(
752        MCInstBuilder(RISCV::ADDI)
753            .addReg(RISCV::X11)
754            .addReg(RISCV::X0)
755            .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask),
756        MCSTI);
757
758    OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr),
759                                 MCSTI);
760  }
761}
762
763static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
764                                    const AsmPrinter &AP) {
765  MCContext &Ctx = AP.OutContext;
766  RISCVMCExpr::VariantKind Kind;
767
768  switch (MO.getTargetFlags()) {
769  default:
770    llvm_unreachable("Unknown target flag on GV operand");
771  case RISCVII::MO_None:
772    Kind = RISCVMCExpr::VK_RISCV_None;
773    break;
774  case RISCVII::MO_CALL:
775    Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
776    break;
777  case RISCVII::MO_LO:
778    Kind = RISCVMCExpr::VK_RISCV_LO;
779    break;
780  case RISCVII::MO_HI:
781    Kind = RISCVMCExpr::VK_RISCV_HI;
782    break;
783  case RISCVII::MO_PCREL_LO:
784    Kind = RISCVMCExpr::VK_RISCV_PCREL_LO;
785    break;
786  case RISCVII::MO_PCREL_HI:
787    Kind = RISCVMCExpr::VK_RISCV_PCREL_HI;
788    break;
789  case RISCVII::MO_GOT_HI:
790    Kind = RISCVMCExpr::VK_RISCV_GOT_HI;
791    break;
792  case RISCVII::MO_TPREL_LO:
793    Kind = RISCVMCExpr::VK_RISCV_TPREL_LO;
794    break;
795  case RISCVII::MO_TPREL_HI:
796    Kind = RISCVMCExpr::VK_RISCV_TPREL_HI;
797    break;
798  case RISCVII::MO_TPREL_ADD:
799    Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD;
800    break;
801  case RISCVII::MO_TLS_GOT_HI:
802    Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI;
803    break;
804  case RISCVII::MO_TLS_GD_HI:
805    Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI;
806    break;
807  case RISCVII::MO_TLSDESC_HI:
808    Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI;
809    break;
810  case RISCVII::MO_TLSDESC_LOAD_LO:
811    Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO;
812    break;
813  case RISCVII::MO_TLSDESC_ADD_LO:
814    Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO;
815    break;
816  case RISCVII::MO_TLSDESC_CALL:
817    Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL;
818    break;
819  }
820
821  const MCExpr *ME =
822      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
823
824  if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
825    ME = MCBinaryExpr::createAdd(
826        ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
827
828  if (Kind != RISCVMCExpr::VK_RISCV_None)
829    ME = RISCVMCExpr::create(ME, Kind, Ctx);
830  return MCOperand::createExpr(ME);
831}
832
833bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
834                                   MCOperand &MCOp) const {
835  switch (MO.getType()) {
836  default:
837    report_fatal_error("lowerOperand: unknown operand type");
838  case MachineOperand::MO_Register:
839    // Ignore all implicit register operands.
840    if (MO.isImplicit())
841      return false;
842    MCOp = MCOperand::createReg(MO.getReg());
843    break;
844  case MachineOperand::MO_RegisterMask:
845    // Regmasks are like implicit defs.
846    return false;
847  case MachineOperand::MO_Immediate:
848    MCOp = MCOperand::createImm(MO.getImm());
849    break;
850  case MachineOperand::MO_MachineBasicBlock:
851    MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this);
852    break;
853  case MachineOperand::MO_GlobalAddress:
854    MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this);
855    break;
856  case MachineOperand::MO_BlockAddress:
857    MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()),
858                              *this);
859    break;
860  case MachineOperand::MO_ExternalSymbol:
861    MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()),
862                              *this);
863    break;
864  case MachineOperand::MO_ConstantPoolIndex:
865    MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this);
866    break;
867  case MachineOperand::MO_JumpTableIndex:
868    MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this);
869    break;
870  case MachineOperand::MO_MCSymbol:
871    MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this);
872    break;
873  }
874  return true;
875}
876
877static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
878                                            MCInst &OutMI) {
879  const RISCVVPseudosTable::PseudoInfo *RVV =
880      RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
881  if (!RVV)
882    return false;
883
884  OutMI.setOpcode(RVV->BaseInstr);
885
886  const MachineBasicBlock *MBB = MI->getParent();
887  assert(MBB && "MI expected to be in a basic block");
888  const MachineFunction *MF = MBB->getParent();
889  assert(MF && "MBB expected to be in a machine function");
890
891  const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
892  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
893  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
894  assert(TRI && "TargetRegisterInfo expected");
895
896  const MCInstrDesc &MCID = MI->getDesc();
897  uint64_t TSFlags = MCID.TSFlags;
898  unsigned NumOps = MI->getNumExplicitOperands();
899
900  // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if
901  // present.
902  if (RISCVII::hasVecPolicyOp(TSFlags))
903    --NumOps;
904  if (RISCVII::hasSEWOp(TSFlags))
905    --NumOps;
906  if (RISCVII::hasVLOp(TSFlags))
907    --NumOps;
908  if (RISCVII::hasRoundModeOp(TSFlags))
909    --NumOps;
910
911  bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
912  for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
913    const MachineOperand &MO = MI->getOperand(OpNo);
914    // Skip vl ouput. It should be the second output.
915    if (hasVLOutput && OpNo == 1)
916      continue;
917
918    // Skip merge op. It should be the first operand after the defs.
919    if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) {
920      assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 &&
921             "Expected tied to first def.");
922      const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
923      // Skip if the next operand in OutMI is not supposed to be tied. Unless it
924      // is a _TIED instruction.
925      if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) <
926              0 &&
927          !RISCVII::isTiedPseudo(TSFlags))
928        continue;
929    }
930
931    MCOperand MCOp;
932    switch (MO.getType()) {
933    default:
934      llvm_unreachable("Unknown operand type");
935    case MachineOperand::MO_Register: {
936      Register Reg = MO.getReg();
937
938      if (RISCV::VRM2RegClass.contains(Reg) ||
939          RISCV::VRM4RegClass.contains(Reg) ||
940          RISCV::VRM8RegClass.contains(Reg)) {
941        Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
942        assert(Reg && "Subregister does not exist");
943      } else if (RISCV::FPR16RegClass.contains(Reg)) {
944        Reg =
945            TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass);
946        assert(Reg && "Subregister does not exist");
947      } else if (RISCV::FPR64RegClass.contains(Reg)) {
948        Reg = TRI->getSubReg(Reg, RISCV::sub_32);
949        assert(Reg && "Superregister does not exist");
950      } else if (RISCV::VRN2M1RegClass.contains(Reg) ||
951                 RISCV::VRN2M2RegClass.contains(Reg) ||
952                 RISCV::VRN2M4RegClass.contains(Reg) ||
953                 RISCV::VRN3M1RegClass.contains(Reg) ||
954                 RISCV::VRN3M2RegClass.contains(Reg) ||
955                 RISCV::VRN4M1RegClass.contains(Reg) ||
956                 RISCV::VRN4M2RegClass.contains(Reg) ||
957                 RISCV::VRN5M1RegClass.contains(Reg) ||
958                 RISCV::VRN6M1RegClass.contains(Reg) ||
959                 RISCV::VRN7M1RegClass.contains(Reg) ||
960                 RISCV::VRN8M1RegClass.contains(Reg)) {
961        Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
962        assert(Reg && "Subregister does not exist");
963      }
964
965      MCOp = MCOperand::createReg(Reg);
966      break;
967    }
968    case MachineOperand::MO_Immediate:
969      MCOp = MCOperand::createImm(MO.getImm());
970      break;
971    }
972    OutMI.addOperand(MCOp);
973  }
974
975  // Unmasked pseudo instructions need to append dummy mask operand to
976  // V instructions. All V instructions are modeled as the masked version.
977  const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
978  if (OutMI.getNumOperands() < OutMCID.getNumOperands()) {
979    assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass ==
980               RISCV::VMV0RegClassID &&
981           "Expected only mask operand to be missing");
982    OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
983  }
984
985  assert(OutMI.getNumOperands() == OutMCID.getNumOperands());
986  return true;
987}
988
989bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
990  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
991    return false;
992
993  OutMI.setOpcode(MI->getOpcode());
994
995  for (const MachineOperand &MO : MI->operands()) {
996    MCOperand MCOp;
997    if (lowerOperand(MO, MCOp))
998      OutMI.addOperand(MCOp);
999  }
1000
1001  switch (OutMI.getOpcode()) {
1002  case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
1003    const Function &F = MI->getParent()->getParent()->getFunction();
1004    if (F.hasFnAttribute("patchable-function-entry")) {
1005      unsigned Num;
1006      if (F.getFnAttribute("patchable-function-entry")
1007              .getValueAsString()
1008              .getAsInteger(10, Num))
1009        return false;
1010      emitNops(Num);
1011      return true;
1012    }
1013    break;
1014  }
1015  }
1016  return false;
1017}
1018