1//== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9//  This file defines the HTMLRewriter class, which is used to translate the
10//  text of a source file into prettified HTML.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Rewrite/Core/HTMLRewrite.h"
15#include "clang/Basic/SourceManager.h"
16#include "clang/Lex/Preprocessor.h"
17#include "clang/Lex/TokenConcatenation.h"
18#include "clang/Rewrite/Core/Rewriter.h"
19#include "llvm/ADT/SmallString.h"
20#include "llvm/Support/ErrorHandling.h"
21#include "llvm/Support/MemoryBuffer.h"
22#include "llvm/Support/raw_ostream.h"
23#include <memory>
24using namespace clang;
25
26
27/// HighlightRange - Highlight a range in the source code with the specified
28/// start/end tags.  B/E must be in the same file.  This ensures that
29/// start/end tags are placed at the start/end of each line if the range is
30/// multiline.
31void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
32                          const char *StartTag, const char *EndTag,
33                          bool IsTokenRange) {
34  SourceManager &SM = R.getSourceMgr();
35  B = SM.getExpansionLoc(B);
36  E = SM.getExpansionLoc(E);
37  FileID FID = SM.getFileID(B);
38  assert(SM.getFileID(E) == FID && "B/E not in the same file!");
39
40  unsigned BOffset = SM.getFileOffset(B);
41  unsigned EOffset = SM.getFileOffset(E);
42
43  // Include the whole end token in the range.
44  if (IsTokenRange)
45    EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
46
47  bool Invalid = false;
48  const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
49  if (Invalid)
50    return;
51
52  HighlightRange(R.getEditBuffer(FID), BOffset, EOffset,
53                 BufferStart, StartTag, EndTag);
54}
55
56/// HighlightRange - This is the same as the above method, but takes
57/// decomposed file locations.
58void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
59                          const char *BufferStart,
60                          const char *StartTag, const char *EndTag) {
61  // Insert the tag at the absolute start/end of the range.
62  RB.InsertTextAfter(B, StartTag);
63  RB.InsertTextBefore(E, EndTag);
64
65  // Scan the range to see if there is a \r or \n.  If so, and if the line is
66  // not blank, insert tags on that line as well.
67  bool HadOpenTag = true;
68
69  unsigned LastNonWhiteSpace = B;
70  for (unsigned i = B; i != E; ++i) {
71    switch (BufferStart[i]) {
72    case '\r':
73    case '\n':
74      // Okay, we found a newline in the range.  If we have an open tag, we need
75      // to insert a close tag at the first non-whitespace before the newline.
76      if (HadOpenTag)
77        RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag);
78
79      // Instead of inserting an open tag immediately after the newline, we
80      // wait until we see a non-whitespace character.  This prevents us from
81      // inserting tags around blank lines, and also allows the open tag to
82      // be put *after* whitespace on a non-blank line.
83      HadOpenTag = false;
84      break;
85    case '\0':
86    case ' ':
87    case '\t':
88    case '\f':
89    case '\v':
90      // Ignore whitespace.
91      break;
92
93    default:
94      // If there is no tag open, do it now.
95      if (!HadOpenTag) {
96        RB.InsertTextAfter(i, StartTag);
97        HadOpenTag = true;
98      }
99
100      // Remember this character.
101      LastNonWhiteSpace = i;
102      break;
103    }
104  }
105}
106
107void html::EscapeText(Rewriter &R, FileID FID,
108                      bool EscapeSpaces, bool ReplaceTabs) {
109
110  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
111  const char* C = Buf->getBufferStart();
112  const char* FileEnd = Buf->getBufferEnd();
113
114  assert (C <= FileEnd);
115
116  RewriteBuffer &RB = R.getEditBuffer(FID);
117
118  unsigned ColNo = 0;
119  for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) {
120    switch (*C) {
121    default: ++ColNo; break;
122    case '\n':
123    case '\r':
124      ColNo = 0;
125      break;
126
127    case ' ':
128      if (EscapeSpaces)
129        RB.ReplaceText(FilePos, 1, "&nbsp;");
130      ++ColNo;
131      break;
132    case '\f':
133      RB.ReplaceText(FilePos, 1, "<hr>");
134      ColNo = 0;
135      break;
136
137    case '\t': {
138      if (!ReplaceTabs)
139        break;
140      unsigned NumSpaces = 8-(ColNo&7);
141      if (EscapeSpaces)
142        RB.ReplaceText(FilePos, 1,
143                       StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
144                                       "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
145      else
146        RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
147      ColNo += NumSpaces;
148      break;
149    }
150    case '<':
151      RB.ReplaceText(FilePos, 1, "&lt;");
152      ++ColNo;
153      break;
154
155    case '>':
156      RB.ReplaceText(FilePos, 1, "&gt;");
157      ++ColNo;
158      break;
159
160    case '&':
161      RB.ReplaceText(FilePos, 1, "&amp;");
162      ++ColNo;
163      break;
164    }
165  }
166}
167
168std::string html::EscapeText(StringRef s, bool EscapeSpaces, bool ReplaceTabs) {
169
170  unsigned len = s.size();
171  std::string Str;
172  llvm::raw_string_ostream os(Str);
173
174  for (unsigned i = 0 ; i < len; ++i) {
175
176    char c = s[i];
177    switch (c) {
178    default:
179      os << c; break;
180
181    case ' ':
182      if (EscapeSpaces) os << "&nbsp;";
183      else os << ' ';
184      break;
185
186    case '\t':
187      if (ReplaceTabs) {
188        if (EscapeSpaces)
189          for (unsigned i = 0; i < 4; ++i)
190            os << "&nbsp;";
191        else
192          for (unsigned i = 0; i < 4; ++i)
193            os << " ";
194      }
195      else
196        os << c;
197
198      break;
199
200    case '<': os << "&lt;"; break;
201    case '>': os << "&gt;"; break;
202    case '&': os << "&amp;"; break;
203    }
204  }
205
206  return os.str();
207}
208
209static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
210                          unsigned B, unsigned E) {
211  SmallString<256> Str;
212  llvm::raw_svector_ostream OS(Str);
213
214  OS << "<tr class=\"codeline\" data-linenumber=\"" << LineNo << "\">"
215     << "<td class=\"num\" id=\"LN" << LineNo << "\">" << LineNo
216     << "</td><td class=\"line\">";
217
218  if (B == E) { // Handle empty lines.
219    OS << " </td></tr>";
220    RB.InsertTextBefore(B, OS.str());
221  } else {
222    RB.InsertTextBefore(B, OS.str());
223    RB.InsertTextBefore(E, "</td></tr>");
224  }
225}
226
227void html::AddLineNumbers(Rewriter& R, FileID FID) {
228
229  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
230  const char* FileBeg = Buf->getBufferStart();
231  const char* FileEnd = Buf->getBufferEnd();
232  const char* C = FileBeg;
233  RewriteBuffer &RB = R.getEditBuffer(FID);
234
235  assert (C <= FileEnd);
236
237  unsigned LineNo = 0;
238  unsigned FilePos = 0;
239
240  while (C != FileEnd) {
241
242    ++LineNo;
243    unsigned LineStartPos = FilePos;
244    unsigned LineEndPos = FileEnd - FileBeg;
245
246    assert (FilePos <= LineEndPos);
247    assert (C < FileEnd);
248
249    // Scan until the newline (or end-of-file).
250
251    while (C != FileEnd) {
252      char c = *C;
253      ++C;
254
255      if (c == '\n') {
256        LineEndPos = FilePos++;
257        break;
258      }
259
260      ++FilePos;
261    }
262
263    AddLineNumber(RB, LineNo, LineStartPos, LineEndPos);
264  }
265
266  // Add one big table tag that surrounds all of the code.
267  std::string s;
268  llvm::raw_string_ostream os(s);
269  os << "<table class=\"code\" data-fileid=\"" << FID.getHashValue() << "\">\n";
270  RB.InsertTextBefore(0, os.str());
271  RB.InsertTextAfter(FileEnd - FileBeg, "</table>");
272}
273
274void html::AddHeaderFooterInternalBuiltinCSS(Rewriter &R, FileID FID,
275                                             StringRef title) {
276
277  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
278  const char* FileStart = Buf->getBufferStart();
279  const char* FileEnd = Buf->getBufferEnd();
280
281  SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
282  SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
283
284  std::string s;
285  llvm::raw_string_ostream os(s);
286  os << "<!doctype html>\n" // Use HTML 5 doctype
287        "<html>\n<head>\n";
288
289  if (!title.empty())
290    os << "<title>" << html::EscapeText(title) << "</title>\n";
291
292  os << R"<<<(
293<style type="text/css">
294body { color:#000000; background-color:#ffffff }
295body { font-family:Helvetica, sans-serif; font-size:10pt }
296h1 { font-size:14pt }
297.FileName { margin-top: 5px; margin-bottom: 5px; display: inline; }
298.FileNav { margin-left: 5px; margin-right: 5px; display: inline; }
299.FileNav a { text-decoration:none; font-size: larger; }
300.divider { margin-top: 30px; margin-bottom: 30px; height: 15px; }
301.divider { background-color: gray; }
302.code { border-collapse:collapse; width:100%; }
303.code { font-family: "Monospace", monospace; font-size:10pt }
304.code { line-height: 1.2em }
305.comment { color: green; font-style: oblique }
306.keyword { color: blue }
307.string_literal { color: red }
308.directive { color: darkmagenta }
309
310/* Macros and variables could have pop-up notes hidden by default.
311  - Macro pop-up:    expansion of the macro
312  - Variable pop-up: value (table) of the variable */
313.macro_popup, .variable_popup { display: none; }
314
315/* Pop-up appears on mouse-hover event. */
316.macro:hover .macro_popup, .variable:hover .variable_popup {
317  display: block;
318  padding: 2px;
319  -webkit-border-radius:5px;
320  -webkit-box-shadow:1px 1px 7px #000;
321  border-radius:5px;
322  box-shadow:1px 1px 7px #000;
323  position: absolute;
324  top: -1em;
325  left:10em;
326  z-index: 1
327}
328
329.macro_popup {
330  border: 2px solid red;
331  background-color:#FFF0F0;
332  font-weight: normal;
333}
334
335.variable_popup {
336  border: 2px solid blue;
337  background-color:#F0F0FF;
338  font-weight: bold;
339  font-family: Helvetica, sans-serif;
340  font-size: 9pt;
341}
342
343/* Pop-up notes needs a relative position as a base where they pops up. */
344.macro, .variable {
345  background-color: PaleGoldenRod;
346  position: relative;
347}
348.macro { color: DarkMagenta; }
349
350#tooltiphint {
351  position: fixed;
352  width: 50em;
353  margin-left: -25em;
354  left: 50%;
355  padding: 10px;
356  border: 1px solid #b0b0b0;
357  border-radius: 2px;
358  box-shadow: 1px 1px 7px black;
359  background-color: #c0c0c0;
360  z-index: 2;
361}
362
363.num { width:2.5em; padding-right:2ex; background-color:#eeeeee }
364.num { text-align:right; font-size:8pt }
365.num { color:#444444 }
366.line { padding-left: 1ex; border-left: 3px solid #ccc }
367.line { white-space: pre }
368.msg { -webkit-box-shadow:1px 1px 7px #000 }
369.msg { box-shadow:1px 1px 7px #000 }
370.msg { -webkit-border-radius:5px }
371.msg { border-radius:5px }
372.msg { font-family:Helvetica, sans-serif; font-size:8pt }
373.msg { float:left }
374.msg { padding:0.25em 1ex 0.25em 1ex }
375.msg { margin-top:10px; margin-bottom:10px }
376.msg { font-weight:bold }
377.msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }
378.msgT { padding:0x; spacing:0x }
379.msgEvent { background-color:#fff8b4; color:#000000 }
380.msgControl { background-color:#bbbbbb; color:#000000 }
381.msgNote { background-color:#ddeeff; color:#000000 }
382.mrange { background-color:#dfddf3 }
383.mrange { border-bottom:1px solid #6F9DBE }
384.PathIndex { font-weight: bold; padding:0px 5px; margin-right:5px; }
385.PathIndex { -webkit-border-radius:8px }
386.PathIndex { border-radius:8px }
387.PathIndexEvent { background-color:#bfba87 }
388.PathIndexControl { background-color:#8c8c8c }
389.PathIndexPopUp { background-color: #879abc; }
390.PathNav a { text-decoration:none; font-size: larger }
391.CodeInsertionHint { font-weight: bold; background-color: #10dd10 }
392.CodeRemovalHint { background-color:#de1010 }
393.CodeRemovalHint { border-bottom:1px solid #6F9DBE }
394.selected{ background-color:orange !important; }
395
396table.simpletable {
397  padding: 5px;
398  font-size:12pt;
399  margin:20px;
400  border-collapse: collapse; border-spacing: 0px;
401}
402td.rowname {
403  text-align: right;
404  vertical-align: top;
405  font-weight: bold;
406  color:#444444;
407  padding-right:2ex;
408}
409
410/* Hidden text. */
411input.spoilerhider + label {
412  cursor: pointer;
413  text-decoration: underline;
414  display: block;
415}
416input.spoilerhider {
417 display: none;
418}
419input.spoilerhider ~ .spoiler {
420  overflow: hidden;
421  margin: 10px auto 0;
422  height: 0;
423  opacity: 0;
424}
425input.spoilerhider:checked + label + .spoiler{
426  height: auto;
427  opacity: 1;
428}
429</style>
430</head>
431<body>)<<<";
432
433  // Generate header
434  R.InsertTextBefore(StartLoc, os.str());
435  // Generate footer
436
437  R.InsertTextAfter(EndLoc, "</body></html>\n");
438}
439
440/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
441/// information about keywords, macro expansions etc.  This uses the macro
442/// table state from the end of the file, so it won't be perfectly perfect,
443/// but it will be reasonably close.
444void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
445  RewriteBuffer &RB = R.getEditBuffer(FID);
446
447  const SourceManager &SM = PP.getSourceManager();
448  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
449  Lexer L(FID, FromFile, SM, PP.getLangOpts());
450  const char *BufferStart = L.getBuffer().data();
451
452  // Inform the preprocessor that we want to retain comments as tokens, so we
453  // can highlight them.
454  L.SetCommentRetentionState(true);
455
456  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
457  // macros.
458  Token Tok;
459  L.LexFromRawLexer(Tok);
460
461  while (Tok.isNot(tok::eof)) {
462    // Since we are lexing unexpanded tokens, all tokens are from the main
463    // FileID.
464    unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
465    unsigned TokLen = Tok.getLength();
466    switch (Tok.getKind()) {
467    default: break;
468    case tok::identifier:
469      llvm_unreachable("tok::identifier in raw lexing mode!");
470    case tok::raw_identifier: {
471      // Fill in Result.IdentifierInfo and update the token kind,
472      // looking up the identifier in the identifier table.
473      PP.LookUpIdentifierInfo(Tok);
474
475      // If this is a pp-identifier, for a keyword, highlight it as such.
476      if (Tok.isNot(tok::identifier))
477        HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
478                       "<span class='keyword'>", "</span>");
479      break;
480    }
481    case tok::comment:
482      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
483                     "<span class='comment'>", "</span>");
484      break;
485    case tok::utf8_string_literal:
486      // Chop off the u part of u8 prefix
487      ++TokOffs;
488      --TokLen;
489      // FALL THROUGH to chop the 8
490      LLVM_FALLTHROUGH;
491    case tok::wide_string_literal:
492    case tok::utf16_string_literal:
493    case tok::utf32_string_literal:
494      // Chop off the L, u, U or 8 prefix
495      ++TokOffs;
496      --TokLen;
497      LLVM_FALLTHROUGH;
498    case tok::string_literal:
499      // FIXME: Exclude the optional ud-suffix from the highlighted range.
500      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
501                     "<span class='string_literal'>", "</span>");
502      break;
503    case tok::hash: {
504      // If this is a preprocessor directive, all tokens to end of line are too.
505      if (!Tok.isAtStartOfLine())
506        break;
507
508      // Eat all of the tokens until we get to the next one at the start of
509      // line.
510      unsigned TokEnd = TokOffs+TokLen;
511      L.LexFromRawLexer(Tok);
512      while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
513        TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
514        L.LexFromRawLexer(Tok);
515      }
516
517      // Find end of line.  This is a hack.
518      HighlightRange(RB, TokOffs, TokEnd, BufferStart,
519                     "<span class='directive'>", "</span>");
520
521      // Don't skip the next token.
522      continue;
523    }
524    }
525
526    L.LexFromRawLexer(Tok);
527  }
528}
529
530/// HighlightMacros - This uses the macro table state from the end of the
531/// file, to re-expand macros and insert (into the HTML) information about the
532/// macro expansions.  This won't be perfectly perfect, but it will be
533/// reasonably close.
534void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
535  // Re-lex the raw token stream into a token buffer.
536  const SourceManager &SM = PP.getSourceManager();
537  std::vector<Token> TokenStream;
538
539  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
540  Lexer L(FID, FromFile, SM, PP.getLangOpts());
541
542  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
543  // macros.
544  while (1) {
545    Token Tok;
546    L.LexFromRawLexer(Tok);
547
548    // If this is a # at the start of a line, discard it from the token stream.
549    // We don't want the re-preprocess step to see #defines, #includes or other
550    // preprocessor directives.
551    if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
552      continue;
553
554    // If this is a ## token, change its kind to unknown so that repreprocessing
555    // it will not produce an error.
556    if (Tok.is(tok::hashhash))
557      Tok.setKind(tok::unknown);
558
559    // If this raw token is an identifier, the raw lexer won't have looked up
560    // the corresponding identifier info for it.  Do this now so that it will be
561    // macro expanded when we re-preprocess it.
562    if (Tok.is(tok::raw_identifier))
563      PP.LookUpIdentifierInfo(Tok);
564
565    TokenStream.push_back(Tok);
566
567    if (Tok.is(tok::eof)) break;
568  }
569
570  // Temporarily change the diagnostics object so that we ignore any generated
571  // diagnostics from this pass.
572  DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
573                             &PP.getDiagnostics().getDiagnosticOptions(),
574                      new IgnoringDiagConsumer);
575
576  // FIXME: This is a huge hack; we reuse the input preprocessor because we want
577  // its state, but we aren't actually changing it (we hope). This should really
578  // construct a copy of the preprocessor.
579  Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
580  DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
581  TmpPP.setDiagnostics(TmpDiags);
582
583  // Inform the preprocessor that we don't want comments.
584  TmpPP.SetCommentRetentionState(false, false);
585
586  // We don't want pragmas either. Although we filtered out #pragma, removing
587  // _Pragma and __pragma is much harder.
588  bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
589  TmpPP.setPragmasEnabled(false);
590
591  // Enter the tokens we just lexed.  This will cause them to be macro expanded
592  // but won't enter sub-files (because we removed #'s).
593  TmpPP.EnterTokenStream(TokenStream, false, /*IsReinject=*/false);
594
595  TokenConcatenation ConcatInfo(TmpPP);
596
597  // Lex all the tokens.
598  Token Tok;
599  TmpPP.Lex(Tok);
600  while (Tok.isNot(tok::eof)) {
601    // Ignore non-macro tokens.
602    if (!Tok.getLocation().isMacroID()) {
603      TmpPP.Lex(Tok);
604      continue;
605    }
606
607    // Okay, we have the first token of a macro expansion: highlight the
608    // expansion by inserting a start tag before the macro expansion and
609    // end tag after it.
610    CharSourceRange LLoc = SM.getExpansionRange(Tok.getLocation());
611
612    // Ignore tokens whose instantiation location was not the main file.
613    if (SM.getFileID(LLoc.getBegin()) != FID) {
614      TmpPP.Lex(Tok);
615      continue;
616    }
617
618    assert(SM.getFileID(LLoc.getEnd()) == FID &&
619           "Start and end of expansion must be in the same ultimate file!");
620
621    std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
622    unsigned LineLen = Expansion.size();
623
624    Token PrevPrevTok;
625    Token PrevTok = Tok;
626    // Okay, eat this token, getting the next one.
627    TmpPP.Lex(Tok);
628
629    // Skip all the rest of the tokens that are part of this macro
630    // instantiation.  It would be really nice to pop up a window with all the
631    // spelling of the tokens or something.
632    while (!Tok.is(tok::eof) &&
633           SM.getExpansionLoc(Tok.getLocation()) == LLoc.getBegin()) {
634      // Insert a newline if the macro expansion is getting large.
635      if (LineLen > 60) {
636        Expansion += "<br>";
637        LineLen = 0;
638      }
639
640      LineLen -= Expansion.size();
641
642      // If the tokens were already space separated, or if they must be to avoid
643      // them being implicitly pasted, add a space between them.
644      if (Tok.hasLeadingSpace() ||
645          ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
646        Expansion += ' ';
647
648      // Escape any special characters in the token text.
649      Expansion += EscapeText(TmpPP.getSpelling(Tok));
650      LineLen += Expansion.size();
651
652      PrevPrevTok = PrevTok;
653      PrevTok = Tok;
654      TmpPP.Lex(Tok);
655    }
656
657    // Insert the 'macro_popup' as the end tag, so that multi-line macros all
658    // get highlighted.
659    Expansion = "<span class='macro_popup'>" + Expansion + "</span></span>";
660
661    HighlightRange(R, LLoc.getBegin(), LLoc.getEnd(), "<span class='macro'>",
662                   Expansion.c_str(), LLoc.isTokenRange());
663  }
664
665  // Restore the preprocessor's old state.
666  TmpPP.setDiagnostics(*OldDiags);
667  TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
668}
669