TokenAnalyzer.cpp revision 303233
1//===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9/// 10/// \file 11/// \brief This file implements an abstract TokenAnalyzer and associated helper 12/// classes. TokenAnalyzer can be extended to generate replacements based on 13/// an annotated and pre-processed token stream. 14/// 15//===----------------------------------------------------------------------===// 16 17#include "TokenAnalyzer.h" 18#include "AffectedRangeManager.h" 19#include "Encoding.h" 20#include "FormatToken.h" 21#include "FormatTokenLexer.h" 22#include "TokenAnnotator.h" 23#include "UnwrappedLineParser.h" 24#include "clang/Basic/Diagnostic.h" 25#include "clang/Basic/DiagnosticOptions.h" 26#include "clang/Basic/FileManager.h" 27#include "clang/Basic/SourceManager.h" 28#include "clang/Format/Format.h" 29#include "llvm/ADT/STLExtras.h" 30#include "llvm/Support/Debug.h" 31 32#define DEBUG_TYPE "format-formatter" 33 34namespace clang { 35namespace format { 36 37// This sets up an virtual file system with file \p FileName containing \p 38// Code. 39std::unique_ptr<Environment> 40Environment::CreateVirtualEnvironment(StringRef Code, StringRef FileName, 41 ArrayRef<tooling::Range> Ranges) { 42 // This is referenced by `FileMgr` and will be released by `FileMgr` when it 43 // is deleted. 44 IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem( 45 new vfs::InMemoryFileSystem); 46 // This is passed to `SM` as reference, so the pointer has to be referenced 47 // in `Environment` so that `FileMgr` can out-live this function scope. 48 std::unique_ptr<FileManager> FileMgr( 49 new FileManager(FileSystemOptions(), InMemoryFileSystem)); 50 // This is passed to `SM` as reference, so the pointer has to be referenced 51 // by `Environment` due to the same reason above. 52 std::unique_ptr<DiagnosticsEngine> Diagnostics(new DiagnosticsEngine( 53 IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs), 54 new DiagnosticOptions)); 55 // This will be stored as reference, so the pointer has to be stored in 56 // due to the same reason above. 57 std::unique_ptr<SourceManager> VirtualSM( 58 new SourceManager(*Diagnostics, *FileMgr)); 59 InMemoryFileSystem->addFile( 60 FileName, 0, llvm::MemoryBuffer::getMemBuffer( 61 Code, FileName, /*RequiresNullTerminator=*/false)); 62 FileID ID = VirtualSM->createFileID(FileMgr->getFile(FileName), 63 SourceLocation(), clang::SrcMgr::C_User); 64 assert(ID.isValid()); 65 SourceLocation StartOfFile = VirtualSM->getLocForStartOfFile(ID); 66 std::vector<CharSourceRange> CharRanges; 67 for (const tooling::Range &Range : Ranges) { 68 SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset()); 69 SourceLocation End = Start.getLocWithOffset(Range.getLength()); 70 CharRanges.push_back(CharSourceRange::getCharRange(Start, End)); 71 } 72 return llvm::make_unique<Environment>(ID, std::move(FileMgr), 73 std::move(VirtualSM), 74 std::move(Diagnostics), CharRanges); 75} 76 77TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style) 78 : Style(Style), Env(Env), 79 AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()), 80 UnwrappedLines(1), 81 Encoding(encoding::detectEncoding( 82 Env.getSourceManager().getBufferData(Env.getFileID()))) { 83 DEBUG( 84 llvm::dbgs() << "File encoding: " 85 << (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown") 86 << "\n"); 87 DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language) 88 << "\n"); 89} 90 91tooling::Replacements TokenAnalyzer::process() { 92 tooling::Replacements Result; 93 FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(), Style, 94 Encoding); 95 96 UnwrappedLineParser Parser(Style, Tokens.getKeywords(), Tokens.lex(), *this); 97 Parser.parse(); 98 assert(UnwrappedLines.rbegin()->empty()); 99 for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) { 100 DEBUG(llvm::dbgs() << "Run " << Run << "...\n"); 101 SmallVector<AnnotatedLine *, 16> AnnotatedLines; 102 103 TokenAnnotator Annotator(Style, Tokens.getKeywords()); 104 for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) { 105 AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i])); 106 Annotator.annotate(*AnnotatedLines.back()); 107 } 108 109 tooling::Replacements RunResult = 110 analyze(Annotator, AnnotatedLines, Tokens, Result); 111 112 DEBUG({ 113 llvm::dbgs() << "Replacements for run " << Run << ":\n"; 114 for (tooling::Replacements::iterator I = RunResult.begin(), 115 E = RunResult.end(); 116 I != E; ++I) { 117 llvm::dbgs() << I->toString() << "\n"; 118 } 119 }); 120 for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) { 121 delete AnnotatedLines[i]; 122 } 123 Result.insert(RunResult.begin(), RunResult.end()); 124 } 125 return Result; 126} 127 128void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) { 129 assert(!UnwrappedLines.empty()); 130 UnwrappedLines.back().push_back(TheLine); 131} 132 133void TokenAnalyzer::finishRun() { 134 UnwrappedLines.push_back(SmallVector<UnwrappedLine, 16>()); 135} 136 137} // end namespace format 138} // end namespace clang 139