1//===--- JSONCompilationDatabase.cpp - ------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file contains the implementation of the JSONCompilationDatabase.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Tooling/JSONCompilationDatabase.h"
15#include "clang/Tooling/CompilationDatabase.h"
16#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
17#include "clang/Tooling/Tooling.h"
18#include "llvm/ADT/SmallString.h"
19#include "llvm/Support/Path.h"
20#include <system_error>
21
22namespace clang {
23namespace tooling {
24
25namespace {
26
27/// \brief A parser for escaped strings of command line arguments.
28///
29/// Assumes \-escaping for quoted arguments (see the documentation of
30/// unescapeCommandLine(...)).
31class CommandLineArgumentParser {
32 public:
33  CommandLineArgumentParser(StringRef CommandLine)
34      : Input(CommandLine), Position(Input.begin()-1) {}
35
36  std::vector<std::string> parse() {
37    bool HasMoreInput = true;
38    while (HasMoreInput && nextNonWhitespace()) {
39      std::string Argument;
40      HasMoreInput = parseStringInto(Argument);
41      CommandLine.push_back(Argument);
42    }
43    return CommandLine;
44  }
45
46 private:
47  // All private methods return true if there is more input available.
48
49  bool parseStringInto(std::string &String) {
50    do {
51      if (*Position == '"') {
52        if (!parseDoubleQuotedStringInto(String)) return false;
53      } else if (*Position == '\'') {
54        if (!parseSingleQuotedStringInto(String)) return false;
55      } else {
56        if (!parseFreeStringInto(String)) return false;
57      }
58    } while (*Position != ' ');
59    return true;
60  }
61
62  bool parseDoubleQuotedStringInto(std::string &String) {
63    if (!next()) return false;
64    while (*Position != '"') {
65      if (!skipEscapeCharacter()) return false;
66      String.push_back(*Position);
67      if (!next()) return false;
68    }
69    return next();
70  }
71
72  bool parseSingleQuotedStringInto(std::string &String) {
73    if (!next()) return false;
74    while (*Position != '\'') {
75      String.push_back(*Position);
76      if (!next()) return false;
77    }
78    return next();
79  }
80
81  bool parseFreeStringInto(std::string &String) {
82    do {
83      if (!skipEscapeCharacter()) return false;
84      String.push_back(*Position);
85      if (!next()) return false;
86    } while (*Position != ' ' && *Position != '"' && *Position != '\'');
87    return true;
88  }
89
90  bool skipEscapeCharacter() {
91    if (*Position == '\\') {
92      return next();
93    }
94    return true;
95  }
96
97  bool nextNonWhitespace() {
98    do {
99      if (!next()) return false;
100    } while (*Position == ' ');
101    return true;
102  }
103
104  bool next() {
105    ++Position;
106    return Position != Input.end();
107  }
108
109  const StringRef Input;
110  StringRef::iterator Position;
111  std::vector<std::string> CommandLine;
112};
113
114std::vector<std::string> unescapeCommandLine(
115    StringRef EscapedCommandLine) {
116  CommandLineArgumentParser parser(EscapedCommandLine);
117  return parser.parse();
118}
119
120class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
121  std::unique_ptr<CompilationDatabase>
122  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
123    SmallString<1024> JSONDatabasePath(Directory);
124    llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
125    std::unique_ptr<CompilationDatabase> Database(
126        JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
127    if (!Database)
128      return nullptr;
129    return Database;
130  }
131};
132
133} // end namespace
134
135// Register the JSONCompilationDatabasePlugin with the
136// CompilationDatabasePluginRegistry using this statically initialized variable.
137static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
138X("json-compilation-database", "Reads JSON formatted compilation databases");
139
140// This anchor is used to force the linker to link in the generated object file
141// and thus register the JSONCompilationDatabasePlugin.
142volatile int JSONAnchorSource = 0;
143
144std::unique_ptr<JSONCompilationDatabase>
145JSONCompilationDatabase::loadFromFile(StringRef FilePath,
146                                      std::string &ErrorMessage) {
147  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
148      llvm::MemoryBuffer::getFile(FilePath);
149  if (std::error_code Result = DatabaseBuffer.getError()) {
150    ErrorMessage = "Error while opening JSON database: " + Result.message();
151    return nullptr;
152  }
153  std::unique_ptr<JSONCompilationDatabase> Database(
154      new JSONCompilationDatabase(std::move(*DatabaseBuffer)));
155  if (!Database->parse(ErrorMessage))
156    return nullptr;
157  return Database;
158}
159
160std::unique_ptr<JSONCompilationDatabase>
161JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
162                                        std::string &ErrorMessage) {
163  std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
164      llvm::MemoryBuffer::getMemBuffer(DatabaseString));
165  std::unique_ptr<JSONCompilationDatabase> Database(
166      new JSONCompilationDatabase(std::move(DatabaseBuffer)));
167  if (!Database->parse(ErrorMessage))
168    return nullptr;
169  return Database;
170}
171
172std::vector<CompileCommand>
173JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
174  SmallString<128> NativeFilePath;
175  llvm::sys::path::native(FilePath, NativeFilePath);
176
177  std::string Error;
178  llvm::raw_string_ostream ES(Error);
179  StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
180  if (Match.empty())
181    return std::vector<CompileCommand>();
182  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
183    CommandsRefI = IndexByFile.find(Match);
184  if (CommandsRefI == IndexByFile.end())
185    return std::vector<CompileCommand>();
186  std::vector<CompileCommand> Commands;
187  getCommands(CommandsRefI->getValue(), Commands);
188  return Commands;
189}
190
191std::vector<std::string>
192JSONCompilationDatabase::getAllFiles() const {
193  std::vector<std::string> Result;
194
195  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
196    CommandsRefI = IndexByFile.begin();
197  const llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
198    CommandsRefEnd = IndexByFile.end();
199  for (; CommandsRefI != CommandsRefEnd; ++CommandsRefI) {
200    Result.push_back(CommandsRefI->first().str());
201  }
202
203  return Result;
204}
205
206std::vector<CompileCommand>
207JSONCompilationDatabase::getAllCompileCommands() const {
208  std::vector<CompileCommand> Commands;
209  getCommands(AllCommands, Commands);
210  return Commands;
211}
212
213static std::vector<std::string>
214nodeToCommandLine(const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
215  SmallString<1024> Storage;
216  if (Nodes.size() == 1) {
217    return unescapeCommandLine(Nodes[0]->getValue(Storage));
218  }
219  std::vector<std::string> Arguments;
220  for (auto *Node : Nodes) {
221    Arguments.push_back(Node->getValue(Storage));
222  }
223  return Arguments;
224}
225
226void JSONCompilationDatabase::getCommands(
227    ArrayRef<CompileCommandRef> CommandsRef,
228    std::vector<CompileCommand> &Commands) const {
229  for (int I = 0, E = CommandsRef.size(); I != E; ++I) {
230    SmallString<8> DirectoryStorage;
231    SmallString<32> FilenameStorage;
232    Commands.emplace_back(
233      std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
234      std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
235      nodeToCommandLine(std::get<2>(CommandsRef[I])));
236  }
237}
238
239bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
240  llvm::yaml::document_iterator I = YAMLStream.begin();
241  if (I == YAMLStream.end()) {
242    ErrorMessage = "Error while parsing YAML.";
243    return false;
244  }
245  llvm::yaml::Node *Root = I->getRoot();
246  if (!Root) {
247    ErrorMessage = "Error while parsing YAML.";
248    return false;
249  }
250  llvm::yaml::SequenceNode *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
251  if (!Array) {
252    ErrorMessage = "Expected array.";
253    return false;
254  }
255  for (auto& NextObject : *Array) {
256    llvm::yaml::MappingNode *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
257    if (!Object) {
258      ErrorMessage = "Expected object.";
259      return false;
260    }
261    llvm::yaml::ScalarNode *Directory = nullptr;
262    llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command;
263    llvm::yaml::ScalarNode *File = nullptr;
264    for (auto& NextKeyValue : *Object) {
265      llvm::yaml::ScalarNode *KeyString =
266          dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
267      if (!KeyString) {
268        ErrorMessage = "Expected strings as key.";
269        return false;
270      }
271      SmallString<10> KeyStorage;
272      StringRef KeyValue = KeyString->getValue(KeyStorage);
273      llvm::yaml::Node *Value = NextKeyValue.getValue();
274      if (!Value) {
275        ErrorMessage = "Expected value.";
276        return false;
277      }
278      llvm::yaml::ScalarNode *ValueString =
279          dyn_cast<llvm::yaml::ScalarNode>(Value);
280      llvm::yaml::SequenceNode *SequenceString =
281          dyn_cast<llvm::yaml::SequenceNode>(Value);
282      if (KeyValue == "arguments" && !SequenceString) {
283        ErrorMessage = "Expected sequence as value.";
284        return false;
285      } else if (KeyValue != "arguments" && !ValueString) {
286        ErrorMessage = "Expected string as value.";
287        return false;
288      }
289      if (KeyValue == "directory") {
290        Directory = ValueString;
291      } else if (KeyValue == "arguments") {
292        Command = std::vector<llvm::yaml::ScalarNode *>();
293        for (auto &Argument : *SequenceString) {
294          auto Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
295          if (!Scalar) {
296            ErrorMessage = "Only strings are allowed in 'arguments'.";
297            return false;
298          }
299          Command->push_back(Scalar);
300        }
301      } else if (KeyValue == "command") {
302        if (!Command)
303          Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
304      } else if (KeyValue == "file") {
305        File = ValueString;
306      } else {
307        ErrorMessage = ("Unknown key: \"" +
308                        KeyString->getRawValue() + "\"").str();
309        return false;
310      }
311    }
312    if (!File) {
313      ErrorMessage = "Missing key: \"file\".";
314      return false;
315    }
316    if (!Command) {
317      ErrorMessage = "Missing key: \"command\" or \"arguments\".";
318      return false;
319    }
320    if (!Directory) {
321      ErrorMessage = "Missing key: \"directory\".";
322      return false;
323    }
324    SmallString<8> FileStorage;
325    StringRef FileName = File->getValue(FileStorage);
326    SmallString<128> NativeFilePath;
327    if (llvm::sys::path::is_relative(FileName)) {
328      SmallString<8> DirectoryStorage;
329      SmallString<128> AbsolutePath(
330          Directory->getValue(DirectoryStorage));
331      llvm::sys::path::append(AbsolutePath, FileName);
332      llvm::sys::path::native(AbsolutePath, NativeFilePath);
333    } else {
334      llvm::sys::path::native(FileName, NativeFilePath);
335    }
336    auto Cmd = CompileCommandRef(Directory, File, *Command);
337    IndexByFile[NativeFilePath].push_back(Cmd);
338    AllCommands.push_back(Cmd);
339    MatchTrie.insert(NativeFilePath);
340  }
341  return true;
342}
343
344} // end namespace tooling
345} // end namespace clang
346