1//===-- TraceIntelPTBundleSaver.cpp ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "TraceIntelPTBundleSaver.h"
10#include "PerfContextSwitchDecoder.h"
11#include "TraceIntelPT.h"
12#include "TraceIntelPTConstants.h"
13#include "TraceIntelPTJSONStructs.h"
14#include "lldb/Core/Module.h"
15#include "lldb/Core/ModuleList.h"
16#include "lldb/Target/Process.h"
17#include "lldb/Target/SectionLoadList.h"
18#include "lldb/Target/Target.h"
19#include "lldb/Target/ThreadList.h"
20#include "lldb/lldb-types.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/JSON.h"
23#include <fstream>
24#include <iostream>
25#include <optional>
26#include <sstream>
27#include <string>
28
29using namespace lldb;
30using namespace lldb_private;
31using namespace lldb_private::trace_intel_pt;
32using namespace llvm;
33
34/// Strip the \p directory component from the given \p path. It assumes that \p
35/// directory is a prefix of \p path.
36static std::string GetRelativePath(const FileSpec &directory,
37                                   const FileSpec &path) {
38  return path.GetPath().substr(directory.GetPath().size() + 1);
39}
40
41/// Write a stream of bytes from \p data to the given output file.
42/// It creates or overwrites the output file, but not append.
43static llvm::Error WriteBytesToDisk(FileSpec &output_file,
44                                    ArrayRef<uint8_t> data) {
45  std::basic_fstream<char> out_fs = std::fstream(
46      output_file.GetPath().c_str(), std::ios::out | std::ios::binary);
47  if (!data.empty())
48    out_fs.write(reinterpret_cast<const char *>(&data[0]), data.size());
49
50  out_fs.close();
51  if (!out_fs)
52    return createStringError(inconvertibleErrorCode(),
53                             formatv("couldn't write to the file {0}",
54                                     output_file.GetPath().c_str()));
55  return Error::success();
56}
57
58/// Save the trace bundle description JSON object inside the given directory
59/// as a file named \a trace.json.
60///
61/// \param[in] trace_bundle_description
62///     The trace bundle description as JSON Object.
63///
64/// \param[in] directory
65///     The directory where the JSON file will be saved.
66///
67/// \return
68///     A \a FileSpec pointing to the bundle description file, or an \a
69///     llvm::Error otherwise.
70static Expected<FileSpec>
71SaveTraceBundleDescription(const llvm::json::Value &trace_bundle_description,
72                           const FileSpec &directory) {
73  FileSpec trace_path = directory;
74  trace_path.AppendPathComponent("trace.json");
75  std::ofstream os(trace_path.GetPath());
76  os << formatv("{0:2}", trace_bundle_description).str();
77  os.close();
78  if (!os)
79    return createStringError(inconvertibleErrorCode(),
80                             formatv("couldn't write to the file {0}",
81                                     trace_path.GetPath().c_str()));
82  return trace_path;
83}
84
85/// Build the threads sub-section of the trace bundle description file.
86/// Any associated binary files are created inside the given directory.
87///
88/// \param[in] process
89///     The process being traced.
90///
91/// \param[in] directory
92///     The directory where files will be saved when building the threads
93///     section.
94///
95/// \return
96///     The threads section or \a llvm::Error in case of failures.
97static llvm::Expected<std::vector<JSONThread>>
98BuildThreadsSection(Process &process, FileSpec directory) {
99  std::vector<JSONThread> json_threads;
100  TraceSP trace_sp = process.GetTarget().GetTrace();
101
102  FileSpec threads_dir = directory;
103  threads_dir.AppendPathComponent("threads");
104  sys::fs::create_directories(threads_dir.GetPath().c_str());
105
106  for (ThreadSP thread_sp : process.Threads()) {
107    lldb::tid_t tid = thread_sp->GetID();
108    if (!trace_sp->IsTraced(tid))
109      continue;
110
111    JSONThread json_thread;
112    json_thread.tid = tid;
113
114    if (trace_sp->GetTracedCpus().empty()) {
115      FileSpec output_file = threads_dir;
116      output_file.AppendPathComponent(std::to_string(tid) + ".intelpt_trace");
117      json_thread.ipt_trace = GetRelativePath(directory, output_file);
118
119      llvm::Error err = process.GetTarget().GetTrace()->OnThreadBinaryDataRead(
120          tid, IntelPTDataKinds::kIptTrace,
121          [&](llvm::ArrayRef<uint8_t> data) -> llvm::Error {
122            return WriteBytesToDisk(output_file, data);
123          });
124      if (err)
125        return std::move(err);
126    }
127
128    json_threads.push_back(std::move(json_thread));
129  }
130  return json_threads;
131}
132
133/// \return
134///   an \a llvm::Error in case of failures, \a std::nullopt if the trace is not
135///   written to disk because the trace is empty and the \p compact flag is
136///   present, or the FileSpec of the trace file on disk.
137static Expected<std::optional<FileSpec>>
138WriteContextSwitchTrace(TraceIntelPT &trace_ipt, lldb::cpu_id_t cpu_id,
139                        const FileSpec &cpus_dir, bool compact) {
140  FileSpec output_context_switch_trace = cpus_dir;
141  output_context_switch_trace.AppendPathComponent(std::to_string(cpu_id) +
142                                                  ".perf_context_switch_trace");
143
144  bool should_skip = false;
145
146  Error err = trace_ipt.OnCpuBinaryDataRead(
147      cpu_id, IntelPTDataKinds::kPerfContextSwitchTrace,
148      [&](llvm::ArrayRef<uint8_t> data) -> llvm::Error {
149        if (!compact)
150          return WriteBytesToDisk(output_context_switch_trace, data);
151
152        std::set<lldb::pid_t> pids;
153        for (Process *process : trace_ipt.GetAllProcesses())
154          pids.insert(process->GetID());
155
156        Expected<std::vector<uint8_t>> compact_context_switch_trace =
157            FilterProcessesFromContextSwitchTrace(data, pids);
158        if (!compact_context_switch_trace)
159          return compact_context_switch_trace.takeError();
160
161        if (compact_context_switch_trace->empty()) {
162          should_skip = true;
163          return Error::success();
164        }
165
166        return WriteBytesToDisk(output_context_switch_trace,
167                                *compact_context_switch_trace);
168      });
169  if (err)
170    return std::move(err);
171
172  if (should_skip)
173    return std::nullopt;
174  return output_context_switch_trace;
175}
176
177static Expected<FileSpec> WriteIntelPTTrace(TraceIntelPT &trace_ipt,
178                                            lldb::cpu_id_t cpu_id,
179                                            const FileSpec &cpus_dir) {
180  FileSpec output_trace = cpus_dir;
181  output_trace.AppendPathComponent(std::to_string(cpu_id) + ".intelpt_trace");
182
183  Error err = trace_ipt.OnCpuBinaryDataRead(
184      cpu_id, IntelPTDataKinds::kIptTrace,
185      [&](llvm::ArrayRef<uint8_t> data) -> llvm::Error {
186        return WriteBytesToDisk(output_trace, data);
187      });
188  if (err)
189    return std::move(err);
190  return output_trace;
191}
192
193static llvm::Expected<std::optional<std::vector<JSONCpu>>>
194BuildCpusSection(TraceIntelPT &trace_ipt, FileSpec directory, bool compact) {
195  if (trace_ipt.GetTracedCpus().empty())
196    return std::nullopt;
197
198  std::vector<JSONCpu> json_cpus;
199  FileSpec cpus_dir = directory;
200  cpus_dir.AppendPathComponent("cpus");
201  sys::fs::create_directories(cpus_dir.GetPath().c_str());
202
203  for (lldb::cpu_id_t cpu_id : trace_ipt.GetTracedCpus()) {
204    JSONCpu json_cpu;
205    json_cpu.id = cpu_id;
206    Expected<std::optional<FileSpec>> context_switch_trace_path =
207        WriteContextSwitchTrace(trace_ipt, cpu_id, cpus_dir, compact);
208    if (!context_switch_trace_path)
209      return context_switch_trace_path.takeError();
210    if (!*context_switch_trace_path)
211      continue;
212    json_cpu.context_switch_trace =
213        GetRelativePath(directory, **context_switch_trace_path);
214
215    if (Expected<FileSpec> ipt_trace_path =
216            WriteIntelPTTrace(trace_ipt, cpu_id, cpus_dir))
217      json_cpu.ipt_trace = GetRelativePath(directory, *ipt_trace_path);
218    else
219      return ipt_trace_path.takeError();
220
221    json_cpus.push_back(std::move(json_cpu));
222  }
223  return json_cpus;
224}
225
226/// Build modules sub-section of the trace bundle. The original modules
227/// will be copied over to the \a <directory/modules> folder. Invalid modules
228/// are skipped.
229/// Copying the modules has the benefit of making these
230/// directories self-contained, as the raw traces and modules are part of the
231/// output directory and can be sent to another machine, where lldb can load
232/// them and replicate exactly the same trace session.
233///
234/// \param[in] process
235///     The process being traced.
236///
237/// \param[in] directory
238///     The directory where the modules files will be saved when building
239///     the modules section.
240///     Example: If a module \a libbar.so exists in the path
241///     \a /usr/lib/foo/libbar.so, then it will be copied to
242///     \a <directory>/modules/usr/lib/foo/libbar.so.
243///
244/// \return
245///     The modules section or \a llvm::Error in case of failures.
246static llvm::Expected<std::vector<JSONModule>>
247BuildModulesSection(Process &process, FileSpec directory) {
248  std::vector<JSONModule> json_modules;
249  ModuleList module_list = process.GetTarget().GetImages();
250  for (size_t i = 0; i < module_list.GetSize(); ++i) {
251    ModuleSP module_sp(module_list.GetModuleAtIndex(i));
252    if (!module_sp)
253      continue;
254    std::string system_path = module_sp->GetPlatformFileSpec().GetPath();
255    // TODO: support memory-only libraries like [vdso]
256    if (!module_sp->GetFileSpec().IsAbsolute())
257      continue;
258
259    std::string file = module_sp->GetFileSpec().GetPath();
260    ObjectFile *objfile = module_sp->GetObjectFile();
261    if (objfile == nullptr)
262      continue;
263
264    lldb::addr_t load_addr = LLDB_INVALID_ADDRESS;
265    Address base_addr(objfile->GetBaseAddress());
266    if (base_addr.IsValid() &&
267        !process.GetTarget().GetSectionLoadList().IsEmpty())
268      load_addr = base_addr.GetLoadAddress(&process.GetTarget());
269
270    if (load_addr == LLDB_INVALID_ADDRESS)
271      continue;
272
273    FileSpec path_to_copy_module = directory;
274    path_to_copy_module.AppendPathComponent("modules");
275    path_to_copy_module.AppendPathComponent(system_path);
276    sys::fs::create_directories(path_to_copy_module.GetDirectory().AsCString());
277
278    if (std::error_code ec =
279            llvm::sys::fs::copy_file(file, path_to_copy_module.GetPath()))
280      return createStringError(
281          inconvertibleErrorCode(),
282          formatv("couldn't write to the file. {0}", ec.message()));
283
284    json_modules.push_back(
285        JSONModule{system_path, GetRelativePath(directory, path_to_copy_module),
286                   JSONUINT64{load_addr}, module_sp->GetUUID().GetAsString()});
287  }
288  return json_modules;
289}
290
291/// Build the processes section of the trace bundle description object. Besides
292/// returning the processes information, this method saves to disk all modules
293/// and raw traces corresponding to the traced threads of the given process.
294///
295/// \param[in] process
296///     The process being traced.
297///
298/// \param[in] directory
299///     The directory where files will be saved when building the processes
300///     section.
301///
302/// \return
303///     The processes section or \a llvm::Error in case of failures.
304static llvm::Expected<JSONProcess>
305BuildProcessSection(Process &process, const FileSpec &directory) {
306  Expected<std::vector<JSONThread>> json_threads =
307      BuildThreadsSection(process, directory);
308  if (!json_threads)
309    return json_threads.takeError();
310
311  Expected<std::vector<JSONModule>> json_modules =
312      BuildModulesSection(process, directory);
313  if (!json_modules)
314    return json_modules.takeError();
315
316  return JSONProcess{
317      process.GetID(),
318      process.GetTarget().GetArchitecture().GetTriple().getTriple(),
319      json_threads.get(), json_modules.get()};
320}
321
322/// See BuildProcessSection()
323static llvm::Expected<std::vector<JSONProcess>>
324BuildProcessesSection(TraceIntelPT &trace_ipt, const FileSpec &directory) {
325  std::vector<JSONProcess> processes;
326  for (Process *process : trace_ipt.GetAllProcesses()) {
327    if (llvm::Expected<JSONProcess> json_process =
328            BuildProcessSection(*process, directory))
329      processes.push_back(std::move(*json_process));
330    else
331      return json_process.takeError();
332  }
333  return processes;
334}
335
336static llvm::Expected<JSONKernel>
337BuildKernelSection(TraceIntelPT &trace_ipt, const FileSpec &directory) {
338  JSONKernel json_kernel;
339  std::vector<Process *> processes = trace_ipt.GetAllProcesses();
340  Process *kernel_process = processes[0];
341
342  assert(processes.size() == 1 && "User processeses exist in kernel mode");
343  assert(kernel_process->GetID() == kDefaultKernelProcessID &&
344         "Kernel process not exist");
345
346  Expected<std::vector<JSONModule>> json_modules =
347      BuildModulesSection(*kernel_process, directory);
348  if (!json_modules)
349    return json_modules.takeError();
350
351  JSONModule kernel_image = json_modules.get()[0];
352  return JSONKernel{kernel_image.load_address, kernel_image.system_path};
353}
354
355Expected<FileSpec> TraceIntelPTBundleSaver::SaveToDisk(TraceIntelPT &trace_ipt,
356                                                       FileSpec directory,
357                                                       bool compact) {
358  if (std::error_code ec =
359          sys::fs::create_directories(directory.GetPath().c_str()))
360    return llvm::errorCodeToError(ec);
361
362  Expected<pt_cpu> cpu_info = trace_ipt.GetCPUInfo();
363  if (!cpu_info)
364    return cpu_info.takeError();
365
366  FileSystem::Instance().Resolve(directory);
367
368  Expected<std::optional<std::vector<JSONCpu>>> json_cpus =
369      BuildCpusSection(trace_ipt, directory, compact);
370  if (!json_cpus)
371    return json_cpus.takeError();
372
373  std::optional<std::vector<JSONProcess>> json_processes;
374  std::optional<JSONKernel> json_kernel;
375
376  if (trace_ipt.GetTraceMode() == TraceIntelPT::TraceMode::KernelMode) {
377    Expected<std::optional<JSONKernel>> exp_json_kernel =
378        BuildKernelSection(trace_ipt, directory);
379    if (!exp_json_kernel)
380      return exp_json_kernel.takeError();
381    else
382      json_kernel = *exp_json_kernel;
383  } else {
384    Expected<std::optional<std::vector<JSONProcess>>> exp_json_processes =
385        BuildProcessesSection(trace_ipt, directory);
386    if (!exp_json_processes)
387      return exp_json_processes.takeError();
388    else
389      json_processes = *exp_json_processes;
390  }
391
392  JSONTraceBundleDescription json_intel_pt_bundle_desc{
393      "intel-pt",
394      *cpu_info,
395      json_processes,
396      *json_cpus,
397      trace_ipt.GetPerfZeroTscConversion(),
398      json_kernel};
399
400  return SaveTraceBundleDescription(toJSON(json_intel_pt_bundle_desc),
401                                    directory);
402}
403