ArchiveWriter.cpp revision 327952 
1//===- ArchiveWriter.cpp - ar File Format implementation --------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the writeArchive function.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Object/ArchiveWriter.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/StringRef.h"
17#include "llvm/BinaryFormat/Magic.h"
18#include "llvm/IR/LLVMContext.h"
19#include "llvm/Object/Archive.h"
20#include "llvm/Object/ObjectFile.h"
21#include "llvm/Object/SymbolicFile.h"
22#include "llvm/Support/EndianStream.h"
23#include "llvm/Support/Errc.h"
24#include "llvm/Support/ErrorHandling.h"
25#include "llvm/Support/Format.h"
26#include "llvm/Support/Path.h"
27#include "llvm/Support/ToolOutputFile.h"
28#include "llvm/Support/raw_ostream.h"
29
30#if !defined(_MSC_VER) && !defined(__MINGW32__)
31#include <unistd.h>
32#else
33#include <io.h>
34#endif
35
36using namespace llvm;
37
38// The SYM64 format is used when an archive's member offsets are larger than
39// 32-bits can hold. The need for this shift in format is detected by
40// writeArchive. To test this we need to generate a file with a member that has
41// an offset larger than 32-bits but this demands a very slow test. To speed
42// the test up we use this flag to pretend like the cutoff happens before
43// 32-bits and instead happens at some much smaller value.
44static cl::opt<int> Sym64Threshold("sym64-threshold", cl::Hidden,
45                                   cl::init(32));
46
47NewArchiveMember::NewArchiveMember(MemoryBufferRef BufRef)
48    : Buf(MemoryBuffer::getMemBuffer(BufRef, false)),
49      MemberName(BufRef.getBufferIdentifier()) {}
50
51Expected<NewArchiveMember>
52NewArchiveMember::getOldMember(const object::Archive::Child &OldMember,
53                               bool Deterministic) {
54  Expected<llvm::MemoryBufferRef> BufOrErr = OldMember.getMemoryBufferRef();
55  if (!BufOrErr)
56    return BufOrErr.takeError();
57
58  NewArchiveMember M;
59  assert(M.IsNew == false);
60  M.Buf = MemoryBuffer::getMemBuffer(*BufOrErr, false);
61  M.MemberName = M.Buf->getBufferIdentifier();
62  if (!Deterministic) {
63    auto ModTimeOrErr = OldMember.getLastModified();
64    if (!ModTimeOrErr)
65      return ModTimeOrErr.takeError();
66    M.ModTime = ModTimeOrErr.get();
67    Expected<unsigned> UIDOrErr = OldMember.getUID();
68    if (!UIDOrErr)
69      return UIDOrErr.takeError();
70    M.UID = UIDOrErr.get();
71    Expected<unsigned> GIDOrErr = OldMember.getGID();
72    if (!GIDOrErr)
73      return GIDOrErr.takeError();
74    M.GID = GIDOrErr.get();
75    Expected<sys::fs::perms> AccessModeOrErr = OldMember.getAccessMode();
76    if (!AccessModeOrErr)
77      return AccessModeOrErr.takeError();
78    M.Perms = AccessModeOrErr.get();
79  }
80  return std::move(M);
81}
82
83Expected<NewArchiveMember> NewArchiveMember::getFile(StringRef FileName,
84                                                     bool Deterministic) {
85  sys::fs::file_status Status;
86  int FD;
87  if (auto EC = sys::fs::openFileForRead(FileName, FD))
88    return errorCodeToError(EC);
89  assert(FD != -1);
90
91  if (auto EC = sys::fs::status(FD, Status))
92    return errorCodeToError(EC);
93
94  // Opening a directory doesn't make sense. Let it fail.
95  // Linux cannot open directories with open(2), although
96  // cygwin and *bsd can.
97  if (Status.type() == sys::fs::file_type::directory_file)
98    return errorCodeToError(make_error_code(errc::is_a_directory));
99
100  ErrorOr<std::unique_ptr<MemoryBuffer>> MemberBufferOrErr =
101      MemoryBuffer::getOpenFile(FD, FileName, Status.getSize(), false);
102  if (!MemberBufferOrErr)
103    return errorCodeToError(MemberBufferOrErr.getError());
104
105  if (close(FD) != 0)
106    return errorCodeToError(std::error_code(errno, std::generic_category()));
107
108  NewArchiveMember M;
109  M.IsNew = true;
110  M.Buf = std::move(*MemberBufferOrErr);
111  M.MemberName = M.Buf->getBufferIdentifier();
112  if (!Deterministic) {
113    M.ModTime = std::chrono::time_point_cast<std::chrono::seconds>(
114        Status.getLastModificationTime());
115    M.UID = Status.getUser();
116    M.GID = Status.getGroup();
117    M.Perms = Status.permissions();
118  }
119  return std::move(M);
120}
121
122template <typename T>
123static void printWithSpacePadding(raw_ostream &OS, T Data, unsigned Size) {
124  uint64_t OldPos = OS.tell();
125  OS << Data;
126  unsigned SizeSoFar = OS.tell() - OldPos;
127  assert(SizeSoFar <= Size && "Data doesn't fit in Size");
128  OS.indent(Size - SizeSoFar);
129}
130
131static bool isBSDLike(object::Archive::Kind Kind) {
132  switch (Kind) {
133  case object::Archive::K_GNU:
134  case object::Archive::K_GNU64:
135    return false;
136  case object::Archive::K_BSD:
137  case object::Archive::K_DARWIN:
138    return true;
139  case object::Archive::K_DARWIN64:
140  case object::Archive::K_COFF:
141    break;
142  }
143  llvm_unreachable("not supported for writting");
144}
145
146template <class T>
147static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val) {
148  if (isBSDLike(Kind))
149    support::endian::Writer<support::little>(Out).write(Val);
150  else
151    support::endian::Writer<support::big>(Out).write(Val);
152}
153
154static void printRestOfMemberHeader(
155    raw_ostream &Out, const sys::TimePoint<std::chrono::seconds> &ModTime,
156    unsigned UID, unsigned GID, unsigned Perms, unsigned Size) {
157  printWithSpacePadding(Out, sys::toTimeT(ModTime), 12);
158
159  // The format has only 6 chars for uid and gid. Truncate if the provided
160  // values don't fit.
161  printWithSpacePadding(Out, UID % 1000000, 6);
162  printWithSpacePadding(Out, GID % 1000000, 6);
163
164  printWithSpacePadding(Out, format("%o", Perms), 8);
165  printWithSpacePadding(Out, Size, 10);
166  Out << "`\n";
167}
168
169static void
170printGNUSmallMemberHeader(raw_ostream &Out, StringRef Name,
171                          const sys::TimePoint<std::chrono::seconds> &ModTime,
172                          unsigned UID, unsigned GID, unsigned Perms,
173                          unsigned Size) {
174  printWithSpacePadding(Out, Twine(Name) + "/", 16);
175  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms, Size);
176}
177
178static void
179printBSDMemberHeader(raw_ostream &Out, uint64_t Pos, StringRef Name,
180                     const sys::TimePoint<std::chrono::seconds> &ModTime,
181                     unsigned UID, unsigned GID, unsigned Perms,
182                     unsigned Size) {
183  uint64_t PosAfterHeader = Pos + 60 + Name.size();
184  // Pad so that even 64 bit object files are aligned.
185  unsigned Pad = OffsetToAlignment(PosAfterHeader, 8);
186  unsigned NameWithPadding = Name.size() + Pad;
187  printWithSpacePadding(Out, Twine("#1/") + Twine(NameWithPadding), 16);
188  printRestOfMemberHeader(Out, ModTime, UID, GID, Perms,
189                          NameWithPadding + Size);
190  Out << Name;
191  while (Pad--)
192    Out.write(uint8_t(0));
193}
194
195static bool useStringTable(bool Thin, StringRef Name) {
196  return Thin || Name.size() >= 16 || Name.contains('/');
197}
198
199// Compute the relative path from From to To.
200static std::string computeRelativePath(StringRef From, StringRef To) {
201  if (sys::path::is_absolute(From) || sys::path::is_absolute(To))
202    return To;
203
204  StringRef DirFrom = sys::path::parent_path(From);
205  auto FromI = sys::path::begin(DirFrom);
206  auto ToI = sys::path::begin(To);
207  while (*FromI == *ToI) {
208    ++FromI;
209    ++ToI;
210  }
211
212  SmallString<128> Relative;
213  for (auto FromE = sys::path::end(DirFrom); FromI != FromE; ++FromI)
214    sys::path::append(Relative, "..");
215
216  for (auto ToE = sys::path::end(To); ToI != ToE; ++ToI)
217    sys::path::append(Relative, *ToI);
218
219#ifdef LLVM_ON_WIN32
220  // Replace backslashes with slashes so that the path is portable between *nix
221  // and Windows.
222  std::replace(Relative.begin(), Relative.end(), '\\', '/');
223#endif
224
225  return Relative.str();
226}
227
228static bool is64BitKind(object::Archive::Kind Kind) {
229  switch (Kind) {
230  case object::Archive::K_GNU:
231  case object::Archive::K_BSD:
232  case object::Archive::K_DARWIN:
233  case object::Archive::K_COFF:
234    return false;
235  case object::Archive::K_DARWIN64:
236  case object::Archive::K_GNU64:
237    return true;
238  }
239  llvm_unreachable("not supported for writting");
240}
241
242static void addToStringTable(raw_ostream &Out, StringRef ArcName,
243                             const NewArchiveMember &M, bool Thin) {
244  StringRef ID = M.Buf->getBufferIdentifier();
245  if (Thin) {
246    if (M.IsNew)
247      Out << computeRelativePath(ArcName, ID);
248    else
249      Out << ID;
250  } else
251    Out << M.MemberName;
252  Out << "/\n";
253}
254
255static void printMemberHeader(raw_ostream &Out, uint64_t Pos,
256                              raw_ostream &StringTable,
257                              object::Archive::Kind Kind, bool Thin,
258                              StringRef ArcName, const NewArchiveMember &M,
259                              unsigned Size) {
260  if (isBSDLike(Kind))
261    return printBSDMemberHeader(Out, Pos, M.MemberName, M.ModTime, M.UID, M.GID,
262                                M.Perms, Size);
263  if (!useStringTable(Thin, M.MemberName))
264    return printGNUSmallMemberHeader(Out, M.MemberName, M.ModTime, M.UID, M.GID,
265                                     M.Perms, Size);
266  Out << '/';
267  uint64_t NamePos = StringTable.tell();
268  addToStringTable(StringTable, ArcName, M, Thin);
269  printWithSpacePadding(Out, NamePos, 15);
270  printRestOfMemberHeader(Out, M.ModTime, M.UID, M.GID, M.Perms, Size);
271}
272
273namespace {
274struct MemberData {
275  std::vector<unsigned> Symbols;
276  std::string Header;
277  StringRef Data;
278  StringRef Padding;
279};
280} // namespace
281
282static MemberData computeStringTable(StringRef Names) {
283  unsigned Size = Names.size();
284  unsigned Pad = OffsetToAlignment(Size, 2);
285  std::string Header;
286  raw_string_ostream Out(Header);
287  printWithSpacePadding(Out, "//", 48);
288  printWithSpacePadding(Out, Size + Pad, 10);
289  Out << "`\n";
290  Out.flush();
291  return {{}, std::move(Header), Names, Pad ? "\n" : ""};
292}
293
294static sys::TimePoint<std::chrono::seconds> now(bool Deterministic) {
295  using namespace std::chrono;
296
297  if (!Deterministic)
298    return time_point_cast<seconds>(system_clock::now());
299  return sys::TimePoint<seconds>();
300}
301
302static bool isArchiveSymbol(const object::BasicSymbolRef &S) {
303  uint32_t Symflags = S.getFlags();
304  if (Symflags & object::SymbolRef::SF_FormatSpecific)
305    return false;
306  if (!(Symflags & object::SymbolRef::SF_Global))
307    return false;
308  if (Symflags & object::SymbolRef::SF_Undefined &&
309      !(Symflags & object::SymbolRef::SF_Indirect))
310    return false;
311  return true;
312}
313
314static void printNBits(raw_ostream &Out, object::Archive::Kind Kind,
315                       uint64_t Val) {
316  if (is64BitKind(Kind))
317    print<uint64_t>(Out, Kind, Val);
318  else
319    print<uint32_t>(Out, Kind, Val);
320}
321
322static void writeSymbolTable(raw_ostream &Out, object::Archive::Kind Kind,
323                             bool Deterministic, ArrayRef<MemberData> Members,
324                             StringRef StringTable) {
325  if (StringTable.empty())
326    return;
327
328  unsigned NumSyms = 0;
329  for (const MemberData &M : Members)
330    NumSyms += M.Symbols.size();
331
332  unsigned Size = 0;
333  Size += is64BitKind(Kind) ? 8 : 4; // Number of entries
334  if (isBSDLike(Kind))
335    Size += NumSyms * 8; // Table
336  else if (is64BitKind(Kind))
337    Size += NumSyms * 8; // Table
338  else
339    Size += NumSyms * 4; // Table
340  if (isBSDLike(Kind))
341    Size += 4; // byte count
342  Size += StringTable.size();
343  // ld64 expects the members to be 8-byte aligned for 64-bit content and at
344  // least 4-byte aligned for 32-bit content.  Opt for the larger encoding
345  // uniformly.
346  // We do this for all bsd formats because it simplifies aligning members.
347  unsigned Alignment = isBSDLike(Kind) ? 8 : 2;
348  unsigned Pad = OffsetToAlignment(Size, Alignment);
349  Size += Pad;
350
351  if (isBSDLike(Kind))
352    printBSDMemberHeader(Out, Out.tell(), "__.SYMDEF", now(Deterministic), 0, 0,
353                         0, Size);
354  else if (is64BitKind(Kind))
355    printGNUSmallMemberHeader(Out, "/SYM64", now(Deterministic), 0, 0, 0, Size);
356  else
357    printGNUSmallMemberHeader(Out, "", now(Deterministic), 0, 0, 0, Size);
358
359  uint64_t Pos = Out.tell() + Size;
360
361  if (isBSDLike(Kind))
362    print<uint32_t>(Out, Kind, NumSyms * 8);
363  else
364    printNBits(Out, Kind, NumSyms);
365
366  for (const MemberData &M : Members) {
367    for (unsigned StringOffset : M.Symbols) {
368      if (isBSDLike(Kind))
369        print<uint32_t>(Out, Kind, StringOffset);
370      printNBits(Out, Kind, Pos); // member offset
371    }
372    Pos += M.Header.size() + M.Data.size() + M.Padding.size();
373  }
374
375  if (isBSDLike(Kind))
376    // byte count of the string table
377    print<uint32_t>(Out, Kind, StringTable.size());
378  Out << StringTable;
379
380  while (Pad--)
381    Out.write(uint8_t(0));
382}
383
384static Expected<std::vector<unsigned>>
385getSymbols(MemoryBufferRef Buf, raw_ostream &SymNames, bool &HasObject) {
386  std::vector<unsigned> Ret;
387  LLVMContext Context;
388
389  Expected<std::unique_ptr<object::SymbolicFile>> ObjOrErr =
390      object::SymbolicFile::createSymbolicFile(Buf, llvm::file_magic::unknown,
391                                               &Context);
392  if (!ObjOrErr) {
393    // FIXME: check only for "not an object file" errors.
394    consumeError(ObjOrErr.takeError());
395    return Ret;
396  }
397
398  HasObject = true;
399  object::SymbolicFile &Obj = *ObjOrErr.get();
400  for (const object::BasicSymbolRef &S : Obj.symbols()) {
401    if (!isArchiveSymbol(S))
402      continue;
403    Ret.push_back(SymNames.tell());
404    if (auto EC = S.printName(SymNames))
405      return errorCodeToError(EC);
406    SymNames << '\0';
407  }
408  return Ret;
409}
410
411static Expected<std::vector<MemberData>>
412computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
413                  object::Archive::Kind Kind, bool Thin, StringRef ArcName,
414                  ArrayRef<NewArchiveMember> NewMembers) {
415  static char PaddingData[8] = {'\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'};
416
417  // This ignores the symbol table, but we only need the value mod 8 and the
418  // symbol table is aligned to be a multiple of 8 bytes
419  uint64_t Pos = 0;
420
421  std::vector<MemberData> Ret;
422  bool HasObject = false;
423  for (const NewArchiveMember &M : NewMembers) {
424    std::string Header;
425    raw_string_ostream Out(Header);
426
427    MemoryBufferRef Buf = M.Buf->getMemBufferRef();
428    StringRef Data = Thin ? "" : Buf.getBuffer();
429
430    // ld64 expects the members to be 8-byte aligned for 64-bit content and at
431    // least 4-byte aligned for 32-bit content.  Opt for the larger encoding
432    // uniformly.  This matches the behaviour with cctools and ensures that ld64
433    // is happy with archives that we generate.
434    unsigned MemberPadding = Kind == object::Archive::K_DARWIN
435                                 ? OffsetToAlignment(Data.size(), 8)
436                                 : 0;
437    unsigned TailPadding = OffsetToAlignment(Data.size() + MemberPadding, 2);
438    StringRef Padding = StringRef(PaddingData, MemberPadding + TailPadding);
439
440    printMemberHeader(Out, Pos, StringTable, Kind, Thin, ArcName, M,
441                      Buf.getBufferSize() + MemberPadding);
442    Out.flush();
443
444    Expected<std::vector<unsigned>> Symbols =
445        getSymbols(Buf, SymNames, HasObject);
446    if (auto E = Symbols.takeError())
447      return std::move(E);
448
449    Pos += Header.size() + Data.size() + Padding.size();
450    Ret.push_back({std::move(*Symbols), std::move(Header), Data, Padding});
451  }
452  // If there are no symbols, emit an empty symbol table, to satisfy Solaris
453  // tools, older versions of which expect a symbol table in a non-empty
454  // archive, regardless of whether there are any symbols in it.
455  if (HasObject && SymNames.tell() == 0)
456    SymNames << '\0' << '\0' << '\0';
457  return Ret;
458}
459
460Error llvm::writeArchive(StringRef ArcName,
461                         ArrayRef<NewArchiveMember> NewMembers,
462                         bool WriteSymtab, object::Archive::Kind Kind,
463                         bool Deterministic, bool Thin,
464                         std::unique_ptr<MemoryBuffer> OldArchiveBuf) {
465  assert((!Thin || !isBSDLike(Kind)) && "Only the gnu format has a thin mode");
466
467  SmallString<0> SymNamesBuf;
468  raw_svector_ostream SymNames(SymNamesBuf);
469  SmallString<0> StringTableBuf;
470  raw_svector_ostream StringTable(StringTableBuf);
471
472  Expected<std::vector<MemberData>> DataOrErr =
473      computeMemberData(StringTable, SymNames, Kind, Thin, ArcName, NewMembers);
474  if (Error E = DataOrErr.takeError())
475    return E;
476  std::vector<MemberData> &Data = *DataOrErr;
477
478  if (!StringTableBuf.empty())
479    Data.insert(Data.begin(), computeStringTable(StringTableBuf));
480
481  // We would like to detect if we need to switch to a 64-bit symbol table.
482  if (WriteSymtab) {
483    uint64_t MaxOffset = 0;
484    uint64_t LastOffset = MaxOffset;
485    for (const auto& M : Data) {
486      // Record the start of the member's offset
487      LastOffset = MaxOffset;
488      // Account for the size of each part associated with the member.
489      MaxOffset += M.Header.size() + M.Data.size() + M.Padding.size();
490      // We assume 32-bit symbols to see if 32-bit symbols are possible or not.
491      MaxOffset += M.Symbols.size() * 4;
492    }
493    // If LastOffset isn't going to fit in a 32-bit varible we need to switch
494    // to 64-bit. Note that the file can be larger than 4GB as long as the last
495    // member starts before the 4GB offset.
496    if (LastOffset >= (1ULL << Sym64Threshold))
497      Kind = object::Archive::K_GNU64;
498  }
499
500  Expected<sys::fs::TempFile> Temp =
501      sys::fs::TempFile::create(ArcName + ".temp-archive-%%%%%%%.a");
502  if (!Temp)
503    return Temp.takeError();
504
505  raw_fd_ostream Out(Temp->FD, false);
506  if (Thin)
507    Out << "!<thin>\n";
508  else
509    Out << "!<arch>\n";
510
511  if (WriteSymtab)
512    writeSymbolTable(Out, Kind, Deterministic, Data, SymNamesBuf);
513
514  for (const MemberData &M : Data)
515    Out << M.Header << M.Data << M.Padding;
516
517  Out.flush();
518
519  // At this point, we no longer need whatever backing memory
520  // was used to generate the NewMembers. On Windows, this buffer
521  // could be a mapped view of the file we want to replace (if
522  // we're updating an existing archive, say). In that case, the
523  // rename would still succeed, but it would leave behind a
524  // temporary file (actually the original file renamed) because
525  // a file cannot be deleted while there's a handle open on it,
526  // only renamed. So by freeing this buffer, this ensures that
527  // the last open handle on the destination file, if any, is
528  // closed before we attempt to rename.
529  OldArchiveBuf.reset();
530
531  return Temp->keep(ArcName);
532}
533