Deleted Added
sdiff udiff text old ( 193326 ) new ( 198092 )
full compact
CacheTokens.cpp (193326) CacheTokens.cpp (198092)
1//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//

--- 6 unchanged lines hidden (view full) ---

15#include "clang/Frontend/Utils.h"
16#include "clang/Basic/FileManager.h"
17#include "clang/Basic/SourceManager.h"
18#include "clang/Basic/IdentifierTable.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Basic/OnDiskHashTable.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Preprocessor.h"
1//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//

--- 6 unchanged lines hidden (view full) ---

15#include "clang/Frontend/Utils.h"
16#include "clang/Basic/FileManager.h"
17#include "clang/Basic/SourceManager.h"
18#include "clang/Basic/IdentifierTable.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Basic/OnDiskHashTable.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Preprocessor.h"
23#include "llvm/ADT/StringMap.h"
23#include "llvm/Support/Compiler.h"
24#include "llvm/Support/MemoryBuffer.h"
25#include "llvm/Support/raw_ostream.h"
26#include "llvm/System/Path.h"
24#include "llvm/Support/MemoryBuffer.h"
25#include "llvm/Support/raw_ostream.h"
26#include "llvm/System/Path.h"
27#include "llvm/Support/Compiler.h"
28#include "llvm/Support/Streams.h"
27#include "llvm/ADT/StringMap.h"
29
30// FIXME: put this somewhere else?
31#ifndef S_ISDIR
32#define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
33#endif
34
35using namespace clang;
36using namespace clang::io;
37
38//===----------------------------------------------------------------------===//
39// PTH-specific stuff.
40//===----------------------------------------------------------------------===//
41
42namespace {
43class VISIBILITY_HIDDEN PTHEntry {
28
29// FIXME: put this somewhere else?
30#ifndef S_ISDIR
31#define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
32#endif
33
34using namespace clang;
35using namespace clang::io;
36
37//===----------------------------------------------------------------------===//
38// PTH-specific stuff.
39//===----------------------------------------------------------------------===//
40
41namespace {
42class VISIBILITY_HIDDEN PTHEntry {
44 Offset TokenData, PPCondData;
43 Offset TokenData, PPCondData;
45
44
46public:
45public:
47 PTHEntry() {}
48
49 PTHEntry(Offset td, Offset ppcd)
50 : TokenData(td), PPCondData(ppcd) {}
46 PTHEntry() {}
47
48 PTHEntry(Offset td, Offset ppcd)
49 : TokenData(td), PPCondData(ppcd) {}
51
52 Offset getTokenOffset() const { return TokenData; }
50
51 Offset getTokenOffset() const { return TokenData; }
53 Offset getPPCondTableOffset() const { return PPCondData; }
54};
52 Offset getPPCondTableOffset() const { return PPCondData; }
53};
55
56
54
55
57class VISIBILITY_HIDDEN PTHEntryKeyVariant {
58 union { const FileEntry* FE; const char* Path; };
59 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
60 struct stat *StatBuf;
61public:
62 PTHEntryKeyVariant(const FileEntry *fe)
63 : FE(fe), Kind(IsFE), StatBuf(0) {}
64
65 PTHEntryKeyVariant(struct stat* statbuf, const char* path)
66 : Path(path), Kind(IsDE), StatBuf(new struct stat(*statbuf)) {}
67
68 PTHEntryKeyVariant(const char* path)
69 : Path(path), Kind(IsNoExist), StatBuf(0) {}
56class VISIBILITY_HIDDEN PTHEntryKeyVariant {
57 union { const FileEntry* FE; const char* Path; };
58 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
59 struct stat *StatBuf;
60public:
61 PTHEntryKeyVariant(const FileEntry *fe)
62 : FE(fe), Kind(IsFE), StatBuf(0) {}
63
64 PTHEntryKeyVariant(struct stat* statbuf, const char* path)
65 : Path(path), Kind(IsDE), StatBuf(new struct stat(*statbuf)) {}
66
67 PTHEntryKeyVariant(const char* path)
68 : Path(path), Kind(IsNoExist), StatBuf(0) {}
70
69
71 bool isFile() const { return Kind == IsFE; }
70 bool isFile() const { return Kind == IsFE; }
72
71
73 const char* getCString() const {
74 return Kind == IsFE ? FE->getName() : Path;
75 }
72 const char* getCString() const {
73 return Kind == IsFE ? FE->getName() : Path;
74 }
76
75
77 unsigned getKind() const { return (unsigned) Kind; }
76 unsigned getKind() const { return (unsigned) Kind; }
78
77
79 void EmitData(llvm::raw_ostream& Out) {
80 switch (Kind) {
81 case IsFE:
82 // Emit stat information.
83 ::Emit32(Out, FE->getInode());
84 ::Emit32(Out, FE->getDevice());
85 ::Emit16(Out, FE->getFileMode());
86 ::Emit64(Out, FE->getModificationTime());

--- 7 unchanged lines hidden (view full) ---

94 ::Emit64(Out, (uint64_t) StatBuf->st_mtime);
95 ::Emit64(Out, (uint64_t) StatBuf->st_size);
96 delete StatBuf;
97 break;
98 default:
99 break;
100 }
101 }
78 void EmitData(llvm::raw_ostream& Out) {
79 switch (Kind) {
80 case IsFE:
81 // Emit stat information.
82 ::Emit32(Out, FE->getInode());
83 ::Emit32(Out, FE->getDevice());
84 ::Emit16(Out, FE->getFileMode());
85 ::Emit64(Out, FE->getModificationTime());

--- 7 unchanged lines hidden (view full) ---

93 ::Emit64(Out, (uint64_t) StatBuf->st_mtime);
94 ::Emit64(Out, (uint64_t) StatBuf->st_size);
95 delete StatBuf;
96 break;
97 default:
98 break;
99 }
100 }
102
101
103 unsigned getRepresentationLength() const {
104 return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8;
105 }
106};
102 unsigned getRepresentationLength() const {
103 return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8;
104 }
105};
107
106
108class VISIBILITY_HIDDEN FileEntryPTHEntryInfo {
109public:
110 typedef PTHEntryKeyVariant key_type;
111 typedef key_type key_type_ref;
107class VISIBILITY_HIDDEN FileEntryPTHEntryInfo {
108public:
109 typedef PTHEntryKeyVariant key_type;
110 typedef key_type key_type_ref;
112
111
113 typedef PTHEntry data_type;
114 typedef const PTHEntry& data_type_ref;
112 typedef PTHEntry data_type;
113 typedef const PTHEntry& data_type_ref;
115
114
116 static unsigned ComputeHash(PTHEntryKeyVariant V) {
117 return BernsteinHash(V.getCString());
118 }
115 static unsigned ComputeHash(PTHEntryKeyVariant V) {
116 return BernsteinHash(V.getCString());
117 }
119
120 static std::pair<unsigned,unsigned>
118
119 static std::pair
121 EmitKeyDataLength(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
122 const PTHEntry& E) {
123
124 unsigned n = strlen(V.getCString()) + 1 + 1;
125 ::Emit16(Out, n);
120 EmitKeyDataLength(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
121 const PTHEntry& E) {
122
123 unsigned n = strlen(V.getCString()) + 1 + 1;
124 ::Emit16(Out, n);
126
125
127 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
128 ::Emit8(Out, m);
129
130 return std::make_pair(n, m);
131 }
126 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
127 ::Emit8(Out, m);
128
129 return std::make_pair(n, m);
130 }
132
131
133 static void EmitKey(llvm::raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
134 // Emit the entry kind.
135 ::Emit8(Out, (unsigned) V.getKind());
136 // Emit the string.
137 Out.write(V.getCString(), n - 1);
138 }
132 static void EmitKey(llvm::raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
133 // Emit the entry kind.
134 ::Emit8(Out, (unsigned) V.getKind());
135 // Emit the string.
136 Out.write(V.getCString(), n - 1);
137 }
139
140 static void EmitData(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
138
139 static void EmitData(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
141 const PTHEntry& E, unsigned) {
142
143
144 // For file entries emit the offsets into the PTH file for token data
145 // and the preprocessor blocks table.
146 if (V.isFile()) {
147 ::Emit32(Out, E.getTokenOffset());
148 ::Emit32(Out, E.getPPCondTableOffset());
149 }
140 const PTHEntry& E, unsigned) {
141
142
143 // For file entries emit the offsets into the PTH file for token data
144 // and the preprocessor blocks table.
145 if (V.isFile()) {
146 ::Emit32(Out, E.getTokenOffset());
147 ::Emit32(Out, E.getPPCondTableOffset());
148 }
150
149
151 // Emit any other data associated with the key (i.e., stat information).
152 V.EmitData(Out);
150 // Emit any other data associated with the key (i.e., stat information).
151 V.EmitData(Out);
153 }
152 }
154};
153};
155
154
156class OffsetOpt {
157 bool valid;
158 Offset off;
159public:
160 OffsetOpt() : valid(false) {}
161 bool hasOffset() const { return valid; }
162 Offset getOffset() const { assert(valid); return off; }
163 void setOffset(Offset o) { off = o; valid = true; }

--- 12 unchanged lines hidden (view full) ---

176 uint32_t idcount;
177 PTHMap PM;
178 CachedStrsTy CachedStrs;
179 Offset CurStrOffset;
180 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
181
182 //// Get the persistent id for the given IdentifierInfo*.
183 uint32_t ResolveID(const IdentifierInfo* II);
155class OffsetOpt {
156 bool valid;
157 Offset off;
158public:
159 OffsetOpt() : valid(false) {}
160 bool hasOffset() const { return valid; }
161 Offset getOffset() const { assert(valid); return off; }
162 void setOffset(Offset o) { off = o; valid = true; }

--- 12 unchanged lines hidden (view full) ---

175 uint32_t idcount;
176 PTHMap PM;
177 CachedStrsTy CachedStrs;
178 Offset CurStrOffset;
179 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
180
181 //// Get the persistent id for the given IdentifierInfo*.
182 uint32_t ResolveID(const IdentifierInfo* II);
184
183
185 /// Emit a token to the PTH file.
186 void EmitToken(const Token& T);
187
188 void Emit8(uint32_t V) {
189 Out << (unsigned char)(V);
190 }
184 /// Emit a token to the PTH file.
185 void EmitToken(const Token& T);
186
187 void Emit8(uint32_t V) {
188 Out << (unsigned char)(V);
189 }
191
190
192 void Emit16(uint32_t V) { ::Emit16(Out, V); }
191 void Emit16(uint32_t V) { ::Emit16(Out, V); }
193
192
194 void Emit24(uint32_t V) {
195 Out << (unsigned char)(V);
196 Out << (unsigned char)(V >> 8);
197 Out << (unsigned char)(V >> 16);
198 assert((V >> 24) == 0);
199 }
200
201 void Emit32(uint32_t V) { ::Emit32(Out, V); }
202
203 void EmitBuf(const char *Ptr, unsigned NumBytes) {
204 Out.write(Ptr, NumBytes);
205 }
193 void Emit24(uint32_t V) {
194 Out << (unsigned char)(V);
195 Out << (unsigned char)(V >> 8);
196 Out << (unsigned char)(V >> 16);
197 assert((V >> 24) == 0);
198 }
199
200 void Emit32(uint32_t V) { ::Emit32(Out, V); }
201
202 void EmitBuf(const char *Ptr, unsigned NumBytes) {
203 Out.write(Ptr, NumBytes);
204 }
206
205
207 /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
208 /// a hashtable mapping from identifier strings to persistent IDs.
209 /// The second is a straight table mapping from persistent IDs to string data
210 /// (the keys of the first table).
211 std::pair<Offset, Offset> EmitIdentifierTable();
206 /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
207 /// a hashtable mapping from identifier strings to persistent IDs.
208 /// The second is a straight table mapping from persistent IDs to string data
209 /// (the keys of the first table).
210 std::pair<Offset, Offset> EmitIdentifierTable();
212
211
213 /// EmitFileTable - Emit a table mapping from file name strings to PTH
214 /// token data.
215 Offset EmitFileTable() { return PM.Emit(Out); }
216
217 PTHEntry LexTokens(Lexer& L);
218 Offset EmitCachedSpellings();
219
220public:
212 /// EmitFileTable - Emit a table mapping from file name strings to PTH
213 /// token data.
214 Offset EmitFileTable() { return PM.Emit(Out); }
215
216 PTHEntry LexTokens(Lexer& L);
217 Offset EmitCachedSpellings();
218
219public:
221 PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp)
220 PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp)
222 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
221 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
223
222
224 PTHMap &getPM() { return PM; }
225 void GeneratePTH(const std::string *MainFile = 0);
226};
227} // end anonymous namespace
223 PTHMap &getPM() { return PM; }
224 void GeneratePTH(const std::string *MainFile = 0);
225};
226} // end anonymous namespace
228
229uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
227
228uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
230 // Null IdentifierInfo's map to the persistent ID 0.
231 if (!II)
232 return 0;
229 // Null IdentifierInfo's map to the persistent ID 0.
230 if (!II)
231 return 0;
233
232
234 IDMap::iterator I = IM.find(II);
235 if (I != IM.end())
236 return I->second; // We've already added 1.
233 IDMap::iterator I = IM.find(II);
234 if (I != IM.end())
235 return I->second; // We've already added 1.
237
236
238 IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
239 return idcount;
240}
241
242void PTHWriter::EmitToken(const Token& T) {
243 // Emit the token kind, flags, and length.
244 Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)|
245 (((uint32_t) T.getLength()) << 16));
237 IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
238 return idcount;
239}
240
241void PTHWriter::EmitToken(const Token& T) {
242 // Emit the token kind, flags, and length.
243 Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)|
244 (((uint32_t) T.getLength()) << 16));
246
245
247 if (!T.isLiteral()) {
248 Emit32(ResolveID(T.getIdentifierInfo()));
249 } else {
250 // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
251 // source code.
252 const char* s = T.getLiteralData();
253 unsigned len = T.getLength();
254
255 // Get the string entry.
256 llvm::StringMapEntry<OffsetOpt> *E = &CachedStrs.GetOrCreateValue(s, s+len);
246 if (!T.isLiteral()) {
247 Emit32(ResolveID(T.getIdentifierInfo()));
248 } else {
249 // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
250 // source code.
251 const char* s = T.getLiteralData();
252 unsigned len = T.getLength();
253
254 // Get the string entry.
255 llvm::StringMapEntry<OffsetOpt> *E = &CachedStrs.GetOrCreateValue(s, s+len);
257
256
258 // If this is a new string entry, bump the PTH offset.
259 if (!E->getValue().hasOffset()) {
260 E->getValue().setOffset(CurStrOffset);
261 StrEntries.push_back(E);
262 CurStrOffset += len + 1;
263 }
257 // If this is a new string entry, bump the PTH offset.
258 if (!E->getValue().hasOffset()) {
259 E->getValue().setOffset(CurStrOffset);
260 StrEntries.push_back(E);
261 CurStrOffset += len + 1;
262 }
264
263
265 // Emit the relative offset into the PTH file for the spelling string.
266 Emit32(E->getValue().getOffset());
267 }
264 // Emit the relative offset into the PTH file for the spelling string.
265 Emit32(E->getValue().getOffset());
266 }
268
267
269 // Emit the offset into the original source file of this token so that we
270 // can reconstruct its SourceLocation.
271 Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
272}
273
274PTHEntry PTHWriter::LexTokens(Lexer& L) {
275 // Pad 0's so that we emit tokens to a 4-byte alignment.
276 // This speed up reading them back in.
277 Pad(Out, 4);
278 Offset off = (Offset) Out.tell();
268 // Emit the offset into the original source file of this token so that we
269 // can reconstruct its SourceLocation.
270 Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
271}
272
273PTHEntry PTHWriter::LexTokens(Lexer& L) {
274 // Pad 0's so that we emit tokens to a 4-byte alignment.
275 // This speed up reading them back in.
276 Pad(Out, 4);
277 Offset off = (Offset) Out.tell();
279
278
280 // Keep track of matching '#if' ... '#endif'.
281 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
282 PPCondTable PPCond;
283 std::vector<unsigned> PPStartCond;
284 bool ParsingPreprocessorDirective = false;
285 Token Tok;
279 // Keep track of matching '#if' ... '#endif'.
280 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
281 PPCondTable PPCond;
282 std::vector<unsigned> PPStartCond;
283 bool ParsingPreprocessorDirective = false;
284 Token Tok;
286
285
287 do {
288 L.LexFromRawLexer(Tok);
289 NextToken:
290
291 if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
292 ParsingPreprocessorDirective) {
293 // Insert an eom token into the token cache. It has the same
294 // position as the next token that is not on the same line as the
295 // preprocessor directive. Observe that we continue processing
296 // 'Tok' when we exit this branch.
297 Token Tmp = Tok;
298 Tmp.setKind(tok::eom);
299 Tmp.clearFlag(Token::StartOfLine);
300 Tmp.setIdentifierInfo(0);
301 EmitToken(Tmp);
302 ParsingPreprocessorDirective = false;
303 }
286 do {
287 L.LexFromRawLexer(Tok);
288 NextToken:
289
290 if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
291 ParsingPreprocessorDirective) {
292 // Insert an eom token into the token cache. It has the same
293 // position as the next token that is not on the same line as the
294 // preprocessor directive. Observe that we continue processing
295 // 'Tok' when we exit this branch.
296 Token Tmp = Tok;
297 Tmp.setKind(tok::eom);
298 Tmp.clearFlag(Token::StartOfLine);
299 Tmp.setIdentifierInfo(0);
300 EmitToken(Tmp);
301 ParsingPreprocessorDirective = false;
302 }
304
303
305 if (Tok.is(tok::identifier)) {
306 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
307 EmitToken(Tok);
308 continue;
309 }
310
311 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
312 // Special processing for #include. Store the '#' token and lex
313 // the next token.
314 assert(!ParsingPreprocessorDirective);
315 Offset HashOff = (Offset) Out.tell();
316 EmitToken(Tok);
317
318 // Get the next token.
319 L.LexFromRawLexer(Tok);
320
321 // If we see the start of line, then we had a null directive "#".
322 if (Tok.isAtStartOfLine())
323 goto NextToken;
304 if (Tok.is(tok::identifier)) {
305 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
306 EmitToken(Tok);
307 continue;
308 }
309
310 if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
311 // Special processing for #include. Store the '#' token and lex
312 // the next token.
313 assert(!ParsingPreprocessorDirective);
314 Offset HashOff = (Offset) Out.tell();
315 EmitToken(Tok);
316
317 // Get the next token.
318 L.LexFromRawLexer(Tok);
319
320 // If we see the start of line, then we had a null directive "#".
321 if (Tok.isAtStartOfLine())
322 goto NextToken;
324
323
325 // Did we see 'include'/'import'/'include_next'?
326 if (Tok.isNot(tok::identifier)) {
327 EmitToken(Tok);
328 continue;
329 }
324 // Did we see 'include'/'import'/'include_next'?
325 if (Tok.isNot(tok::identifier)) {
326 EmitToken(Tok);
327 continue;
328 }
330
329
331 IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
332 Tok.setIdentifierInfo(II);
333 tok::PPKeywordKind K = II->getPPKeywordID();
330 IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
331 Tok.setIdentifierInfo(II);
332 tok::PPKeywordKind K = II->getPPKeywordID();
334
333
335 ParsingPreprocessorDirective = true;
334 ParsingPreprocessorDirective = true;
336
335
337 switch (K) {
338 case tok::pp_not_keyword:
339 // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
340 // them through.
341 default:
342 break;
336 switch (K) {
337 case tok::pp_not_keyword:
338 // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
339 // them through.
340 default:
341 break;
343
342
344 case tok::pp_include:
345 case tok::pp_import:
343 case tok::pp_include:
344 case tok::pp_import:
346 case tok::pp_include_next: {
345 case tok::pp_include_next: {
347 // Save the 'include' token.
348 EmitToken(Tok);
349 // Lex the next token as an include string.
350 L.setParsingPreprocessorDirective(true);
346 // Save the 'include' token.
347 EmitToken(Tok);
348 // Lex the next token as an include string.
349 L.setParsingPreprocessorDirective(true);
351 L.LexIncludeFilename(Tok);
350 L.LexIncludeFilename(Tok);
352 L.setParsingPreprocessorDirective(false);
353 assert(!Tok.isAtStartOfLine());
354 if (Tok.is(tok::identifier))
355 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
351 L.setParsingPreprocessorDirective(false);
352 assert(!Tok.isAtStartOfLine());
353 if (Tok.is(tok::identifier))
354 Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
356
355
357 break;
358 }
359 case tok::pp_if:
360 case tok::pp_ifdef:
361 case tok::pp_ifndef: {
362 // Add an entry for '#if' and friends. We initially set the target
363 // index to 0. This will get backpatched when we hit #endif.
364 PPStartCond.push_back(PPCond.size());

--- 5 unchanged lines hidden (view full) ---

370 // This will later be set to zero when emitting to the PTH file. We
371 // use 0 for uninitialized indices because that is easier to debug.
372 unsigned index = PPCond.size();
373 // Backpatch the opening '#if' entry.
374 assert(!PPStartCond.empty());
375 assert(PPCond.size() > PPStartCond.back());
376 assert(PPCond[PPStartCond.back()].second == 0);
377 PPCond[PPStartCond.back()].second = index;
356 break;
357 }
358 case tok::pp_if:
359 case tok::pp_ifdef:
360 case tok::pp_ifndef: {
361 // Add an entry for '#if' and friends. We initially set the target
362 // index to 0. This will get backpatched when we hit #endif.
363 PPStartCond.push_back(PPCond.size());

--- 5 unchanged lines hidden (view full) ---

369 // This will later be set to zero when emitting to the PTH file. We
370 // use 0 for uninitialized indices because that is easier to debug.
371 unsigned index = PPCond.size();
372 // Backpatch the opening '#if' entry.
373 assert(!PPStartCond.empty());
374 assert(PPCond.size() > PPStartCond.back());
375 assert(PPCond[PPStartCond.back()].second == 0);
376 PPCond[PPStartCond.back()].second = index;
378 PPStartCond.pop_back();
379 // Add the new entry to PPCond.
377 PPStartCond.pop_back();
378 // Add the new entry to PPCond.
380 PPCond.push_back(std::make_pair(HashOff, index));
381 EmitToken(Tok);
379 PPCond.push_back(std::make_pair(HashOff, index));
380 EmitToken(Tok);
382
381
383 // Some files have gibberish on the same line as '#endif'.
384 // Discard these tokens.
385 do
386 L.LexFromRawLexer(Tok);
387 while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
388 // We have the next token in hand.
389 // Don't immediately lex the next one.
382 // Some files have gibberish on the same line as '#endif'.
383 // Discard these tokens.
384 do
385 L.LexFromRawLexer(Tok);
386 while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
387 // We have the next token in hand.
388 // Don't immediately lex the next one.
390 goto NextToken;
389 goto NextToken;
391 }
392 case tok::pp_elif:
393 case tok::pp_else: {
394 // Add an entry for #elif or #else.
395 // This serves as both a closing and opening of a conditional block.
396 // This means that its entry will get backpatched later.
397 unsigned index = PPCond.size();
398 // Backpatch the previous '#if' entry.

--- 4 unchanged lines hidden (view full) ---

403 PPStartCond.pop_back();
404 // Now add '#elif' as a new block opening.
405 PPCond.push_back(std::make_pair(HashOff, 0U));
406 PPStartCond.push_back(index);
407 break;
408 }
409 }
410 }
390 }
391 case tok::pp_elif:
392 case tok::pp_else: {
393 // Add an entry for #elif or #else.
394 // This serves as both a closing and opening of a conditional block.
395 // This means that its entry will get backpatched later.
396 unsigned index = PPCond.size();
397 // Backpatch the previous '#if' entry.

--- 4 unchanged lines hidden (view full) ---

402 PPStartCond.pop_back();
403 // Now add '#elif' as a new block opening.
404 PPCond.push_back(std::make_pair(HashOff, 0U));
405 PPStartCond.push_back(index);
406 break;
407 }
408 }
409 }
411
410
412 EmitToken(Tok);
413 }
414 while (Tok.isNot(tok::eof));
415
416 assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
417
418 // Next write out PPCond.
419 Offset PPCondOff = (Offset) Out.tell();

--- 11 unchanged lines hidden (view full) ---

431 }
432
433 return PTHEntry(off, PPCondOff);
434}
435
436Offset PTHWriter::EmitCachedSpellings() {
437 // Write each cached strings to the PTH file.
438 Offset SpellingsOff = Out.tell();
411 EmitToken(Tok);
412 }
413 while (Tok.isNot(tok::eof));
414
415 assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
416
417 // Next write out PPCond.
418 Offset PPCondOff = (Offset) Out.tell();

--- 11 unchanged lines hidden (view full) ---

430 }
431
432 return PTHEntry(off, PPCondOff);
433}
434
435Offset PTHWriter::EmitCachedSpellings() {
436 // Write each cached strings to the PTH file.
437 Offset SpellingsOff = Out.tell();
439
438
440 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
441 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
442 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/);
439 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
440 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
441 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/);
443
442
444 return SpellingsOff;
445}
446
447void PTHWriter::GeneratePTH(const std::string *MainFile) {
448 // Generate the prologue.
449 Out << "cfe-pth";
450 Emit32(PTHManager::Version);
443 return SpellingsOff;
444}
445
446void PTHWriter::GeneratePTH(const std::string *MainFile) {
447 // Generate the prologue.
448 Out << "cfe-pth";
449 Emit32(PTHManager::Version);
451
450
452 // Leave 4 words for the prologue.
453 Offset PrologueOffset = Out.tell();
454 for (unsigned i = 0; i < 4; ++i)
455 Emit32(0);
451 // Leave 4 words for the prologue.
452 Offset PrologueOffset = Out.tell();
453 for (unsigned i = 0; i < 4; ++i)
454 Emit32(0);
456
455
457 // Write the name of the MainFile.
458 if (MainFile && !MainFile->empty()) {
459 Emit16(MainFile->length());
460 EmitBuf(MainFile->data(), MainFile->length());
461 } else {
462 // String with 0 bytes.
463 Emit16(0);
464 }
465 Emit8(0);
456 // Write the name of the MainFile.
457 if (MainFile && !MainFile->empty()) {
458 Emit16(MainFile->length());
459 EmitBuf(MainFile->data(), MainFile->length());
460 } else {
461 // String with 0 bytes.
462 Emit16(0);
463 }
464 Emit8(0);
466
465
467 // Iterate over all the files in SourceManager. Create a lexer
468 // for each file and cache the tokens.
469 SourceManager &SM = PP.getSourceManager();
470 const LangOptions &LOpts = PP.getLangOptions();
466 // Iterate over all the files in SourceManager. Create a lexer
467 // for each file and cache the tokens.
468 SourceManager &SM = PP.getSourceManager();
469 const LangOptions &LOpts = PP.getLangOptions();
471
470
472 for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
473 E = SM.fileinfo_end(); I != E; ++I) {
474 const SrcMgr::ContentCache &C = *I->second;
475 const FileEntry *FE = C.Entry;
471 for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
472 E = SM.fileinfo_end(); I != E; ++I) {
473 const SrcMgr::ContentCache &C = *I->second;
474 const FileEntry *FE = C.Entry;
476
475
477 // FIXME: Handle files with non-absolute paths.
478 llvm::sys::Path P(FE->getName());
479 if (!P.isAbsolute())
480 continue;
481
482 const llvm::MemoryBuffer *B = C.getBuffer();
483 if (!B) continue;
484
485 FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
486 Lexer L(FID, SM, LOpts);
487 PM.insert(FE, LexTokens(L));
488 }
489
490 // Write out the identifier table.
491 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
476 // FIXME: Handle files with non-absolute paths.
477 llvm::sys::Path P(FE->getName());
478 if (!P.isAbsolute())
479 continue;
480
481 const llvm::MemoryBuffer *B = C.getBuffer();
482 if (!B) continue;
483
484 FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
485 Lexer L(FID, SM, LOpts);
486 PM.insert(FE, LexTokens(L));
487 }
488
489 // Write out the identifier table.
490 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
492
491
493 // Write out the cached strings table.
494 Offset SpellingOff = EmitCachedSpellings();
492 // Write out the cached strings table.
493 Offset SpellingOff = EmitCachedSpellings();
495
494
496 // Write out the file table.
495 // Write out the file table.
497 Offset FileTableOff = EmitFileTable();
498
496 Offset FileTableOff = EmitFileTable();
497
499 // Finally, write the prologue.
500 Out.seek(PrologueOffset);
501 Emit32(IdTableOff.first);
502 Emit32(IdTableOff.second);
503 Emit32(FileTableOff);
504 Emit32(SpellingOff);
505}
506
507namespace {
508/// StatListener - A simple "interpose" object used to monitor stat calls
509/// invoked by FileManager while processing the original sources used
510/// as input to PTH generation. StatListener populates the PTHWriter's
511/// file map with stat information for directories as well as negative stats.
512/// Stat information for files are populated elsewhere.
513class StatListener : public StatSysCallCache {
514 PTHMap &PM;
515public:
516 StatListener(PTHMap &pm) : PM(pm) {}
517 ~StatListener() {}
498 // Finally, write the prologue.
499 Out.seek(PrologueOffset);
500 Emit32(IdTableOff.first);
501 Emit32(IdTableOff.second);
502 Emit32(FileTableOff);
503 Emit32(SpellingOff);
504}
505
506namespace {
507/// StatListener - A simple "interpose" object used to monitor stat calls
508/// invoked by FileManager while processing the original sources used
509/// as input to PTH generation. StatListener populates the PTHWriter's
510/// file map with stat information for directories as well as negative stats.
511/// Stat information for files are populated elsewhere.
512class StatListener : public StatSysCallCache {
513 PTHMap &PM;
514public:
515 StatListener(PTHMap &pm) : PM(pm) {}
516 ~StatListener() {}
518
517
519 int stat(const char *path, struct stat *buf) {
520 int result = ::stat(path, buf);
518 int stat(const char *path, struct stat *buf) {
519 int result = ::stat(path, buf);
521
520
522 if (result != 0) // Failed 'stat'.
523 PM.insert(path, PTHEntry());
524 else if (S_ISDIR(buf->st_mode)) {
525 // Only cache directories with absolute paths.
526 if (!llvm::sys::Path(path).isAbsolute())
527 return result;
521 if (result != 0) // Failed 'stat'.
522 PM.insert(path, PTHEntry());
523 else if (S_ISDIR(buf->st_mode)) {
524 // Only cache directories with absolute paths.
525 if (!llvm::sys::Path(path).isAbsolute())
526 return result;
528
527
529 PM.insert(PTHEntryKeyVariant(buf, path), PTHEntry());
530 }
528 PM.insert(PTHEntryKeyVariant(buf, path), PTHEntry());
529 }
531
530
532 return result;
533 }
534};
535} // end anonymous namespace
536
537
538void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) {
539 // Get the name of the main file.
540 const SourceManager &SrcMgr = PP.getSourceManager();
541 const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
542 llvm::sys::Path MainFilePath(MainFile->getName());
543 std::string MainFileName;
531 return result;
532 }
533};
534} // end anonymous namespace
535
536
537void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) {
538 // Get the name of the main file.
539 const SourceManager &SrcMgr = PP.getSourceManager();
540 const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
541 llvm::sys::Path MainFilePath(MainFile->getName());
542 std::string MainFileName;
544
543
545 if (!MainFilePath.isAbsolute()) {
546 llvm::sys::Path P = llvm::sys::Path::GetCurrentDirectory();
544 if (!MainFilePath.isAbsolute()) {
545 llvm::sys::Path P = llvm::sys::Path::GetCurrentDirectory();
547 P.appendComponent(MainFilePath.toString());
548 MainFileName = P.toString();
546 P.appendComponent(MainFilePath.str());
547 MainFileName = P.str();
549 } else {
548 } else {
550 MainFileName = MainFilePath.toString();
549 MainFileName = MainFilePath.str();
551 }
552
553 // Create the PTHWriter.
554 PTHWriter PW(*OS, PP);
550 }
551
552 // Create the PTHWriter.
553 PTHWriter PW(*OS, PP);
555
554
556 // Install the 'stat' system call listener in the FileManager.
557 PP.getFileManager().setStatCache(new StatListener(PW.getPM()));
555 // Install the 'stat' system call listener in the FileManager.
556 PP.getFileManager().setStatCache(new StatListener(PW.getPM()));
558
557
559 // Lex through the entire file. This will populate SourceManager with
560 // all of the header information.
561 Token Tok;
562 PP.EnterMainSourceFile();
563 do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
558 // Lex through the entire file. This will populate SourceManager with
559 // all of the header information.
560 Token Tok;
561 PP.EnterMainSourceFile();
562 do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
564
563
565 // Generate the PTH file.
566 PP.getFileManager().setStatCache(0);
567 PW.GeneratePTH(&MainFileName);
568}
569
570//===----------------------------------------------------------------------===//
571
572class PTHIdKey {
573public:
574 const IdentifierInfo* II;
575 uint32_t FileOffset;
576};
577
578namespace {
579class VISIBILITY_HIDDEN PTHIdentifierTableTrait {
580public:
581 typedef PTHIdKey* key_type;
582 typedef key_type key_type_ref;
564 // Generate the PTH file.
565 PP.getFileManager().setStatCache(0);
566 PW.GeneratePTH(&MainFileName);
567}
568
569//===----------------------------------------------------------------------===//
570
571class PTHIdKey {
572public:
573 const IdentifierInfo* II;
574 uint32_t FileOffset;
575};
576
577namespace {
578class VISIBILITY_HIDDEN PTHIdentifierTableTrait {
579public:
580 typedef PTHIdKey* key_type;
581 typedef key_type key_type_ref;
583
582
584 typedef uint32_t data_type;
585 typedef data_type data_type_ref;
583 typedef uint32_t data_type;
584 typedef data_type data_type_ref;
586
585
587 static unsigned ComputeHash(PTHIdKey* key) {
588 return BernsteinHash(key->II->getName());
589 }
586 static unsigned ComputeHash(PTHIdKey* key) {
587 return BernsteinHash(key->II->getName());
588 }
590
591 static std::pair<unsigned,unsigned>
592 EmitKeyDataLength(llvm::raw_ostream& Out, const PTHIdKey* key, uint32_t) {
589
590 static std::pair
591 EmitKeyDataLength(llvm::raw_ostream& Out, const PTHIdKey* key, uint32_t) {
593 unsigned n = strlen(key->II->getName()) + 1;
594 ::Emit16(Out, n);
595 return std::make_pair(n, sizeof(uint32_t));
596 }
592 unsigned n = strlen(key->II->getName()) + 1;
593 ::Emit16(Out, n);
594 return std::make_pair(n, sizeof(uint32_t));
595 }
597
596
598 static void EmitKey(llvm::raw_ostream& Out, PTHIdKey* key, unsigned n) {
599 // Record the location of the key data. This is used when generating
600 // the mapping from persistent IDs to strings.
601 key->FileOffset = Out.tell();
602 Out.write(key->II->getName(), n);
603 }
597 static void EmitKey(llvm::raw_ostream& Out, PTHIdKey* key, unsigned n) {
598 // Record the location of the key data. This is used when generating
599 // the mapping from persistent IDs to strings.
600 key->FileOffset = Out.tell();
601 Out.write(key->II->getName(), n);
602 }
604
603
605 static void EmitData(llvm::raw_ostream& Out, PTHIdKey*, uint32_t pID,
606 unsigned) {
607 ::Emit32(Out, pID);
604 static void EmitData(llvm::raw_ostream& Out, PTHIdKey*, uint32_t pID,
605 unsigned) {
606 ::Emit32(Out, pID);
608 }
607 }
609};
610} // end anonymous namespace
611
612/// EmitIdentifierTable - Emits two tables to the PTH file. The first is
613/// a hashtable mapping from identifier strings to persistent IDs. The second
614/// is a straight table mapping from persistent IDs to string data (the
615/// keys of the first table).
616///
617std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
618 // Build two maps:
619 // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset)
620 // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
621
622 // Note that we use 'calloc', so all the bytes are 0.
623 PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey));
624
625 // Create the hashtable.
626 OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
608};
609} // end anonymous namespace
610
611/// EmitIdentifierTable - Emits two tables to the PTH file. The first is
612/// a hashtable mapping from identifier strings to persistent IDs. The second
613/// is a straight table mapping from persistent IDs to string data (the
614/// keys of the first table).
615///
616std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
617 // Build two maps:
618 // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset)
619 // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
620
621 // Note that we use 'calloc', so all the bytes are 0.
622 PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey));
623
624 // Create the hashtable.
625 OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
627
626
628 // Generate mapping from persistent IDs -> IdentifierInfo*.
629 for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
630 // Decrement by 1 because we are using a vector for the lookup and
631 // 0 is reserved for NULL.
632 assert(I->second > 0);
633 assert(I->second-1 < idcount);
634 unsigned idx = I->second-1;
627 // Generate mapping from persistent IDs -> IdentifierInfo*.
628 for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
629 // Decrement by 1 because we are using a vector for the lookup and
630 // 0 is reserved for NULL.
631 assert(I->second > 0);
632 assert(I->second-1 < idcount);
633 unsigned idx = I->second-1;
635
634
636 // Store the mapping from persistent ID to IdentifierInfo*
637 IIDMap[idx].II = I->first;
635 // Store the mapping from persistent ID to IdentifierInfo*
636 IIDMap[idx].II = I->first;
638
637
639 // Store the reverse mapping in a hashtable.
640 IIOffMap.insert(&IIDMap[idx], I->second);
641 }
638 // Store the reverse mapping in a hashtable.
639 IIOffMap.insert(&IIDMap[idx], I->second);
640 }
642
641
643 // Write out the inverse map first. This causes the PCIDKey entries to
644 // record PTH file offsets for the string data. This is used to write
645 // the second table.
646 Offset StringTableOffset = IIOffMap.Emit(Out);
642 // Write out the inverse map first. This causes the PCIDKey entries to
643 // record PTH file offsets for the string data. This is used to write
644 // the second table.
645 Offset StringTableOffset = IIOffMap.Emit(Out);
647
648 // Now emit the table mapping from persistent IDs to PTH file offsets.
646
647 // Now emit the table mapping from persistent IDs to PTH file offsets.
649 Offset IDOff = Out.tell();
650 Emit32(idcount); // Emit the number of identifiers.
651 for (unsigned i = 0 ; i < idcount; ++i)
652 Emit32(IIDMap[i].FileOffset);
648 Offset IDOff = Out.tell();
649 Emit32(idcount); // Emit the number of identifiers.
650 for (unsigned i = 0 ; i < idcount; ++i)
651 Emit32(IIDMap[i].FileOffset);
653
652
654 // Finally, release the inverse map.
655 free(IIDMap);
653 // Finally, release the inverse map.
654 free(IIDMap);
656
655
657 return std::make_pair(IDOff, StringTableOffset);
658}
656 return std::make_pair(IDOff, StringTableOffset);
657}