Deleted Added
sdiff udiff text old ( 218893 ) new ( 226633 )
full compact
TokenConcatenation.cpp (218893) TokenConcatenation.cpp (226633)
1//===--- TokenConcatenation.cpp - Token Concatenation Avoidance -----------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the TokenConcatenation class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Lex/TokenConcatenation.h"
15#include "clang/Lex/Preprocessor.h"
16#include "llvm/Support/ErrorHandling.h"
17using namespace clang;
18
19
1//===--- TokenConcatenation.cpp - Token Concatenation Avoidance -----------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the TokenConcatenation class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Lex/TokenConcatenation.h"
15#include "clang/Lex/Preprocessor.h"
16#include "llvm/Support/ErrorHandling.h"
17using namespace clang;
18
19
20/// StartsWithL - Return true if the spelling of this token starts with 'L'.
21bool TokenConcatenation::StartsWithL(const Token &Tok) const {
22 if (!Tok.needsCleaning()) {
23 SourceManager &SM = PP.getSourceManager();
24 return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L';
25 }
20/// IsStringPrefix - Return true if Str is a string prefix.
21/// 'L', 'u', 'U', or 'u8'. Including raw versions.
22static bool IsStringPrefix(StringRef Str, bool CPlusPlus0x) {
26
23
27 if (Tok.getLength() < 256) {
28 char Buffer[256];
29 const char *TokPtr = Buffer;
30 PP.getSpelling(Tok, TokPtr);
31 return TokPtr[0] == 'L';
24 if (Str[0] == 'L' ||
25 (CPlusPlus0x && (Str[0] == 'u' || Str[0] == 'U' || Str[0] == 'R'))) {
26
27 if (Str.size() == 1)
28 return true; // "L", "u", "U", and "R"
29
30 // Check for raw flavors. Need to make sure the first character wasn't
31 // already R. Need CPlusPlus0x check for "LR".
32 if (Str[1] == 'R' && Str[0] != 'R' && Str.size() == 2 && CPlusPlus0x)
33 return true; // "LR", "uR", "UR"
34
35 // Check for "u8" and "u8R"
36 if (Str[0] == 'u' && Str[1] == '8') {
37 if (Str.size() == 2) return true; // "u8"
38 if (Str.size() == 3 && Str[2] == 'R') return true; // "u8R"
39 }
32 }
33
40 }
41
34 return PP.getSpelling(Tok)[0] == 'L';
42 return false;
35}
36
43}
44
37/// IsIdentifierL - Return true if the spelling of this token is literally
38/// 'L'.
39bool TokenConcatenation::IsIdentifierL(const Token &Tok) const {
45/// IsIdentifierStringPrefix - Return true if the spelling of the token
46/// is literally 'L', 'u', 'U', or 'u8'. Including raw versions.
47bool TokenConcatenation::IsIdentifierStringPrefix(const Token &Tok) const {
48 const LangOptions &LangOpts = PP.getLangOptions();
49
40 if (!Tok.needsCleaning()) {
50 if (!Tok.needsCleaning()) {
41 if (Tok.getLength() != 1)
51 if (Tok.getLength() < 1 || Tok.getLength() > 3)
42 return false;
43 SourceManager &SM = PP.getSourceManager();
52 return false;
53 SourceManager &SM = PP.getSourceManager();
44 return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L';
54 const char *Ptr = SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));
55 return IsStringPrefix(StringRef(Ptr, Tok.getLength()),
56 LangOpts.CPlusPlus0x);
45 }
46
47 if (Tok.getLength() < 256) {
48 char Buffer[256];
49 const char *TokPtr = Buffer;
57 }
58
59 if (Tok.getLength() < 256) {
60 char Buffer[256];
61 const char *TokPtr = Buffer;
50 if (PP.getSpelling(Tok, TokPtr) != 1)
51 return false;
52 return TokPtr[0] == 'L';
62 unsigned length = PP.getSpelling(Tok, TokPtr);
63 return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus0x);
53 }
54
64 }
65
55 return PP.getSpelling(Tok) == "L";
66 return IsStringPrefix(StringRef(PP.getSpelling(Tok)), LangOpts.CPlusPlus0x);
56}
57
58TokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) {
59 memset(TokenInfo, 0, sizeof(TokenInfo));
60
61 // These tokens have custom code in AvoidConcat.
62 TokenInfo[tok::identifier ] |= aci_custom;
63 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;

--- 63 unchanged lines hidden (view full) ---

127/// don't want to track enough to tell "x.." from "...".
128bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
129 const Token &PrevTok,
130 const Token &Tok) const {
131 // First, check to see if the tokens were directly adjacent in the original
132 // source. If they were, it must be okay to stick them together: if there
133 // were an issue, the tokens would have been lexed differently.
134 if (PrevTok.getLocation().isFileID() && Tok.getLocation().isFileID() &&
67}
68
69TokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) {
70 memset(TokenInfo, 0, sizeof(TokenInfo));
71
72 // These tokens have custom code in AvoidConcat.
73 TokenInfo[tok::identifier ] |= aci_custom;
74 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;

--- 63 unchanged lines hidden (view full) ---

138/// don't want to track enough to tell "x.." from "...".
139bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
140 const Token &PrevTok,
141 const Token &Tok) const {
142 // First, check to see if the tokens were directly adjacent in the original
143 // source. If they were, it must be okay to stick them together: if there
144 // were an issue, the tokens would have been lexed differently.
145 if (PrevTok.getLocation().isFileID() && Tok.getLocation().isFileID() &&
135 PrevTok.getLocation().getFileLocWithOffset(PrevTok.getLength()) ==
146 PrevTok.getLocation().getLocWithOffset(PrevTok.getLength()) ==
136 Tok.getLocation())
137 return false;
138
139 tok::TokenKind PrevKind = PrevTok.getKind();
140 if (PrevTok.getIdentifierInfo()) // Language keyword or named operator.
141 PrevKind = tok::identifier;
142
143 // Look up information on when we should avoid concatenation with prevtok.

--- 30 unchanged lines hidden (view full) ---

174 llvm_unreachable("tok::raw_identifier in non-raw lexing mode!");
175 return true;
176
177 case tok::identifier: // id+id or id+number or id+L"foo".
178 // id+'.'... will not append.
179 if (Tok.is(tok::numeric_constant))
180 return GetFirstChar(PP, Tok) != '.';
181
147 Tok.getLocation())
148 return false;
149
150 tok::TokenKind PrevKind = PrevTok.getKind();
151 if (PrevTok.getIdentifierInfo()) // Language keyword or named operator.
152 PrevKind = tok::identifier;
153
154 // Look up information on when we should avoid concatenation with prevtok.

--- 30 unchanged lines hidden (view full) ---

185 llvm_unreachable("tok::raw_identifier in non-raw lexing mode!");
186 return true;
187
188 case tok::identifier: // id+id or id+number or id+L"foo".
189 // id+'.'... will not append.
190 if (Tok.is(tok::numeric_constant))
191 return GetFirstChar(PP, Tok) != '.';
192
182 if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) /* ||
183 Tok.is(tok::wide_char_literal)*/)
193 if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) ||
194 Tok.is(tok::utf8_string_literal) || Tok.is(tok::utf16_string_literal) ||
195 Tok.is(tok::utf32_string_literal) || Tok.is(tok::wide_char_constant) ||
196 Tok.is(tok::utf16_char_constant) || Tok.is(tok::utf32_char_constant))
184 return true;
185
186 // If this isn't identifier + string, we're done.
187 if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal))
188 return false;
189
197 return true;
198
199 // If this isn't identifier + string, we're done.
200 if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal))
201 return false;
202
190 // FIXME: need a wide_char_constant!
191
192 // If the string was a wide string L"foo" or wide char L'f', it would
193 // concat with the previous identifier into fooL"bar". Avoid this.
194 if (StartsWithL(Tok))
195 return true;
196
197 // Otherwise, this is a narrow character or string. If the *identifier*
203 // Otherwise, this is a narrow character or string. If the *identifier*
198 // is a literal 'L', avoid pasting L "foo" -> L"foo".
199 return IsIdentifierL(PrevTok);
204 // is a literal 'L', 'u8', 'u' or 'U', avoid pasting L "foo" -> L"foo".
205 return IsIdentifierStringPrefix(PrevTok);
200 case tok::numeric_constant:
201 return isalnum(FirstChar) || Tok.is(tok::numeric_constant) ||
202 FirstChar == '+' || FirstChar == '-' || FirstChar == '.';
203 case tok::period: // ..., .*, .1234
204 return (FirstChar == '.' && PrevPrevTok.is(tok::period)) ||
205 isdigit(FirstChar) ||
206 (PP.getLangOptions().CPlusPlus && FirstChar == '*');
207 case tok::amp: // &&

--- 24 unchanged lines hidden --- 		206 case tok::numeric_constant:
207 return isalnum(FirstChar) || Tok.is(tok::numeric_constant) ||
208 FirstChar == '+' || FirstChar == '-' || FirstChar == '.';
209 case tok::period: // ..., .*, .1234
210 return (FirstChar == '.' && PrevPrevTok.is(tok::period)) ||
211 isdigit(FirstChar) ||
212 (PP.getLangOptions().CPlusPlus && FirstChar == '*');
213 case tok::amp: // &&

--- 24 unchanged lines hidden ---