LiteralSupport.cpp revision 251662
1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the NumericLiteralParser, CharLiteralParser, and 11// StringLiteralParser interfaces. 12// 13//===----------------------------------------------------------------------===// 14 15#include "clang/Lex/LiteralSupport.h" 16#include "clang/Basic/CharInfo.h" 17#include "clang/Basic/TargetInfo.h" 18#include "clang/Lex/LexDiagnostic.h" 19#include "clang/Lex/Preprocessor.h" 20#include "llvm/ADT/StringExtras.h" 21#include "llvm/Support/ConvertUTF.h" 22#include "llvm/Support/ErrorHandling.h" 23 24using namespace clang; 25 26static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) { 27 switch (kind) { 28 default: llvm_unreachable("Unknown token type!"); 29 case tok::char_constant: 30 case tok::string_literal: 31 case tok::utf8_string_literal: 32 return Target.getCharWidth(); 33 case tok::wide_char_constant: 34 case tok::wide_string_literal: 35 return Target.getWCharWidth(); 36 case tok::utf16_char_constant: 37 case tok::utf16_string_literal: 38 return Target.getChar16Width(); 39 case tok::utf32_char_constant: 40 case tok::utf32_string_literal: 41 return Target.getChar32Width(); 42 } 43} 44 45static CharSourceRange MakeCharSourceRange(const LangOptions &Features, 46 FullSourceLoc TokLoc, 47 const char *TokBegin, 48 const char *TokRangeBegin, 49 const char *TokRangeEnd) { 50 SourceLocation Begin = 51 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 52 TokLoc.getManager(), Features); 53 SourceLocation End = 54 Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin, 55 TokLoc.getManager(), Features); 56 return CharSourceRange::getCharRange(Begin, End); 57} 58 59/// \brief Produce a diagnostic highlighting some portion of a literal. 60/// 61/// Emits the diagnostic \p DiagID, highlighting the range of characters from 62/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be 63/// a substring of a spelling buffer for the token beginning at \p TokBegin. 64static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, 65 const LangOptions &Features, FullSourceLoc TokLoc, 66 const char *TokBegin, const char *TokRangeBegin, 67 const char *TokRangeEnd, unsigned DiagID) { 68 SourceLocation Begin = 69 Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin, 70 TokLoc.getManager(), Features); 71 return Diags->Report(Begin, DiagID) << 72 MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd); 73} 74 75/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in 76/// either a character or a string literal. 77static unsigned ProcessCharEscape(const char *ThisTokBegin, 78 const char *&ThisTokBuf, 79 const char *ThisTokEnd, bool &HadError, 80 FullSourceLoc Loc, unsigned CharWidth, 81 DiagnosticsEngine *Diags, 82 const LangOptions &Features) { 83 const char *EscapeBegin = ThisTokBuf; 84 85 // Skip the '\' char. 86 ++ThisTokBuf; 87 88 // We know that this character can't be off the end of the buffer, because 89 // that would have been \", which would not have been the end of string. 90 unsigned ResultChar = *ThisTokBuf++; 91 switch (ResultChar) { 92 // These map to themselves. 93 case '\\': case '\'': case '"': case '?': break; 94 95 // These have fixed mappings. 96 case 'a': 97 // TODO: K&R: the meaning of '\\a' is different in traditional C 98 ResultChar = 7; 99 break; 100 case 'b': 101 ResultChar = 8; 102 break; 103 case 'e': 104 if (Diags) 105 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 106 diag::ext_nonstandard_escape) << "e"; 107 ResultChar = 27; 108 break; 109 case 'E': 110 if (Diags) 111 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 112 diag::ext_nonstandard_escape) << "E"; 113 ResultChar = 27; 114 break; 115 case 'f': 116 ResultChar = 12; 117 break; 118 case 'n': 119 ResultChar = 10; 120 break; 121 case 'r': 122 ResultChar = 13; 123 break; 124 case 't': 125 ResultChar = 9; 126 break; 127 case 'v': 128 ResultChar = 11; 129 break; 130 case 'x': { // Hex escape. 131 ResultChar = 0; 132 if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 133 if (Diags) 134 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 135 diag::err_hex_escape_no_digits) << "x"; 136 HadError = 1; 137 break; 138 } 139 140 // Hex escapes are a maximal series of hex digits. 141 bool Overflow = false; 142 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) { 143 int CharVal = llvm::hexDigitValue(ThisTokBuf[0]); 144 if (CharVal == -1) break; 145 // About to shift out a digit? 146 Overflow |= (ResultChar & 0xF0000000) ? true : false; 147 ResultChar <<= 4; 148 ResultChar |= CharVal; 149 } 150 151 // See if any bits will be truncated when evaluated as a character. 152 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 153 Overflow = true; 154 ResultChar &= ~0U >> (32-CharWidth); 155 } 156 157 // Check for overflow. 158 if (Overflow && Diags) // Too many digits to fit in 159 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 160 diag::warn_hex_escape_too_large); 161 break; 162 } 163 case '0': case '1': case '2': case '3': 164 case '4': case '5': case '6': case '7': { 165 // Octal escapes. 166 --ThisTokBuf; 167 ResultChar = 0; 168 169 // Octal escapes are a series of octal digits with maximum length 3. 170 // "\0123" is a two digit sequence equal to "\012" "3". 171 unsigned NumDigits = 0; 172 do { 173 ResultChar <<= 3; 174 ResultChar |= *ThisTokBuf++ - '0'; 175 ++NumDigits; 176 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 && 177 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7'); 178 179 // Check for overflow. Reject '\777', but not L'\777'. 180 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 181 if (Diags) 182 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 183 diag::warn_octal_escape_too_large); 184 ResultChar &= ~0U >> (32-CharWidth); 185 } 186 break; 187 } 188 189 // Otherwise, these are not valid escapes. 190 case '(': case '{': case '[': case '%': 191 // GCC accepts these as extensions. We warn about them as such though. 192 if (Diags) 193 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 194 diag::ext_nonstandard_escape) 195 << std::string(1, ResultChar); 196 break; 197 default: 198 if (Diags == 0) 199 break; 200 201 if (isPrintable(ResultChar)) 202 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 203 diag::ext_unknown_escape) 204 << std::string(1, ResultChar); 205 else 206 Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf, 207 diag::ext_unknown_escape) 208 << "x" + llvm::utohexstr(ResultChar); 209 break; 210 } 211 212 return ResultChar; 213} 214 215/// ProcessUCNEscape - Read the Universal Character Name, check constraints and 216/// return the UTF32. 217static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 218 const char *ThisTokEnd, 219 uint32_t &UcnVal, unsigned short &UcnLen, 220 FullSourceLoc Loc, DiagnosticsEngine *Diags, 221 const LangOptions &Features, 222 bool in_char_string_literal = false) { 223 const char *UcnBegin = ThisTokBuf; 224 225 // Skip the '\u' char's. 226 ThisTokBuf += 2; 227 228 if (ThisTokBuf == ThisTokEnd || !isHexDigit(*ThisTokBuf)) { 229 if (Diags) 230 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 231 diag::err_hex_escape_no_digits) << StringRef(&ThisTokBuf[-1], 1); 232 return false; 233 } 234 UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8); 235 unsigned short UcnLenSave = UcnLen; 236 for (; ThisTokBuf != ThisTokEnd && UcnLenSave; ++ThisTokBuf, UcnLenSave--) { 237 int CharVal = llvm::hexDigitValue(ThisTokBuf[0]); 238 if (CharVal == -1) break; 239 UcnVal <<= 4; 240 UcnVal |= CharVal; 241 } 242 // If we didn't consume the proper number of digits, there is a problem. 243 if (UcnLenSave) { 244 if (Diags) 245 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 246 diag::err_ucn_escape_incomplete); 247 return false; 248 } 249 250 // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2] 251 if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints 252 UcnVal > 0x10FFFF) { // maximum legal UTF32 value 253 if (Diags) 254 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 255 diag::err_ucn_escape_invalid); 256 return false; 257 } 258 259 // C++11 allows UCNs that refer to control characters and basic source 260 // characters inside character and string literals 261 if (UcnVal < 0xa0 && 262 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) { // $, @, ` 263 bool IsError = (!Features.CPlusPlus11 || !in_char_string_literal); 264 if (Diags) { 265 char BasicSCSChar = UcnVal; 266 if (UcnVal >= 0x20 && UcnVal < 0x7f) 267 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 268 IsError ? diag::err_ucn_escape_basic_scs : 269 diag::warn_cxx98_compat_literal_ucn_escape_basic_scs) 270 << StringRef(&BasicSCSChar, 1); 271 else 272 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 273 IsError ? diag::err_ucn_control_character : 274 diag::warn_cxx98_compat_literal_ucn_control_character); 275 } 276 if (IsError) 277 return false; 278 } 279 280 if (!Features.CPlusPlus && !Features.C99 && Diags) 281 Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf, 282 diag::warn_ucn_not_valid_in_c89_literal); 283 284 return true; 285} 286 287/// MeasureUCNEscape - Determine the number of bytes within the resulting string 288/// which this UCN will occupy. 289static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 290 const char *ThisTokEnd, unsigned CharByteWidth, 291 const LangOptions &Features, bool &HadError) { 292 // UTF-32: 4 bytes per escape. 293 if (CharByteWidth == 4) 294 return 4; 295 296 uint32_t UcnVal = 0; 297 unsigned short UcnLen = 0; 298 FullSourceLoc Loc; 299 300 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, 301 UcnLen, Loc, 0, Features, true)) { 302 HadError = true; 303 return 0; 304 } 305 306 // UTF-16: 2 bytes for BMP, 4 bytes otherwise. 307 if (CharByteWidth == 2) 308 return UcnVal <= 0xFFFF ? 2 : 4; 309 310 // UTF-8. 311 if (UcnVal < 0x80) 312 return 1; 313 if (UcnVal < 0x800) 314 return 2; 315 if (UcnVal < 0x10000) 316 return 3; 317 return 4; 318} 319 320/// EncodeUCNEscape - Read the Universal Character Name, check constraints and 321/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of 322/// StringLiteralParser. When we decide to implement UCN's for identifiers, 323/// we will likely rework our support for UCN's. 324static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf, 325 const char *ThisTokEnd, 326 char *&ResultBuf, bool &HadError, 327 FullSourceLoc Loc, unsigned CharByteWidth, 328 DiagnosticsEngine *Diags, 329 const LangOptions &Features) { 330 typedef uint32_t UTF32; 331 UTF32 UcnVal = 0; 332 unsigned short UcnLen = 0; 333 if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen, 334 Loc, Diags, Features, true)) { 335 HadError = true; 336 return; 337 } 338 339 assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth) && 340 "only character widths of 1, 2, or 4 bytes supported"); 341 342 (void)UcnLen; 343 assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported"); 344 345 if (CharByteWidth == 4) { 346 // FIXME: Make the type of the result buffer correct instead of 347 // using reinterpret_cast. 348 UTF32 *ResultPtr = reinterpret_cast<UTF32*>(ResultBuf); 349 *ResultPtr = UcnVal; 350 ResultBuf += 4; 351 return; 352 } 353 354 if (CharByteWidth == 2) { 355 // FIXME: Make the type of the result buffer correct instead of 356 // using reinterpret_cast. 357 UTF16 *ResultPtr = reinterpret_cast<UTF16*>(ResultBuf); 358 359 if (UcnVal <= (UTF32)0xFFFF) { 360 *ResultPtr = UcnVal; 361 ResultBuf += 2; 362 return; 363 } 364 365 // Convert to UTF16. 366 UcnVal -= 0x10000; 367 *ResultPtr = 0xD800 + (UcnVal >> 10); 368 *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF); 369 ResultBuf += 4; 370 return; 371 } 372 373 assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters"); 374 375 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8. 376 // The conversion below was inspired by: 377 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c 378 // First, we determine how many bytes the result will require. 379 typedef uint8_t UTF8; 380 381 unsigned short bytesToWrite = 0; 382 if (UcnVal < (UTF32)0x80) 383 bytesToWrite = 1; 384 else if (UcnVal < (UTF32)0x800) 385 bytesToWrite = 2; 386 else if (UcnVal < (UTF32)0x10000) 387 bytesToWrite = 3; 388 else 389 bytesToWrite = 4; 390 391 const unsigned byteMask = 0xBF; 392 const unsigned byteMark = 0x80; 393 394 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed 395 // into the first byte, depending on how many bytes follow. 396 static const UTF8 firstByteMark[5] = { 397 0x00, 0x00, 0xC0, 0xE0, 0xF0 398 }; 399 // Finally, we write the bytes into ResultBuf. 400 ResultBuf += bytesToWrite; 401 switch (bytesToWrite) { // note: everything falls through. 402 case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 403 case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 404 case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 405 case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); 406 } 407 // Update the buffer. 408 ResultBuf += bytesToWrite; 409} 410 411 412/// integer-constant: [C99 6.4.4.1] 413/// decimal-constant integer-suffix 414/// octal-constant integer-suffix 415/// hexadecimal-constant integer-suffix 416/// user-defined-integer-literal: [C++11 lex.ext] 417/// decimal-literal ud-suffix 418/// octal-literal ud-suffix 419/// hexadecimal-literal ud-suffix 420/// decimal-constant: 421/// nonzero-digit 422/// decimal-constant digit 423/// octal-constant: 424/// 0 425/// octal-constant octal-digit 426/// hexadecimal-constant: 427/// hexadecimal-prefix hexadecimal-digit 428/// hexadecimal-constant hexadecimal-digit 429/// hexadecimal-prefix: one of 430/// 0x 0X 431/// integer-suffix: 432/// unsigned-suffix [long-suffix] 433/// unsigned-suffix [long-long-suffix] 434/// long-suffix [unsigned-suffix] 435/// long-long-suffix [unsigned-sufix] 436/// nonzero-digit: 437/// 1 2 3 4 5 6 7 8 9 438/// octal-digit: 439/// 0 1 2 3 4 5 6 7 440/// hexadecimal-digit: 441/// 0 1 2 3 4 5 6 7 8 9 442/// a b c d e f 443/// A B C D E F 444/// unsigned-suffix: one of 445/// u U 446/// long-suffix: one of 447/// l L 448/// long-long-suffix: one of 449/// ll LL 450/// 451/// floating-constant: [C99 6.4.4.2] 452/// TODO: add rules... 453/// 454NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling, 455 SourceLocation TokLoc, 456 Preprocessor &PP) 457 : PP(PP), ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) { 458 459 // This routine assumes that the range begin/end matches the regex for integer 460 // and FP constants (specifically, the 'pp-number' regex), and assumes that 461 // the byte at "*end" is both valid and not part of the regex. Because of 462 // this, it doesn't have to check for 'overscan' in various places. 463 assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?"); 464 465 s = DigitsBegin = ThisTokBegin; 466 saw_exponent = false; 467 saw_period = false; 468 saw_ud_suffix = false; 469 isLong = false; 470 isUnsigned = false; 471 isLongLong = false; 472 isFloat = false; 473 isImaginary = false; 474 isMicrosoftInteger = false; 475 hadError = false; 476 477 if (*s == '0') { // parse radix 478 ParseNumberStartingWithZero(TokLoc); 479 if (hadError) 480 return; 481 } else { // the first digit is non-zero 482 radix = 10; 483 s = SkipDigits(s); 484 if (s == ThisTokEnd) { 485 // Done. 486 } else if (isHexDigit(*s) && !(*s == 'e' || *s == 'E')) { 487 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), 488 diag::err_invalid_decimal_digit) << StringRef(s, 1); 489 hadError = true; 490 return; 491 } else if (*s == '.') { 492 s++; 493 saw_period = true; 494 s = SkipDigits(s); 495 } 496 if ((*s == 'e' || *s == 'E')) { // exponent 497 const char *Exponent = s; 498 s++; 499 saw_exponent = true; 500 if (*s == '+' || *s == '-') s++; // sign 501 const char *first_non_digit = SkipDigits(s); 502 if (first_non_digit != s) { 503 s = first_non_digit; 504 } else { 505 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent - ThisTokBegin), 506 diag::err_exponent_has_no_digits); 507 hadError = true; 508 return; 509 } 510 } 511 } 512 513 SuffixBegin = s; 514 515 // Parse the suffix. At this point we can classify whether we have an FP or 516 // integer constant. 517 bool isFPConstant = isFloatingLiteral(); 518 519 // Loop over all of the characters of the suffix. If we see something bad, 520 // we break out of the loop. 521 for (; s != ThisTokEnd; ++s) { 522 switch (*s) { 523 case 'f': // FP Suffix for "float" 524 case 'F': 525 if (!isFPConstant) break; // Error for integer constant. 526 if (isFloat || isLong) break; // FF, LF invalid. 527 isFloat = true; 528 continue; // Success. 529 case 'u': 530 case 'U': 531 if (isFPConstant) break; // Error for floating constant. 532 if (isUnsigned) break; // Cannot be repeated. 533 isUnsigned = true; 534 continue; // Success. 535 case 'l': 536 case 'L': 537 if (isLong || isLongLong) break; // Cannot be repeated. 538 if (isFloat) break; // LF invalid. 539 540 // Check for long long. The L's need to be adjacent and the same case. 541 if (s+1 != ThisTokEnd && s[1] == s[0]) { 542 if (isFPConstant) break; // long long invalid for floats. 543 isLongLong = true; 544 ++s; // Eat both of them. 545 } else { 546 isLong = true; 547 } 548 continue; // Success. 549 case 'i': 550 case 'I': 551 if (PP.getLangOpts().MicrosoftExt) { 552 if (isFPConstant || isLong || isLongLong) break; 553 554 // Allow i8, i16, i32, i64, and i128. 555 if (s + 1 != ThisTokEnd) { 556 switch (s[1]) { 557 case '8': 558 s += 2; // i8 suffix 559 isMicrosoftInteger = true; 560 break; 561 case '1': 562 if (s + 2 == ThisTokEnd) break; 563 if (s[2] == '6') { 564 s += 3; // i16 suffix 565 isMicrosoftInteger = true; 566 } 567 else if (s[2] == '2') { 568 if (s + 3 == ThisTokEnd) break; 569 if (s[3] == '8') { 570 s += 4; // i128 suffix 571 isMicrosoftInteger = true; 572 } 573 } 574 break; 575 case '3': 576 if (s + 2 == ThisTokEnd) break; 577 if (s[2] == '2') { 578 s += 3; // i32 suffix 579 isLong = true; 580 isMicrosoftInteger = true; 581 } 582 break; 583 case '6': 584 if (s + 2 == ThisTokEnd) break; 585 if (s[2] == '4') { 586 s += 3; // i64 suffix 587 isLongLong = true; 588 isMicrosoftInteger = true; 589 } 590 break; 591 default: 592 break; 593 } 594 break; 595 } 596 } 597 // fall through. 598 case 'j': 599 case 'J': 600 if (isImaginary) break; // Cannot be repeated. 601 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), 602 diag::ext_imaginary_constant); 603 isImaginary = true; 604 continue; // Success. 605 } 606 // If we reached here, there was an error or a ud-suffix. 607 break; 608 } 609 610 if (s != ThisTokEnd) { 611 if (PP.getLangOpts().CPlusPlus11 && s == SuffixBegin && *s == '_') { 612 // We have a ud-suffix! By C++11 [lex.ext]p10, ud-suffixes not starting 613 // with an '_' are ill-formed. 614 saw_ud_suffix = true; 615 return; 616 } 617 618 // Report an error if there are any. 619 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin), 620 isFPConstant ? diag::err_invalid_suffix_float_constant : 621 diag::err_invalid_suffix_integer_constant) 622 << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin); 623 hadError = true; 624 return; 625 } 626} 627 628/// ParseNumberStartingWithZero - This method is called when the first character 629/// of the number is found to be a zero. This means it is either an octal 630/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or 631/// a floating point number (01239.123e4). Eat the prefix, determining the 632/// radix etc. 633void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { 634 assert(s[0] == '0' && "Invalid method call"); 635 s++; 636 637 // Handle a hex number like 0x1234. 638 if ((*s == 'x' || *s == 'X') && (isHexDigit(s[1]) || s[1] == '.')) { 639 s++; 640 radix = 16; 641 DigitsBegin = s; 642 s = SkipHexDigits(s); 643 bool noSignificand = (s == DigitsBegin); 644 if (s == ThisTokEnd) { 645 // Done. 646 } else if (*s == '.') { 647 s++; 648 saw_period = true; 649 const char *floatDigitsBegin = s; 650 s = SkipHexDigits(s); 651 noSignificand &= (floatDigitsBegin == s); 652 } 653 654 if (noSignificand) { 655 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin), 656 diag::err_hexconstant_requires_digits); 657 hadError = true; 658 return; 659 } 660 661 // A binary exponent can appear with or with a '.'. If dotted, the 662 // binary exponent is required. 663 if (*s == 'p' || *s == 'P') { 664 const char *Exponent = s; 665 s++; 666 saw_exponent = true; 667 if (*s == '+' || *s == '-') s++; // sign 668 const char *first_non_digit = SkipDigits(s); 669 if (first_non_digit == s) { 670 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 671 diag::err_exponent_has_no_digits); 672 hadError = true; 673 return; 674 } 675 s = first_non_digit; 676 677 if (!PP.getLangOpts().HexFloats) 678 PP.Diag(TokLoc, diag::ext_hexconstant_invalid); 679 } else if (saw_period) { 680 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 681 diag::err_hexconstant_requires_exponent); 682 hadError = true; 683 } 684 return; 685 } 686 687 // Handle simple binary numbers 0b01010 688 if (*s == 'b' || *s == 'B') { 689 // 0b101010 is a C++1y / GCC extension. 690 PP.Diag(TokLoc, 691 PP.getLangOpts().CPlusPlus1y 692 ? diag::warn_cxx11_compat_binary_literal 693 : PP.getLangOpts().CPlusPlus 694 ? diag::ext_binary_literal_cxx1y 695 : diag::ext_binary_literal); 696 ++s; 697 radix = 2; 698 DigitsBegin = s; 699 s = SkipBinaryDigits(s); 700 if (s == ThisTokEnd) { 701 // Done. 702 } else if (isHexDigit(*s)) { 703 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 704 diag::err_invalid_binary_digit) << StringRef(s, 1); 705 hadError = true; 706 } 707 // Other suffixes will be diagnosed by the caller. 708 return; 709 } 710 711 // For now, the radix is set to 8. If we discover that we have a 712 // floating point constant, the radix will change to 10. Octal floating 713 // point constants are not permitted (only decimal and hexadecimal). 714 radix = 8; 715 DigitsBegin = s; 716 s = SkipOctalDigits(s); 717 if (s == ThisTokEnd) 718 return; // Done, simple octal number like 01234 719 720 // If we have some other non-octal digit that *is* a decimal digit, see if 721 // this is part of a floating point number like 094.123 or 09e1. 722 if (isDigit(*s)) { 723 const char *EndDecimal = SkipDigits(s); 724 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') { 725 s = EndDecimal; 726 radix = 10; 727 } 728 } 729 730 // If we have a hex digit other than 'e' (which denotes a FP exponent) then 731 // the code is using an incorrect base. 732 if (isHexDigit(*s) && *s != 'e' && *s != 'E') { 733 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 734 diag::err_invalid_octal_digit) << StringRef(s, 1); 735 hadError = true; 736 return; 737 } 738 739 if (*s == '.') { 740 s++; 741 radix = 10; 742 saw_period = true; 743 s = SkipDigits(s); // Skip suffix. 744 } 745 if (*s == 'e' || *s == 'E') { // exponent 746 const char *Exponent = s; 747 s++; 748 radix = 10; 749 saw_exponent = true; 750 if (*s == '+' || *s == '-') s++; // sign 751 const char *first_non_digit = SkipDigits(s); 752 if (first_non_digit != s) { 753 s = first_non_digit; 754 } else { 755 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 756 diag::err_exponent_has_no_digits); 757 hadError = true; 758 return; 759 } 760 } 761} 762 763static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) { 764 switch (Radix) { 765 case 2: 766 return NumDigits <= 64; 767 case 8: 768 return NumDigits <= 64 / 3; // Digits are groups of 3 bits. 769 case 10: 770 return NumDigits <= 19; // floor(log10(2^64)) 771 case 16: 772 return NumDigits <= 64 / 4; // Digits are groups of 4 bits. 773 default: 774 llvm_unreachable("impossible Radix"); 775 } 776} 777 778/// GetIntegerValue - Convert this numeric literal value to an APInt that 779/// matches Val's input width. If there is an overflow, set Val to the low bits 780/// of the result and return true. Otherwise, return false. 781bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) { 782 // Fast path: Compute a conservative bound on the maximum number of 783 // bits per digit in this radix. If we can't possibly overflow a 784 // uint64 based on that bound then do the simple conversion to 785 // integer. This avoids the expensive overflow checking below, and 786 // handles the common cases that matter (small decimal integers and 787 // hex/octal values which don't overflow). 788 const unsigned NumDigits = SuffixBegin - DigitsBegin; 789 if (alwaysFitsInto64Bits(radix, NumDigits)) { 790 uint64_t N = 0; 791 for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr) 792 N = N * radix + llvm::hexDigitValue(*Ptr); 793 794 // This will truncate the value to Val's input width. Simply check 795 // for overflow by comparing. 796 Val = N; 797 return Val.getZExtValue() != N; 798 } 799 800 Val = 0; 801 const char *Ptr = DigitsBegin; 802 803 llvm::APInt RadixVal(Val.getBitWidth(), radix); 804 llvm::APInt CharVal(Val.getBitWidth(), 0); 805 llvm::APInt OldVal = Val; 806 807 bool OverflowOccurred = false; 808 while (Ptr < SuffixBegin) { 809 unsigned C = llvm::hexDigitValue(*Ptr++); 810 811 // If this letter is out of bound for this radix, reject it. 812 assert(C < radix && "NumericLiteralParser ctor should have rejected this"); 813 814 CharVal = C; 815 816 // Add the digit to the value in the appropriate radix. If adding in digits 817 // made the value smaller, then this overflowed. 818 OldVal = Val; 819 820 // Multiply by radix, did overflow occur on the multiply? 821 Val *= RadixVal; 822 OverflowOccurred |= Val.udiv(RadixVal) != OldVal; 823 824 // Add value, did overflow occur on the value? 825 // (a + b) ult b <=> overflow 826 Val += CharVal; 827 OverflowOccurred |= Val.ult(CharVal); 828 } 829 return OverflowOccurred; 830} 831 832llvm::APFloat::opStatus 833NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) { 834 using llvm::APFloat; 835 836 unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin); 837 return Result.convertFromString(StringRef(ThisTokBegin, n), 838 APFloat::rmNearestTiesToEven); 839} 840 841 842/// \verbatim 843/// user-defined-character-literal: [C++11 lex.ext] 844/// character-literal ud-suffix 845/// ud-suffix: 846/// identifier 847/// character-literal: [C++11 lex.ccon] 848/// ' c-char-sequence ' 849/// u' c-char-sequence ' 850/// U' c-char-sequence ' 851/// L' c-char-sequence ' 852/// c-char-sequence: 853/// c-char 854/// c-char-sequence c-char 855/// c-char: 856/// any member of the source character set except the single-quote ', 857/// backslash \, or new-line character 858/// escape-sequence 859/// universal-character-name 860/// escape-sequence: 861/// simple-escape-sequence 862/// octal-escape-sequence 863/// hexadecimal-escape-sequence 864/// simple-escape-sequence: 865/// one of \' \" \? \\ \a \b \f \n \r \t \v 866/// octal-escape-sequence: 867/// \ octal-digit 868/// \ octal-digit octal-digit 869/// \ octal-digit octal-digit octal-digit 870/// hexadecimal-escape-sequence: 871/// \x hexadecimal-digit 872/// hexadecimal-escape-sequence hexadecimal-digit 873/// universal-character-name: [C++11 lex.charset] 874/// \u hex-quad 875/// \U hex-quad hex-quad 876/// hex-quad: 877/// hex-digit hex-digit hex-digit hex-digit 878/// \endverbatim 879/// 880CharLiteralParser::CharLiteralParser(const char *begin, const char *end, 881 SourceLocation Loc, Preprocessor &PP, 882 tok::TokenKind kind) { 883 // At this point we know that the character matches the regex "(L|u|U)?'.*'". 884 HadError = false; 885 886 Kind = kind; 887 888 const char *TokBegin = begin; 889 890 // Skip over wide character determinant. 891 if (Kind != tok::char_constant) { 892 ++begin; 893 } 894 895 // Skip over the entry quote. 896 assert(begin[0] == '\'' && "Invalid token lexed"); 897 ++begin; 898 899 // Remove an optional ud-suffix. 900 if (end[-1] != '\'') { 901 const char *UDSuffixEnd = end; 902 do { 903 --end; 904 } while (end[-1] != '\''); 905 UDSuffixBuf.assign(end, UDSuffixEnd); 906 UDSuffixOffset = end - TokBegin; 907 } 908 909 // Trim the ending quote. 910 assert(end != begin && "Invalid token lexed"); 911 --end; 912 913 // FIXME: The "Value" is an uint64_t so we can handle char literals of 914 // up to 64-bits. 915 // FIXME: This extensively assumes that 'char' is 8-bits. 916 assert(PP.getTargetInfo().getCharWidth() == 8 && 917 "Assumes char is 8 bits"); 918 assert(PP.getTargetInfo().getIntWidth() <= 64 && 919 (PP.getTargetInfo().getIntWidth() & 7) == 0 && 920 "Assumes sizeof(int) on target is <= 64 and a multiple of char"); 921 assert(PP.getTargetInfo().getWCharWidth() <= 64 && 922 "Assumes sizeof(wchar) on target is <= 64"); 923 924 SmallVector<uint32_t,4> codepoint_buffer; 925 codepoint_buffer.resize(end-begin); 926 uint32_t *buffer_begin = &codepoint_buffer.front(); 927 uint32_t *buffer_end = buffer_begin + codepoint_buffer.size(); 928 929 // Unicode escapes representing characters that cannot be correctly 930 // represented in a single code unit are disallowed in character literals 931 // by this implementation. 932 uint32_t largest_character_for_kind; 933 if (tok::wide_char_constant == Kind) { 934 largest_character_for_kind = 0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth()); 935 } else if (tok::utf16_char_constant == Kind) { 936 largest_character_for_kind = 0xFFFF; 937 } else if (tok::utf32_char_constant == Kind) { 938 largest_character_for_kind = 0x10FFFF; 939 } else { 940 largest_character_for_kind = 0x7Fu; 941 } 942 943 while (begin!=end) { 944 // Is this a span of non-escape characters? 945 if (begin[0] != '\\') { 946 char const *start = begin; 947 do { 948 ++begin; 949 } while (begin != end && *begin != '\\'); 950 951 char const *tmp_in_start = start; 952 uint32_t *tmp_out_start = buffer_begin; 953 ConversionResult res = 954 ConvertUTF8toUTF32(reinterpret_cast<UTF8 const **>(&start), 955 reinterpret_cast<UTF8 const *>(begin), 956 &buffer_begin,buffer_end,strictConversion); 957 if (res!=conversionOK) { 958 // If we see bad encoding for unprefixed character literals, warn and 959 // simply copy the byte values, for compatibility with gcc and 960 // older versions of clang. 961 bool NoErrorOnBadEncoding = isAscii(); 962 unsigned Msg = diag::err_bad_character_encoding; 963 if (NoErrorOnBadEncoding) 964 Msg = diag::warn_bad_character_encoding; 965 PP.Diag(Loc, Msg); 966 if (NoErrorOnBadEncoding) { 967 start = tmp_in_start; 968 buffer_begin = tmp_out_start; 969 for ( ; start != begin; ++start, ++buffer_begin) 970 *buffer_begin = static_cast<uint8_t>(*start); 971 } else { 972 HadError = true; 973 } 974 } else { 975 for (; tmp_out_start <buffer_begin; ++tmp_out_start) { 976 if (*tmp_out_start > largest_character_for_kind) { 977 HadError = true; 978 PP.Diag(Loc, diag::err_character_too_large); 979 } 980 } 981 } 982 983 continue; 984 } 985 // Is this a Universal Character Name excape? 986 if (begin[1] == 'u' || begin[1] == 'U') { 987 unsigned short UcnLen = 0; 988 if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen, 989 FullSourceLoc(Loc, PP.getSourceManager()), 990 &PP.getDiagnostics(), PP.getLangOpts(), 991 true)) 992 { 993 HadError = true; 994 } else if (*buffer_begin > largest_character_for_kind) { 995 HadError = true; 996 PP.Diag(Loc, diag::err_character_too_large); 997 } 998 999 ++buffer_begin; 1000 continue; 1001 } 1002 unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo()); 1003 uint64_t result = 1004 ProcessCharEscape(TokBegin, begin, end, HadError, 1005 FullSourceLoc(Loc,PP.getSourceManager()), 1006 CharWidth, &PP.getDiagnostics(), PP.getLangOpts()); 1007 *buffer_begin++ = result; 1008 } 1009 1010 unsigned NumCharsSoFar = buffer_begin-&codepoint_buffer.front(); 1011 1012 if (NumCharsSoFar > 1) { 1013 if (isWide()) 1014 PP.Diag(Loc, diag::warn_extraneous_char_constant); 1015 else if (isAscii() && NumCharsSoFar == 4) 1016 PP.Diag(Loc, diag::ext_four_char_character_literal); 1017 else if (isAscii()) 1018 PP.Diag(Loc, diag::ext_multichar_character_literal); 1019 else 1020 PP.Diag(Loc, diag::err_multichar_utf_character_literal); 1021 IsMultiChar = true; 1022 } else 1023 IsMultiChar = false; 1024 1025 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0); 1026 1027 // Narrow character literals act as though their value is concatenated 1028 // in this implementation, but warn on overflow. 1029 bool multi_char_too_long = false; 1030 if (isAscii() && isMultiChar()) { 1031 LitVal = 0; 1032 for (size_t i=0;i<NumCharsSoFar;++i) { 1033 // check for enough leading zeros to shift into 1034 multi_char_too_long |= (LitVal.countLeadingZeros() < 8); 1035 LitVal <<= 8; 1036 LitVal = LitVal + (codepoint_buffer[i] & 0xFF); 1037 } 1038 } else if (NumCharsSoFar > 0) { 1039 // otherwise just take the last character 1040 LitVal = buffer_begin[-1]; 1041 } 1042 1043 if (!HadError && multi_char_too_long) { 1044 PP.Diag(Loc,diag::warn_char_constant_too_large); 1045 } 1046 1047 // Transfer the value from APInt to uint64_t 1048 Value = LitVal.getZExtValue(); 1049 1050 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1") 1051 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple 1052 // character constants are not sign extended in the this implementation: 1053 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC. 1054 if (isAscii() && NumCharsSoFar == 1 && (Value & 128) && 1055 PP.getLangOpts().CharIsSigned) 1056 Value = (signed char)Value; 1057} 1058 1059/// \verbatim 1060/// string-literal: [C++0x lex.string] 1061/// encoding-prefix " [s-char-sequence] " 1062/// encoding-prefix R raw-string 1063/// encoding-prefix: 1064/// u8 1065/// u 1066/// U 1067/// L 1068/// s-char-sequence: 1069/// s-char 1070/// s-char-sequence s-char 1071/// s-char: 1072/// any member of the source character set except the double-quote ", 1073/// backslash \, or new-line character 1074/// escape-sequence 1075/// universal-character-name 1076/// raw-string: 1077/// " d-char-sequence ( r-char-sequence ) d-char-sequence " 1078/// r-char-sequence: 1079/// r-char 1080/// r-char-sequence r-char 1081/// r-char: 1082/// any member of the source character set, except a right parenthesis ) 1083/// followed by the initial d-char-sequence (which may be empty) 1084/// followed by a double quote ". 1085/// d-char-sequence: 1086/// d-char 1087/// d-char-sequence d-char 1088/// d-char: 1089/// any member of the basic source character set except: 1090/// space, the left parenthesis (, the right parenthesis ), 1091/// the backslash \, and the control characters representing horizontal 1092/// tab, vertical tab, form feed, and newline. 1093/// escape-sequence: [C++0x lex.ccon] 1094/// simple-escape-sequence 1095/// octal-escape-sequence 1096/// hexadecimal-escape-sequence 1097/// simple-escape-sequence: 1098/// one of \' \" \? \\ \a \b \f \n \r \t \v 1099/// octal-escape-sequence: 1100/// \ octal-digit 1101/// \ octal-digit octal-digit 1102/// \ octal-digit octal-digit octal-digit 1103/// hexadecimal-escape-sequence: 1104/// \x hexadecimal-digit 1105/// hexadecimal-escape-sequence hexadecimal-digit 1106/// universal-character-name: 1107/// \u hex-quad 1108/// \U hex-quad hex-quad 1109/// hex-quad: 1110/// hex-digit hex-digit hex-digit hex-digit 1111/// \endverbatim 1112/// 1113StringLiteralParser:: 1114StringLiteralParser(const Token *StringToks, unsigned NumStringToks, 1115 Preprocessor &PP, bool Complain) 1116 : SM(PP.getSourceManager()), Features(PP.getLangOpts()), 1117 Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() : 0), 1118 MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown), 1119 ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) { 1120 init(StringToks, NumStringToks); 1121} 1122 1123void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){ 1124 // The literal token may have come from an invalid source location (e.g. due 1125 // to a PCH error), in which case the token length will be 0. 1126 if (NumStringToks == 0 || StringToks[0].getLength() < 2) 1127 return DiagnoseLexingError(SourceLocation()); 1128 1129 // Scan all of the string portions, remember the max individual token length, 1130 // computing a bound on the concatenated string length, and see whether any 1131 // piece is a wide-string. If any of the string portions is a wide-string 1132 // literal, the result is a wide-string literal [C99 6.4.5p4]. 1133 assert(NumStringToks && "expected at least one token"); 1134 MaxTokenLength = StringToks[0].getLength(); 1135 assert(StringToks[0].getLength() >= 2 && "literal token is invalid!"); 1136 SizeBound = StringToks[0].getLength()-2; // -2 for "". 1137 Kind = StringToks[0].getKind(); 1138 1139 hadError = false; 1140 1141 // Implement Translation Phase #6: concatenation of string literals 1142 /// (C99 5.1.1.2p1). The common case is only one string fragment. 1143 for (unsigned i = 1; i != NumStringToks; ++i) { 1144 if (StringToks[i].getLength() < 2) 1145 return DiagnoseLexingError(StringToks[i].getLocation()); 1146 1147 // The string could be shorter than this if it needs cleaning, but this is a 1148 // reasonable bound, which is all we need. 1149 assert(StringToks[i].getLength() >= 2 && "literal token is invalid!"); 1150 SizeBound += StringToks[i].getLength()-2; // -2 for "". 1151 1152 // Remember maximum string piece length. 1153 if (StringToks[i].getLength() > MaxTokenLength) 1154 MaxTokenLength = StringToks[i].getLength(); 1155 1156 // Remember if we see any wide or utf-8/16/32 strings. 1157 // Also check for illegal concatenations. 1158 if (StringToks[i].isNot(Kind) && StringToks[i].isNot(tok::string_literal)) { 1159 if (isAscii()) { 1160 Kind = StringToks[i].getKind(); 1161 } else { 1162 if (Diags) 1163 Diags->Report(StringToks[i].getLocation(), 1164 diag::err_unsupported_string_concat); 1165 hadError = true; 1166 } 1167 } 1168 } 1169 1170 // Include space for the null terminator. 1171 ++SizeBound; 1172 1173 // TODO: K&R warning: "traditional C rejects string constant concatenation" 1174 1175 // Get the width in bytes of char/wchar_t/char16_t/char32_t 1176 CharByteWidth = getCharWidth(Kind, Target); 1177 assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple"); 1178 CharByteWidth /= 8; 1179 1180 // The output buffer size needs to be large enough to hold wide characters. 1181 // This is a worst-case assumption which basically corresponds to L"" "long". 1182 SizeBound *= CharByteWidth; 1183 1184 // Size the temporary buffer to hold the result string data. 1185 ResultBuf.resize(SizeBound); 1186 1187 // Likewise, but for each string piece. 1188 SmallString<512> TokenBuf; 1189 TokenBuf.resize(MaxTokenLength); 1190 1191 // Loop over all the strings, getting their spelling, and expanding them to 1192 // wide strings as appropriate. 1193 ResultPtr = &ResultBuf[0]; // Next byte to fill in. 1194 1195 Pascal = false; 1196 1197 SourceLocation UDSuffixTokLoc; 1198 1199 for (unsigned i = 0, e = NumStringToks; i != e; ++i) { 1200 const char *ThisTokBuf = &TokenBuf[0]; 1201 // Get the spelling of the token, which eliminates trigraphs, etc. We know 1202 // that ThisTokBuf points to a buffer that is big enough for the whole token 1203 // and 'spelled' tokens can only shrink. 1204 bool StringInvalid = false; 1205 unsigned ThisTokLen = 1206 Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features, 1207 &StringInvalid); 1208 if (StringInvalid) 1209 return DiagnoseLexingError(StringToks[i].getLocation()); 1210 1211 const char *ThisTokBegin = ThisTokBuf; 1212 const char *ThisTokEnd = ThisTokBuf+ThisTokLen; 1213 1214 // Remove an optional ud-suffix. 1215 if (ThisTokEnd[-1] != '"') { 1216 const char *UDSuffixEnd = ThisTokEnd; 1217 do { 1218 --ThisTokEnd; 1219 } while (ThisTokEnd[-1] != '"'); 1220 1221 StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd); 1222 1223 if (UDSuffixBuf.empty()) { 1224 UDSuffixBuf.assign(UDSuffix); 1225 UDSuffixToken = i; 1226 UDSuffixOffset = ThisTokEnd - ThisTokBuf; 1227 UDSuffixTokLoc = StringToks[i].getLocation(); 1228 } else if (!UDSuffixBuf.equals(UDSuffix)) { 1229 // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the 1230 // result of a concatenation involving at least one user-defined-string- 1231 // literal, all the participating user-defined-string-literals shall 1232 // have the same ud-suffix. 1233 if (Diags) { 1234 SourceLocation TokLoc = StringToks[i].getLocation(); 1235 Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix) 1236 << UDSuffixBuf << UDSuffix 1237 << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc) 1238 << SourceRange(TokLoc, TokLoc); 1239 } 1240 hadError = true; 1241 } 1242 } 1243 1244 // Strip the end quote. 1245 --ThisTokEnd; 1246 1247 // TODO: Input character set mapping support. 1248 1249 // Skip marker for wide or unicode strings. 1250 if (ThisTokBuf[0] == 'L' || ThisTokBuf[0] == 'u' || ThisTokBuf[0] == 'U') { 1251 ++ThisTokBuf; 1252 // Skip 8 of u8 marker for utf8 strings. 1253 if (ThisTokBuf[0] == '8') 1254 ++ThisTokBuf; 1255 } 1256 1257 // Check for raw string 1258 if (ThisTokBuf[0] == 'R') { 1259 ThisTokBuf += 2; // skip R" 1260 1261 const char *Prefix = ThisTokBuf; 1262 while (ThisTokBuf[0] != '(') 1263 ++ThisTokBuf; 1264 ++ThisTokBuf; // skip '(' 1265 1266 // Remove same number of characters from the end 1267 ThisTokEnd -= ThisTokBuf - Prefix; 1268 assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal"); 1269 1270 // Copy the string over 1271 if (CopyStringFragment(StringToks[i], ThisTokBegin, 1272 StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf))) 1273 hadError = true; 1274 } else { 1275 if (ThisTokBuf[0] != '"') { 1276 // The file may have come from PCH and then changed after loading the 1277 // PCH; Fail gracefully. 1278 return DiagnoseLexingError(StringToks[i].getLocation()); 1279 } 1280 ++ThisTokBuf; // skip " 1281 1282 // Check if this is a pascal string 1283 if (Features.PascalStrings && ThisTokBuf + 1 != ThisTokEnd && 1284 ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') { 1285 1286 // If the \p sequence is found in the first token, we have a pascal string 1287 // Otherwise, if we already have a pascal string, ignore the first \p 1288 if (i == 0) { 1289 ++ThisTokBuf; 1290 Pascal = true; 1291 } else if (Pascal) 1292 ThisTokBuf += 2; 1293 } 1294 1295 while (ThisTokBuf != ThisTokEnd) { 1296 // Is this a span of non-escape characters? 1297 if (ThisTokBuf[0] != '\\') { 1298 const char *InStart = ThisTokBuf; 1299 do { 1300 ++ThisTokBuf; 1301 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\'); 1302 1303 // Copy the character span over. 1304 if (CopyStringFragment(StringToks[i], ThisTokBegin, 1305 StringRef(InStart, ThisTokBuf - InStart))) 1306 hadError = true; 1307 continue; 1308 } 1309 // Is this a Universal Character Name escape? 1310 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') { 1311 EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, 1312 ResultPtr, hadError, 1313 FullSourceLoc(StringToks[i].getLocation(), SM), 1314 CharByteWidth, Diags, Features); 1315 continue; 1316 } 1317 // Otherwise, this is a non-UCN escape character. Process it. 1318 unsigned ResultChar = 1319 ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError, 1320 FullSourceLoc(StringToks[i].getLocation(), SM), 1321 CharByteWidth*8, Diags, Features); 1322 1323 if (CharByteWidth == 4) { 1324 // FIXME: Make the type of the result buffer correct instead of 1325 // using reinterpret_cast. 1326 UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultPtr); 1327 *ResultWidePtr = ResultChar; 1328 ResultPtr += 4; 1329 } else if (CharByteWidth == 2) { 1330 // FIXME: Make the type of the result buffer correct instead of 1331 // using reinterpret_cast. 1332 UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultPtr); 1333 *ResultWidePtr = ResultChar & 0xFFFF; 1334 ResultPtr += 2; 1335 } else { 1336 assert(CharByteWidth == 1 && "Unexpected char width"); 1337 *ResultPtr++ = ResultChar & 0xFF; 1338 } 1339 } 1340 } 1341 } 1342 1343 if (Pascal) { 1344 if (CharByteWidth == 4) { 1345 // FIXME: Make the type of the result buffer correct instead of 1346 // using reinterpret_cast. 1347 UTF32 *ResultWidePtr = reinterpret_cast<UTF32*>(ResultBuf.data()); 1348 ResultWidePtr[0] = GetNumStringChars() - 1; 1349 } else if (CharByteWidth == 2) { 1350 // FIXME: Make the type of the result buffer correct instead of 1351 // using reinterpret_cast. 1352 UTF16 *ResultWidePtr = reinterpret_cast<UTF16*>(ResultBuf.data()); 1353 ResultWidePtr[0] = GetNumStringChars() - 1; 1354 } else { 1355 assert(CharByteWidth == 1 && "Unexpected char width"); 1356 ResultBuf[0] = GetNumStringChars() - 1; 1357 } 1358 1359 // Verify that pascal strings aren't too large. 1360 if (GetStringLength() > 256) { 1361 if (Diags) 1362 Diags->Report(StringToks[0].getLocation(), 1363 diag::err_pascal_string_too_long) 1364 << SourceRange(StringToks[0].getLocation(), 1365 StringToks[NumStringToks-1].getLocation()); 1366 hadError = true; 1367 return; 1368 } 1369 } else if (Diags) { 1370 // Complain if this string literal has too many characters. 1371 unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509; 1372 1373 if (GetNumStringChars() > MaxChars) 1374 Diags->Report(StringToks[0].getLocation(), 1375 diag::ext_string_too_long) 1376 << GetNumStringChars() << MaxChars 1377 << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0) 1378 << SourceRange(StringToks[0].getLocation(), 1379 StringToks[NumStringToks-1].getLocation()); 1380 } 1381} 1382 1383static const char *resyncUTF8(const char *Err, const char *End) { 1384 if (Err == End) 1385 return End; 1386 End = Err + std::min<unsigned>(getNumBytesForUTF8(*Err), End-Err); 1387 while (++Err != End && (*Err & 0xC0) == 0x80) 1388 ; 1389 return Err; 1390} 1391 1392/// \brief This function copies from Fragment, which is a sequence of bytes 1393/// within Tok's contents (which begin at TokBegin) into ResultPtr. 1394/// Performs widening for multi-byte characters. 1395bool StringLiteralParser::CopyStringFragment(const Token &Tok, 1396 const char *TokBegin, 1397 StringRef Fragment) { 1398 const UTF8 *ErrorPtrTmp; 1399 if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp)) 1400 return false; 1401 1402 // If we see bad encoding for unprefixed string literals, warn and 1403 // simply copy the byte values, for compatibility with gcc and older 1404 // versions of clang. 1405 bool NoErrorOnBadEncoding = isAscii(); 1406 if (NoErrorOnBadEncoding) { 1407 memcpy(ResultPtr, Fragment.data(), Fragment.size()); 1408 ResultPtr += Fragment.size(); 1409 } 1410 1411 if (Diags) { 1412 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 1413 1414 FullSourceLoc SourceLoc(Tok.getLocation(), SM); 1415 const DiagnosticBuilder &Builder = 1416 Diag(Diags, Features, SourceLoc, TokBegin, 1417 ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()), 1418 NoErrorOnBadEncoding ? diag::warn_bad_string_encoding 1419 : diag::err_bad_string_encoding); 1420 1421 const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 1422 StringRef NextFragment(NextStart, Fragment.end()-NextStart); 1423 1424 // Decode into a dummy buffer. 1425 SmallString<512> Dummy; 1426 Dummy.reserve(Fragment.size() * CharByteWidth); 1427 char *Ptr = Dummy.data(); 1428 1429 while (!Builder.hasMaxRanges() && 1430 !ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) { 1431 const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp); 1432 NextStart = resyncUTF8(ErrorPtr, Fragment.end()); 1433 Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin, 1434 ErrorPtr, NextStart); 1435 NextFragment = StringRef(NextStart, Fragment.end()-NextStart); 1436 } 1437 } 1438 return !NoErrorOnBadEncoding; 1439} 1440 1441void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) { 1442 hadError = true; 1443 if (Diags) 1444 Diags->Report(Loc, diag::err_lexing_string); 1445} 1446 1447/// getOffsetOfStringByte - This function returns the offset of the 1448/// specified byte of the string data represented by Token. This handles 1449/// advancing over escape sequences in the string. 1450unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok, 1451 unsigned ByteNo) const { 1452 // Get the spelling of the token. 1453 SmallString<32> SpellingBuffer; 1454 SpellingBuffer.resize(Tok.getLength()); 1455 1456 bool StringInvalid = false; 1457 const char *SpellingPtr = &SpellingBuffer[0]; 1458 unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features, 1459 &StringInvalid); 1460 if (StringInvalid) 1461 return 0; 1462 1463 const char *SpellingStart = SpellingPtr; 1464 const char *SpellingEnd = SpellingPtr+TokLen; 1465 1466 // Handle UTF-8 strings just like narrow strings. 1467 if (SpellingPtr[0] == 'u' && SpellingPtr[1] == '8') 1468 SpellingPtr += 2; 1469 1470 assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' && 1471 SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet"); 1472 1473 // For raw string literals, this is easy. 1474 if (SpellingPtr[0] == 'R') { 1475 assert(SpellingPtr[1] == '"' && "Should be a raw string literal!"); 1476 // Skip 'R"'. 1477 SpellingPtr += 2; 1478 while (*SpellingPtr != '(') { 1479 ++SpellingPtr; 1480 assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal"); 1481 } 1482 // Skip '('. 1483 ++SpellingPtr; 1484 return SpellingPtr - SpellingStart + ByteNo; 1485 } 1486 1487 // Skip over the leading quote 1488 assert(SpellingPtr[0] == '"' && "Should be a string literal!"); 1489 ++SpellingPtr; 1490 1491 // Skip over bytes until we find the offset we're looking for. 1492 while (ByteNo) { 1493 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!"); 1494 1495 // Step over non-escapes simply. 1496 if (*SpellingPtr != '\\') { 1497 ++SpellingPtr; 1498 --ByteNo; 1499 continue; 1500 } 1501 1502 // Otherwise, this is an escape character. Advance over it. 1503 bool HadError = false; 1504 if (SpellingPtr[1] == 'u' || SpellingPtr[1] == 'U') { 1505 const char *EscapePtr = SpellingPtr; 1506 unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd, 1507 1, Features, HadError); 1508 if (Len > ByteNo) { 1509 // ByteNo is somewhere within the escape sequence. 1510 SpellingPtr = EscapePtr; 1511 break; 1512 } 1513 ByteNo -= Len; 1514 } else { 1515 ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError, 1516 FullSourceLoc(Tok.getLocation(), SM), 1517 CharByteWidth*8, Diags, Features); 1518 --ByteNo; 1519 } 1520 assert(!HadError && "This method isn't valid on erroneous strings"); 1521 } 1522 1523 return SpellingPtr-SpellingStart; 1524} 1525