LiteralSupport.cpp revision 193576
1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the NumericLiteralParser, CharLiteralParser, and 11// StringLiteralParser interfaces. 12// 13//===----------------------------------------------------------------------===// 14 15#include "clang/Lex/LiteralSupport.h" 16#include "clang/Lex/Preprocessor.h" 17#include "clang/Lex/LexDiagnostic.h" 18#include "clang/Basic/TargetInfo.h" 19#include "llvm/ADT/StringExtras.h" 20using namespace clang; 21 22/// HexDigitValue - Return the value of the specified hex digit, or -1 if it's 23/// not valid. 24static int HexDigitValue(char C) { 25 if (C >= '0' && C <= '9') return C-'0'; 26 if (C >= 'a' && C <= 'f') return C-'a'+10; 27 if (C >= 'A' && C <= 'F') return C-'A'+10; 28 return -1; 29} 30 31/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in 32/// either a character or a string literal. 33static unsigned ProcessCharEscape(const char *&ThisTokBuf, 34 const char *ThisTokEnd, bool &HadError, 35 SourceLocation Loc, bool IsWide, 36 Preprocessor &PP) { 37 // Skip the '\' char. 38 ++ThisTokBuf; 39 40 // We know that this character can't be off the end of the buffer, because 41 // that would have been \", which would not have been the end of string. 42 unsigned ResultChar = *ThisTokBuf++; 43 switch (ResultChar) { 44 // These map to themselves. 45 case '\\': case '\'': case '"': case '?': break; 46 47 // These have fixed mappings. 48 case 'a': 49 // TODO: K&R: the meaning of '\\a' is different in traditional C 50 ResultChar = 7; 51 break; 52 case 'b': 53 ResultChar = 8; 54 break; 55 case 'e': 56 PP.Diag(Loc, diag::ext_nonstandard_escape) << "e"; 57 ResultChar = 27; 58 break; 59 case 'f': 60 ResultChar = 12; 61 break; 62 case 'n': 63 ResultChar = 10; 64 break; 65 case 'r': 66 ResultChar = 13; 67 break; 68 case 't': 69 ResultChar = 9; 70 break; 71 case 'v': 72 ResultChar = 11; 73 break; 74 case 'x': { // Hex escape. 75 ResultChar = 0; 76 if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) { 77 PP.Diag(Loc, diag::err_hex_escape_no_digits); 78 HadError = 1; 79 break; 80 } 81 82 // Hex escapes are a maximal series of hex digits. 83 bool Overflow = false; 84 for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) { 85 int CharVal = HexDigitValue(ThisTokBuf[0]); 86 if (CharVal == -1) break; 87 // About to shift out a digit? 88 Overflow |= (ResultChar & 0xF0000000) ? true : false; 89 ResultChar <<= 4; 90 ResultChar |= CharVal; 91 } 92 93 // See if any bits will be truncated when evaluated as a character. 94 unsigned CharWidth = PP.getTargetInfo().getCharWidth(IsWide); 95 96 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 97 Overflow = true; 98 ResultChar &= ~0U >> (32-CharWidth); 99 } 100 101 // Check for overflow. 102 if (Overflow) // Too many digits to fit in 103 PP.Diag(Loc, diag::warn_hex_escape_too_large); 104 break; 105 } 106 case '0': case '1': case '2': case '3': 107 case '4': case '5': case '6': case '7': { 108 // Octal escapes. 109 --ThisTokBuf; 110 ResultChar = 0; 111 112 // Octal escapes are a series of octal digits with maximum length 3. 113 // "\0123" is a two digit sequence equal to "\012" "3". 114 unsigned NumDigits = 0; 115 do { 116 ResultChar <<= 3; 117 ResultChar |= *ThisTokBuf++ - '0'; 118 ++NumDigits; 119 } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 && 120 ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7'); 121 122 // Check for overflow. Reject '\777', but not L'\777'. 123 unsigned CharWidth = PP.getTargetInfo().getCharWidth(IsWide); 124 125 if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) { 126 PP.Diag(Loc, diag::warn_octal_escape_too_large); 127 ResultChar &= ~0U >> (32-CharWidth); 128 } 129 break; 130 } 131 132 // Otherwise, these are not valid escapes. 133 case '(': case '{': case '[': case '%': 134 // GCC accepts these as extensions. We warn about them as such though. 135 PP.Diag(Loc, diag::ext_nonstandard_escape) 136 << std::string()+(char)ResultChar; 137 break; 138 // FALL THROUGH. 139 default: 140 if (isgraph(ThisTokBuf[0])) 141 PP.Diag(Loc, diag::ext_unknown_escape) << std::string()+(char)ResultChar; 142 else 143 PP.Diag(Loc, diag::ext_unknown_escape) << "x"+llvm::utohexstr(ResultChar); 144 break; 145 } 146 147 return ResultChar; 148} 149 150/// ProcessUCNEscape - Read the Universal Character Name, check constraints and 151/// convert the UTF32 to UTF8. This is a subroutine of StringLiteralParser. 152/// When we decide to implement UCN's for character constants and identifiers, 153/// we will likely rework our support for UCN's. 154static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd, 155 char *&ResultBuf, bool &HadError, 156 SourceLocation Loc, bool IsWide, Preprocessor &PP) 157{ 158 // FIXME: Add a warning - UCN's are only valid in C++ & C99. 159 // FIXME: Handle wide strings. 160 161 // Save the beginning of the string (for error diagnostics). 162 const char *ThisTokBegin = ThisTokBuf; 163 164 // Skip the '\u' char's. 165 ThisTokBuf += 2; 166 167 if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) { 168 PP.Diag(Loc, diag::err_ucn_escape_no_digits); 169 HadError = 1; 170 return; 171 } 172 typedef uint32_t UTF32; 173 174 UTF32 UcnVal = 0; 175 unsigned short UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8); 176 for (; ThisTokBuf != ThisTokEnd && UcnLen; ++ThisTokBuf, UcnLen--) { 177 int CharVal = HexDigitValue(ThisTokBuf[0]); 178 if (CharVal == -1) break; 179 UcnVal <<= 4; 180 UcnVal |= CharVal; 181 } 182 // If we didn't consume the proper number of digits, there is a problem. 183 if (UcnLen) { 184 PP.Diag(PP.AdvanceToTokenCharacter(Loc, ThisTokBuf-ThisTokBegin), 185 diag::err_ucn_escape_incomplete); 186 HadError = 1; 187 return; 188 } 189 // Check UCN constraints (C99 6.4.3p2). 190 if ((UcnVal < 0xa0 && 191 (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60 )) // $, @, ` 192 || (UcnVal >= 0xD800 && UcnVal <= 0xDFFF) 193 || (UcnVal > 0x10FFFF)) /* the maximum legal UTF32 value */ { 194 PP.Diag(Loc, diag::err_ucn_escape_invalid); 195 HadError = 1; 196 return; 197 } 198 // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8. 199 // The conversion below was inspired by: 200 // http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c 201 // First, we determine how many bytes the result will require. 202 typedef uint8_t UTF8; 203 204 unsigned short bytesToWrite = 0; 205 if (UcnVal < (UTF32)0x80) 206 bytesToWrite = 1; 207 else if (UcnVal < (UTF32)0x800) 208 bytesToWrite = 2; 209 else if (UcnVal < (UTF32)0x10000) 210 bytesToWrite = 3; 211 else 212 bytesToWrite = 4; 213 214 const unsigned byteMask = 0xBF; 215 const unsigned byteMark = 0x80; 216 217 // Once the bits are split out into bytes of UTF8, this is a mask OR-ed 218 // into the first byte, depending on how many bytes follow. 219 static const UTF8 firstByteMark[5] = { 220 0x00, 0x00, 0xC0, 0xE0, 0xF0 221 }; 222 // Finally, we write the bytes into ResultBuf. 223 ResultBuf += bytesToWrite; 224 switch (bytesToWrite) { // note: everything falls through. 225 case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 226 case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 227 case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6; 228 case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]); 229 } 230 // Update the buffer. 231 ResultBuf += bytesToWrite; 232} 233 234 235/// integer-constant: [C99 6.4.4.1] 236/// decimal-constant integer-suffix 237/// octal-constant integer-suffix 238/// hexadecimal-constant integer-suffix 239/// decimal-constant: 240/// nonzero-digit 241/// decimal-constant digit 242/// octal-constant: 243/// 0 244/// octal-constant octal-digit 245/// hexadecimal-constant: 246/// hexadecimal-prefix hexadecimal-digit 247/// hexadecimal-constant hexadecimal-digit 248/// hexadecimal-prefix: one of 249/// 0x 0X 250/// integer-suffix: 251/// unsigned-suffix [long-suffix] 252/// unsigned-suffix [long-long-suffix] 253/// long-suffix [unsigned-suffix] 254/// long-long-suffix [unsigned-sufix] 255/// nonzero-digit: 256/// 1 2 3 4 5 6 7 8 9 257/// octal-digit: 258/// 0 1 2 3 4 5 6 7 259/// hexadecimal-digit: 260/// 0 1 2 3 4 5 6 7 8 9 261/// a b c d e f 262/// A B C D E F 263/// unsigned-suffix: one of 264/// u U 265/// long-suffix: one of 266/// l L 267/// long-long-suffix: one of 268/// ll LL 269/// 270/// floating-constant: [C99 6.4.4.2] 271/// TODO: add rules... 272/// 273NumericLiteralParser:: 274NumericLiteralParser(const char *begin, const char *end, 275 SourceLocation TokLoc, Preprocessor &pp) 276 : PP(pp), ThisTokBegin(begin), ThisTokEnd(end) { 277 278 // This routine assumes that the range begin/end matches the regex for integer 279 // and FP constants (specifically, the 'pp-number' regex), and assumes that 280 // the byte at "*end" is both valid and not part of the regex. Because of 281 // this, it doesn't have to check for 'overscan' in various places. 282 assert(!isalnum(*end) && *end != '.' && *end != '_' && 283 "Lexer didn't maximally munch?"); 284 285 s = DigitsBegin = begin; 286 saw_exponent = false; 287 saw_period = false; 288 isLong = false; 289 isUnsigned = false; 290 isLongLong = false; 291 isFloat = false; 292 isImaginary = false; 293 hadError = false; 294 295 if (*s == '0') { // parse radix 296 ParseNumberStartingWithZero(TokLoc); 297 if (hadError) 298 return; 299 } else { // the first digit is non-zero 300 radix = 10; 301 s = SkipDigits(s); 302 if (s == ThisTokEnd) { 303 // Done. 304 } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) { 305 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin), 306 diag::err_invalid_decimal_digit) << std::string(s, s+1); 307 hadError = true; 308 return; 309 } else if (*s == '.') { 310 s++; 311 saw_period = true; 312 s = SkipDigits(s); 313 } 314 if ((*s == 'e' || *s == 'E')) { // exponent 315 const char *Exponent = s; 316 s++; 317 saw_exponent = true; 318 if (*s == '+' || *s == '-') s++; // sign 319 const char *first_non_digit = SkipDigits(s); 320 if (first_non_digit != s) { 321 s = first_non_digit; 322 } else { 323 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-begin), 324 diag::err_exponent_has_no_digits); 325 hadError = true; 326 return; 327 } 328 } 329 } 330 331 SuffixBegin = s; 332 333 // Parse the suffix. At this point we can classify whether we have an FP or 334 // integer constant. 335 bool isFPConstant = isFloatingLiteral(); 336 337 // Loop over all of the characters of the suffix. If we see something bad, 338 // we break out of the loop. 339 for (; s != ThisTokEnd; ++s) { 340 switch (*s) { 341 case 'f': // FP Suffix for "float" 342 case 'F': 343 if (!isFPConstant) break; // Error for integer constant. 344 if (isFloat || isLong) break; // FF, LF invalid. 345 isFloat = true; 346 continue; // Success. 347 case 'u': 348 case 'U': 349 if (isFPConstant) break; // Error for floating constant. 350 if (isUnsigned) break; // Cannot be repeated. 351 isUnsigned = true; 352 continue; // Success. 353 case 'l': 354 case 'L': 355 if (isLong || isLongLong) break; // Cannot be repeated. 356 if (isFloat) break; // LF invalid. 357 358 // Check for long long. The L's need to be adjacent and the same case. 359 if (s+1 != ThisTokEnd && s[1] == s[0]) { 360 if (isFPConstant) break; // long long invalid for floats. 361 isLongLong = true; 362 ++s; // Eat both of them. 363 } else { 364 isLong = true; 365 } 366 continue; // Success. 367 case 'i': 368 if (PP.getLangOptions().Microsoft) { 369 // Allow i8, i16, i32, i64, and i128. 370 if (++s == ThisTokEnd) break; 371 switch (*s) { 372 case '8': 373 s++; // i8 suffix 374 break; 375 case '1': 376 if (++s == ThisTokEnd) break; 377 if (*s == '6') s++; // i16 suffix 378 else if (*s == '2') { 379 if (++s == ThisTokEnd) break; 380 if (*s == '8') s++; // i128 suffix 381 } 382 break; 383 case '3': 384 if (++s == ThisTokEnd) break; 385 if (*s == '2') s++; // i32 suffix 386 break; 387 case '6': 388 if (++s == ThisTokEnd) break; 389 if (*s == '4') s++; // i64 suffix 390 break; 391 default: 392 break; 393 } 394 break; 395 } 396 // fall through. 397 case 'I': 398 case 'j': 399 case 'J': 400 if (isImaginary) break; // Cannot be repeated. 401 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin), 402 diag::ext_imaginary_constant); 403 isImaginary = true; 404 continue; // Success. 405 } 406 // If we reached here, there was an error. 407 break; 408 } 409 410 // Report an error if there are any. 411 if (s != ThisTokEnd) { 412 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin), 413 isFPConstant ? diag::err_invalid_suffix_float_constant : 414 diag::err_invalid_suffix_integer_constant) 415 << std::string(SuffixBegin, ThisTokEnd); 416 hadError = true; 417 return; 418 } 419} 420 421/// ParseNumberStartingWithZero - This method is called when the first character 422/// of the number is found to be a zero. This means it is either an octal 423/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or 424/// a floating point number (01239.123e4). Eat the prefix, determining the 425/// radix etc. 426void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) { 427 assert(s[0] == '0' && "Invalid method call"); 428 s++; 429 430 // Handle a hex number like 0x1234. 431 if ((*s == 'x' || *s == 'X') && (isxdigit(s[1]) || s[1] == '.')) { 432 s++; 433 radix = 16; 434 DigitsBegin = s; 435 s = SkipHexDigits(s); 436 if (s == ThisTokEnd) { 437 // Done. 438 } else if (*s == '.') { 439 s++; 440 saw_period = true; 441 s = SkipHexDigits(s); 442 } 443 // A binary exponent can appear with or with a '.'. If dotted, the 444 // binary exponent is required. 445 if (*s == 'p' || *s == 'P') { 446 const char *Exponent = s; 447 s++; 448 saw_exponent = true; 449 if (*s == '+' || *s == '-') s++; // sign 450 const char *first_non_digit = SkipDigits(s); 451 if (first_non_digit == s) { 452 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 453 diag::err_exponent_has_no_digits); 454 hadError = true; 455 return; 456 } 457 s = first_non_digit; 458 459 if (!PP.getLangOptions().HexFloats) 460 PP.Diag(TokLoc, diag::ext_hexconstant_invalid); 461 } else if (saw_period) { 462 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 463 diag::err_hexconstant_requires_exponent); 464 hadError = true; 465 } 466 return; 467 } 468 469 // Handle simple binary numbers 0b01010 470 if (*s == 'b' || *s == 'B') { 471 // 0b101010 is a GCC extension. 472 PP.Diag(TokLoc, diag::ext_binary_literal); 473 ++s; 474 radix = 2; 475 DigitsBegin = s; 476 s = SkipBinaryDigits(s); 477 if (s == ThisTokEnd) { 478 // Done. 479 } else if (isxdigit(*s)) { 480 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 481 diag::err_invalid_binary_digit) << std::string(s, s+1); 482 hadError = true; 483 } 484 // Other suffixes will be diagnosed by the caller. 485 return; 486 } 487 488 // For now, the radix is set to 8. If we discover that we have a 489 // floating point constant, the radix will change to 10. Octal floating 490 // point constants are not permitted (only decimal and hexadecimal). 491 radix = 8; 492 DigitsBegin = s; 493 s = SkipOctalDigits(s); 494 if (s == ThisTokEnd) 495 return; // Done, simple octal number like 01234 496 497 // If we have some other non-octal digit that *is* a decimal digit, see if 498 // this is part of a floating point number like 094.123 or 09e1. 499 if (isdigit(*s)) { 500 const char *EndDecimal = SkipDigits(s); 501 if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') { 502 s = EndDecimal; 503 radix = 10; 504 } 505 } 506 507 // If we have a hex digit other than 'e' (which denotes a FP exponent) then 508 // the code is using an incorrect base. 509 if (isxdigit(*s) && *s != 'e' && *s != 'E') { 510 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), 511 diag::err_invalid_octal_digit) << std::string(s, s+1); 512 hadError = true; 513 return; 514 } 515 516 if (*s == '.') { 517 s++; 518 radix = 10; 519 saw_period = true; 520 s = SkipDigits(s); // Skip suffix. 521 } 522 if (*s == 'e' || *s == 'E') { // exponent 523 const char *Exponent = s; 524 s++; 525 radix = 10; 526 saw_exponent = true; 527 if (*s == '+' || *s == '-') s++; // sign 528 const char *first_non_digit = SkipDigits(s); 529 if (first_non_digit != s) { 530 s = first_non_digit; 531 } else { 532 PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin), 533 diag::err_exponent_has_no_digits); 534 hadError = true; 535 return; 536 } 537 } 538} 539 540 541/// GetIntegerValue - Convert this numeric literal value to an APInt that 542/// matches Val's input width. If there is an overflow, set Val to the low bits 543/// of the result and return true. Otherwise, return false. 544bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) { 545 // Fast path: Compute a conservative bound on the maximum number of 546 // bits per digit in this radix. If we can't possibly overflow a 547 // uint64 based on that bound then do the simple conversion to 548 // integer. This avoids the expensive overflow checking below, and 549 // handles the common cases that matter (small decimal integers and 550 // hex/octal values which don't overflow). 551 unsigned MaxBitsPerDigit = 1; 552 while ((1U << MaxBitsPerDigit) < radix) 553 MaxBitsPerDigit += 1; 554 if ((SuffixBegin - DigitsBegin) * MaxBitsPerDigit <= 64) { 555 uint64_t N = 0; 556 for (s = DigitsBegin; s != SuffixBegin; ++s) 557 N = N*radix + HexDigitValue(*s); 558 559 // This will truncate the value to Val's input width. Simply check 560 // for overflow by comparing. 561 Val = N; 562 return Val.getZExtValue() != N; 563 } 564 565 Val = 0; 566 s = DigitsBegin; 567 568 llvm::APInt RadixVal(Val.getBitWidth(), radix); 569 llvm::APInt CharVal(Val.getBitWidth(), 0); 570 llvm::APInt OldVal = Val; 571 572 bool OverflowOccurred = false; 573 while (s < SuffixBegin) { 574 unsigned C = HexDigitValue(*s++); 575 576 // If this letter is out of bound for this radix, reject it. 577 assert(C < radix && "NumericLiteralParser ctor should have rejected this"); 578 579 CharVal = C; 580 581 // Add the digit to the value in the appropriate radix. If adding in digits 582 // made the value smaller, then this overflowed. 583 OldVal = Val; 584 585 // Multiply by radix, did overflow occur on the multiply? 586 Val *= RadixVal; 587 OverflowOccurred |= Val.udiv(RadixVal) != OldVal; 588 589 // Add value, did overflow occur on the value? 590 // (a + b) ult b <=> overflow 591 Val += CharVal; 592 OverflowOccurred |= Val.ult(CharVal); 593 } 594 return OverflowOccurred; 595} 596 597llvm::APFloat NumericLiteralParser:: 598GetFloatValue(const llvm::fltSemantics &Format, bool* isExact) { 599 using llvm::APFloat; 600 601 llvm::SmallVector<char,256> floatChars; 602 for (unsigned i = 0, n = ThisTokEnd-ThisTokBegin; i != n; ++i) 603 floatChars.push_back(ThisTokBegin[i]); 604 605 floatChars.push_back('\0'); 606 607 APFloat V (Format, APFloat::fcZero, false); 608 APFloat::opStatus status; 609 610 status = V.convertFromString(&floatChars[0],APFloat::rmNearestTiesToEven); 611 612 if (isExact) 613 *isExact = status == APFloat::opOK; 614 615 return V; 616} 617 618 619CharLiteralParser::CharLiteralParser(const char *begin, const char *end, 620 SourceLocation Loc, Preprocessor &PP) { 621 // At this point we know that the character matches the regex "L?'.*'". 622 HadError = false; 623 624 // Determine if this is a wide character. 625 IsWide = begin[0] == 'L'; 626 if (IsWide) ++begin; 627 628 // Skip over the entry quote. 629 assert(begin[0] == '\'' && "Invalid token lexed"); 630 ++begin; 631 632 // FIXME: The "Value" is an uint64_t so we can handle char literals of 633 // upto 64-bits. 634 // FIXME: This extensively assumes that 'char' is 8-bits. 635 assert(PP.getTargetInfo().getCharWidth() == 8 && 636 "Assumes char is 8 bits"); 637 assert(PP.getTargetInfo().getIntWidth() <= 64 && 638 (PP.getTargetInfo().getIntWidth() & 7) == 0 && 639 "Assumes sizeof(int) on target is <= 64 and a multiple of char"); 640 assert(PP.getTargetInfo().getWCharWidth() <= 64 && 641 "Assumes sizeof(wchar) on target is <= 64"); 642 643 // This is what we will use for overflow detection 644 llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0); 645 646 unsigned NumCharsSoFar = 0; 647 while (begin[0] != '\'') { 648 uint64_t ResultChar; 649 if (begin[0] != '\\') // If this is a normal character, consume it. 650 ResultChar = *begin++; 651 else // Otherwise, this is an escape character. 652 ResultChar = ProcessCharEscape(begin, end, HadError, Loc, IsWide, PP); 653 654 // If this is a multi-character constant (e.g. 'abc'), handle it. These are 655 // implementation defined (C99 6.4.4.4p10). 656 if (NumCharsSoFar) { 657 if (IsWide) { 658 // Emulate GCC's (unintentional?) behavior: L'ab' -> L'b'. 659 LitVal = 0; 660 } else { 661 // Narrow character literals act as though their value is concatenated 662 // in this implementation, but warn on overflow. 663 if (LitVal.countLeadingZeros() < 8) 664 PP.Diag(Loc, diag::warn_char_constant_too_large); 665 LitVal <<= 8; 666 } 667 } 668 669 LitVal = LitVal + ResultChar; 670 ++NumCharsSoFar; 671 } 672 673 // If this is the second character being processed, do special handling. 674 if (NumCharsSoFar > 1) { 675 // Warn about discarding the top bits for multi-char wide-character 676 // constants (L'abcd'). 677 if (IsWide) 678 PP.Diag(Loc, diag::warn_extraneous_wide_char_constant); 679 else if (NumCharsSoFar != 4) 680 PP.Diag(Loc, diag::ext_multichar_character_literal); 681 else 682 PP.Diag(Loc, diag::ext_four_char_character_literal); 683 IsMultiChar = true; 684 } 685 686 // Transfer the value from APInt to uint64_t 687 Value = LitVal.getZExtValue(); 688 689 // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1") 690 // if 'char' is signed for this target (C99 6.4.4.4p10). Note that multiple 691 // character constants are not sign extended in the this implementation: 692 // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC. 693 if (!IsWide && NumCharsSoFar == 1 && (Value & 128) && 694 PP.getLangOptions().CharIsSigned) 695 Value = (signed char)Value; 696} 697 698 699/// string-literal: [C99 6.4.5] 700/// " [s-char-sequence] " 701/// L" [s-char-sequence] " 702/// s-char-sequence: 703/// s-char 704/// s-char-sequence s-char 705/// s-char: 706/// any source character except the double quote ", 707/// backslash \, or newline character 708/// escape-character 709/// universal-character-name 710/// escape-character: [C99 6.4.4.4] 711/// \ escape-code 712/// universal-character-name 713/// escape-code: 714/// character-escape-code 715/// octal-escape-code 716/// hex-escape-code 717/// character-escape-code: one of 718/// n t b r f v a 719/// \ ' " ? 720/// octal-escape-code: 721/// octal-digit 722/// octal-digit octal-digit 723/// octal-digit octal-digit octal-digit 724/// hex-escape-code: 725/// x hex-digit 726/// hex-escape-code hex-digit 727/// universal-character-name: 728/// \u hex-quad 729/// \U hex-quad hex-quad 730/// hex-quad: 731/// hex-digit hex-digit hex-digit hex-digit 732/// 733StringLiteralParser:: 734StringLiteralParser(const Token *StringToks, unsigned NumStringToks, 735 Preprocessor &pp) : PP(pp) { 736 // Scan all of the string portions, remember the max individual token length, 737 // computing a bound on the concatenated string length, and see whether any 738 // piece is a wide-string. If any of the string portions is a wide-string 739 // literal, the result is a wide-string literal [C99 6.4.5p4]. 740 MaxTokenLength = StringToks[0].getLength(); 741 SizeBound = StringToks[0].getLength()-2; // -2 for "". 742 AnyWide = StringToks[0].is(tok::wide_string_literal); 743 744 hadError = false; 745 746 // Implement Translation Phase #6: concatenation of string literals 747 /// (C99 5.1.1.2p1). The common case is only one string fragment. 748 for (unsigned i = 1; i != NumStringToks; ++i) { 749 // The string could be shorter than this if it needs cleaning, but this is a 750 // reasonable bound, which is all we need. 751 SizeBound += StringToks[i].getLength()-2; // -2 for "". 752 753 // Remember maximum string piece length. 754 if (StringToks[i].getLength() > MaxTokenLength) 755 MaxTokenLength = StringToks[i].getLength(); 756 757 // Remember if we see any wide strings. 758 AnyWide |= StringToks[i].is(tok::wide_string_literal); 759 } 760 761 // Include space for the null terminator. 762 ++SizeBound; 763 764 // TODO: K&R warning: "traditional C rejects string constant concatenation" 765 766 // Get the width in bytes of wchar_t. If no wchar_t strings are used, do not 767 // query the target. As such, wchar_tByteWidth is only valid if AnyWide=true. 768 wchar_tByteWidth = ~0U; 769 if (AnyWide) { 770 wchar_tByteWidth = PP.getTargetInfo().getWCharWidth(); 771 assert((wchar_tByteWidth & 7) == 0 && "Assumes wchar_t is byte multiple!"); 772 wchar_tByteWidth /= 8; 773 } 774 775 // The output buffer size needs to be large enough to hold wide characters. 776 // This is a worst-case assumption which basically corresponds to L"" "long". 777 if (AnyWide) 778 SizeBound *= wchar_tByteWidth; 779 780 // Size the temporary buffer to hold the result string data. 781 ResultBuf.resize(SizeBound); 782 783 // Likewise, but for each string piece. 784 llvm::SmallString<512> TokenBuf; 785 TokenBuf.resize(MaxTokenLength); 786 787 // Loop over all the strings, getting their spelling, and expanding them to 788 // wide strings as appropriate. 789 ResultPtr = &ResultBuf[0]; // Next byte to fill in. 790 791 Pascal = false; 792 793 for (unsigned i = 0, e = NumStringToks; i != e; ++i) { 794 const char *ThisTokBuf = &TokenBuf[0]; 795 // Get the spelling of the token, which eliminates trigraphs, etc. We know 796 // that ThisTokBuf points to a buffer that is big enough for the whole token 797 // and 'spelled' tokens can only shrink. 798 unsigned ThisTokLen = PP.getSpelling(StringToks[i], ThisTokBuf); 799 const char *ThisTokEnd = ThisTokBuf+ThisTokLen-1; // Skip end quote. 800 801 // TODO: Input character set mapping support. 802 803 // Skip L marker for wide strings. 804 bool ThisIsWide = false; 805 if (ThisTokBuf[0] == 'L') { 806 ++ThisTokBuf; 807 ThisIsWide = true; 808 } 809 810 assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?"); 811 ++ThisTokBuf; 812 813 // Check if this is a pascal string 814 if (pp.getLangOptions().PascalStrings && ThisTokBuf + 1 != ThisTokEnd && 815 ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') { 816 817 // If the \p sequence is found in the first token, we have a pascal string 818 // Otherwise, if we already have a pascal string, ignore the first \p 819 if (i == 0) { 820 ++ThisTokBuf; 821 Pascal = true; 822 } else if (Pascal) 823 ThisTokBuf += 2; 824 } 825 826 while (ThisTokBuf != ThisTokEnd) { 827 // Is this a span of non-escape characters? 828 if (ThisTokBuf[0] != '\\') { 829 const char *InStart = ThisTokBuf; 830 do { 831 ++ThisTokBuf; 832 } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\'); 833 834 // Copy the character span over. 835 unsigned Len = ThisTokBuf-InStart; 836 if (!AnyWide) { 837 memcpy(ResultPtr, InStart, Len); 838 ResultPtr += Len; 839 } else { 840 // Note: our internal rep of wide char tokens is always little-endian. 841 for (; Len; --Len, ++InStart) { 842 *ResultPtr++ = InStart[0]; 843 // Add zeros at the end. 844 for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i) 845 *ResultPtr++ = 0; 846 } 847 } 848 continue; 849 } 850 // Is this a Universal Character Name escape? 851 if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') { 852 ProcessUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr, 853 hadError, StringToks[i].getLocation(), ThisIsWide, PP); 854 continue; 855 } 856 // Otherwise, this is a non-UCN escape character. Process it. 857 unsigned ResultChar = ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError, 858 StringToks[i].getLocation(), 859 ThisIsWide, PP); 860 861 // Note: our internal rep of wide char tokens is always little-endian. 862 *ResultPtr++ = ResultChar & 0xFF; 863 864 if (AnyWide) { 865 for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i) 866 *ResultPtr++ = ResultChar >> i*8; 867 } 868 } 869 } 870 871 if (Pascal) { 872 ResultBuf[0] = ResultPtr-&ResultBuf[0]-1; 873 874 // Verify that pascal strings aren't too large. 875 if (GetStringLength() > 256) { 876 PP.Diag(StringToks[0].getLocation(), diag::err_pascal_string_too_long) 877 << SourceRange(StringToks[0].getLocation(), 878 StringToks[NumStringToks-1].getLocation()); 879 hadError = 1; 880 return; 881 } 882 } 883} 884 885 886/// getOffsetOfStringByte - This function returns the offset of the 887/// specified byte of the string data represented by Token. This handles 888/// advancing over escape sequences in the string. 889unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok, 890 unsigned ByteNo, 891 Preprocessor &PP) { 892 // Get the spelling of the token. 893 llvm::SmallString<16> SpellingBuffer; 894 SpellingBuffer.resize(Tok.getLength()); 895 896 const char *SpellingPtr = &SpellingBuffer[0]; 897 unsigned TokLen = PP.getSpelling(Tok, SpellingPtr); 898 899 assert(SpellingPtr[0] != 'L' && "Doesn't handle wide strings yet"); 900 901 902 const char *SpellingStart = SpellingPtr; 903 const char *SpellingEnd = SpellingPtr+TokLen; 904 905 // Skip over the leading quote. 906 assert(SpellingPtr[0] == '"' && "Should be a string literal!"); 907 ++SpellingPtr; 908 909 // Skip over bytes until we find the offset we're looking for. 910 while (ByteNo) { 911 assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!"); 912 913 // Step over non-escapes simply. 914 if (*SpellingPtr != '\\') { 915 ++SpellingPtr; 916 --ByteNo; 917 continue; 918 } 919 920 // Otherwise, this is an escape character. Advance over it. 921 bool HadError = false; 922 ProcessCharEscape(SpellingPtr, SpellingEnd, HadError, 923 Tok.getLocation(), false, PP); 924 assert(!HadError && "This method isn't valid on erroneous strings"); 925 --ByteNo; 926 } 927 928 return SpellingPtr-SpellingStart; 929} 930