1/**
2 * Defines lexical tokens.
3 *
4 * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
5 *
6 * Copyright:   Copyright (C) 1999-2022 by The D Language Foundation, All Rights Reserved
7 * Authors:     $(LINK2 https://www.digitalmars.com, Walter Bright)
8 * License:     $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
9 * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
10 * Documentation:  https://dlang.org/phobos/dmd_tokens.html
11 * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
12 */
13
14module dmd.tokens;
15
16import core.stdc.ctype;
17import core.stdc.stdio;
18import core.stdc.string;
19import dmd.globals;
20import dmd.identifier;
21import dmd.root.ctfloat;
22import dmd.common.outbuffer;
23import dmd.root.rmem;
24import dmd.root.utf;
25
26enum TOK : ubyte
27{
28    reserved,
29
30    // Other
31    leftParenthesis,
32    rightParenthesis,
33    leftBracket,
34    rightBracket,
35    leftCurly,
36    rightCurly,
37    colon,
38    semicolon,
39    dotDotDot,
40    endOfFile,
41    cast_,
42    null_,
43    assert_,
44    true_,
45    false_,
46    throw_,
47    new_,
48    delete_,
49    variable,
50    slice,
51    version_,
52    module_,
53    dollar,
54    template_,
55    typeof_,
56    pragma_,
57    typeid_,
58    comment,
59
60    // Operators
61    lessThan,
62    greaterThan,
63    lessOrEqual,
64    greaterOrEqual,
65    equal,
66    notEqual,
67    identity,
68    notIdentity,
69    is_,
70
71    leftShift,
72    rightShift,
73    leftShiftAssign,
74    rightShiftAssign,
75    unsignedRightShift,
76    unsignedRightShiftAssign,
77    concatenateAssign, // ~=
78    add,
79    min,
80    addAssign,
81    minAssign,
82    mul,
83    div,
84    mod,
85    mulAssign,
86    divAssign,
87    modAssign,
88    and,
89    or,
90    xor,
91    andAssign,
92    orAssign,
93    xorAssign,
94    assign,
95    not,
96    tilde,
97    plusPlus,
98    minusMinus,
99    dot,
100    comma,
101    question,
102    andAnd,
103    orOr,
104
105    // Numeric literals
106    int32Literal,
107    uns32Literal,
108    int64Literal,
109    uns64Literal,
110    int128Literal,
111    uns128Literal,
112    float32Literal,
113    float64Literal,
114    float80Literal,
115    imaginary32Literal,
116    imaginary64Literal,
117    imaginary80Literal,
118
119    // Char constants
120    charLiteral,
121    wcharLiteral,
122    dcharLiteral,
123
124    // Leaf operators
125    identifier,
126    string_,
127    this_,
128    super_,
129    error,
130
131    // Basic types
132    void_,
133    int8,
134    uns8,
135    int16,
136    uns16,
137    int32,
138    uns32,
139    int64,
140    uns64,
141    int128,
142    uns128,
143    float32,
144    float64,
145    float80,
146    imaginary32,
147    imaginary64,
148    imaginary80,
149    complex32,
150    complex64,
151    complex80,
152    char_,
153    wchar_,
154    dchar_,
155    bool_,
156
157    // Aggregates
158    struct_,
159    class_,
160    interface_,
161    union_,
162    enum_,
163    import_,
164    alias_,
165    override_,
166    delegate_,
167    function_,
168    mixin_,
169    align_,
170    extern_,
171    private_,
172    protected_,
173    public_,
174    export_,
175    static_,
176    final_,
177    const_,
178    abstract_,
179    debug_,
180    deprecated_,
181    in_,
182    out_,
183    inout_,
184    lazy_,
185    auto_,
186    package_,
187    immutable_,
188
189    // Statements
190    if_,
191    else_,
192    while_,
193    for_,
194    do_,
195    switch_,
196    case_,
197    default_,
198    break_,
199    continue_,
200    with_,
201    synchronized_,
202    return_,
203    goto_,
204    try_,
205    catch_,
206    finally_,
207    asm_,
208    foreach_,
209    foreach_reverse_,
210    scope_,
211    onScopeExit,
212    onScopeFailure,
213    onScopeSuccess,
214
215    // Contracts
216    invariant_,
217
218    // Testing
219    unittest_,
220
221    // Added after 1.0
222    argumentTypes,
223    ref_,
224    macro_,
225
226    parameters,
227    traits,
228    pure_,
229    nothrow_,
230    gshared,
231    line,
232    file,
233    fileFullPath,
234    moduleString,   // __MODULE__
235    functionString, // __FUNCTION__
236    prettyFunction, // __PRETTY_FUNCTION__
237    shared_,
238    at,
239    pow,
240    powAssign,
241    goesTo,
242    vector,
243    pound,
244
245    arrow,      // ->
246    colonColon, // ::
247    wchar_tLiteral,
248    endOfLine,  // \n, \r, \u2028, \u2029
249    whitespace,
250
251    // C only keywords
252    inline,
253    register,
254    restrict,
255    signed,
256    sizeof_,
257    typedef_,
258    unsigned,
259    volatile,
260    _Alignas,
261    _Alignof,
262    _Atomic,
263    _Bool,
264    _Complex,
265    _Generic,
266    _Imaginary,
267    _Noreturn,
268    _Static_assert,
269    _Thread_local,
270
271    // C only extended keywords
272    _import,
273    __cdecl,
274    __declspec,
275    __stdcall,
276    __attribute__,
277}
278
279/// Expression nodes
280enum EXP : ubyte
281{
282    reserved,
283
284    // Other
285    negate,
286    cast_,
287    null_,
288    assert_,
289    true_,
290    false_,
291    array,
292    call,
293    address,
294    type,
295    throw_,
296    new_,
297    delete_,
298    star,
299    symbolOffset,
300    variable,
301    dotVariable,
302    dotIdentifier,
303    dotTemplateInstance,
304    dotType,
305    slice,
306    arrayLength,
307    version_,
308    dollar,
309    template_,
310    dotTemplateDeclaration,
311    declaration,
312    typeof_,
313    pragma_,
314    dSymbol,
315    typeid_,
316    uadd,
317    remove,
318    newAnonymousClass,
319    arrayLiteral,
320    assocArrayLiteral,
321    structLiteral,
322    classReference,
323    thrownException,
324    delegatePointer,
325    delegateFunctionPointer,
326
327    // Operators
328    lessThan,
329    greaterThan,
330    lessOrEqual,
331    greaterOrEqual,
332    equal,
333    notEqual,
334    identity,
335    notIdentity,
336    index,
337    is_,
338
339    leftShift,
340    rightShift,
341    leftShiftAssign,
342    rightShiftAssign,
343    unsignedRightShift,
344    unsignedRightShiftAssign,
345    concatenate,
346    concatenateAssign, // ~=
347    concatenateElemAssign,
348    concatenateDcharAssign,
349    add,
350    min,
351    addAssign,
352    minAssign,
353    mul,
354    div,
355    mod,
356    mulAssign,
357    divAssign,
358    modAssign,
359    and,
360    or,
361    xor,
362    andAssign,
363    orAssign,
364    xorAssign,
365    assign,
366    not,
367    tilde,
368    plusPlus,
369    minusMinus,
370    construct,
371    blit,
372    dot,
373    comma,
374    question,
375    andAnd,
376    orOr,
377    prePlusPlus,
378    preMinusMinus,
379
380    // Leaf operators
381    identifier,
382    string_,
383    this_,
384    super_,
385    halt,
386    tuple,
387    error,
388
389    // Basic types
390    void_,
391    int64,
392    float64,
393    complex80,
394    char_,
395    import_,
396    delegate_,
397    function_,
398    mixin_,
399    in_,
400    default_,
401    break_,
402    continue_,
403    goto_,
404    scope_,
405
406    traits,
407    overloadSet,
408    line,
409    file,
410    fileFullPath,
411    moduleString,   // __MODULE__
412    functionString, // __FUNCTION__
413    prettyFunction, // __PRETTY_FUNCTION__
414    shared_,
415    pow,
416    powAssign,
417    vector,
418
419    voidExpression,
420    cantExpression,
421    showCtfeContext,
422    objcClassReference,
423    vectorArray,
424    arrow,      // ->
425    compoundLiteral, // ( type-name ) { initializer-list }
426    _Generic,
427    interval,
428}
429
430enum FirstCKeyword = TOK.inline;
431
432// Assert that all token enum members have consecutive values and
433// that none of them overlap
434static assert(() {
435    foreach (idx, enumName; __traits(allMembers, TOK)) {
436       static if (idx != __traits(getMember, TOK, enumName)) {
437           pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
438           static assert(0);
439       }
440    }
441    return true;
442}());
443
444/****************************************
445 */
446
447private immutable TOK[] keywords =
448[
449    TOK.this_,
450    TOK.super_,
451    TOK.assert_,
452    TOK.null_,
453    TOK.true_,
454    TOK.false_,
455    TOK.cast_,
456    TOK.new_,
457    TOK.delete_,
458    TOK.throw_,
459    TOK.module_,
460    TOK.pragma_,
461    TOK.typeof_,
462    TOK.typeid_,
463    TOK.template_,
464    TOK.void_,
465    TOK.int8,
466    TOK.uns8,
467    TOK.int16,
468    TOK.uns16,
469    TOK.int32,
470    TOK.uns32,
471    TOK.int64,
472    TOK.uns64,
473    TOK.int128,
474    TOK.uns128,
475    TOK.float32,
476    TOK.float64,
477    TOK.float80,
478    TOK.bool_,
479    TOK.char_,
480    TOK.wchar_,
481    TOK.dchar_,
482    TOK.imaginary32,
483    TOK.imaginary64,
484    TOK.imaginary80,
485    TOK.complex32,
486    TOK.complex64,
487    TOK.complex80,
488    TOK.delegate_,
489    TOK.function_,
490    TOK.is_,
491    TOK.if_,
492    TOK.else_,
493    TOK.while_,
494    TOK.for_,
495    TOK.do_,
496    TOK.switch_,
497    TOK.case_,
498    TOK.default_,
499    TOK.break_,
500    TOK.continue_,
501    TOK.synchronized_,
502    TOK.return_,
503    TOK.goto_,
504    TOK.try_,
505    TOK.catch_,
506    TOK.finally_,
507    TOK.with_,
508    TOK.asm_,
509    TOK.foreach_,
510    TOK.foreach_reverse_,
511    TOK.scope_,
512    TOK.struct_,
513    TOK.class_,
514    TOK.interface_,
515    TOK.union_,
516    TOK.enum_,
517    TOK.import_,
518    TOK.mixin_,
519    TOK.static_,
520    TOK.final_,
521    TOK.const_,
522    TOK.alias_,
523    TOK.override_,
524    TOK.abstract_,
525    TOK.debug_,
526    TOK.deprecated_,
527    TOK.in_,
528    TOK.out_,
529    TOK.inout_,
530    TOK.lazy_,
531    TOK.auto_,
532    TOK.align_,
533    TOK.extern_,
534    TOK.private_,
535    TOK.package_,
536    TOK.protected_,
537    TOK.public_,
538    TOK.export_,
539    TOK.invariant_,
540    TOK.unittest_,
541    TOK.version_,
542    TOK.argumentTypes,
543    TOK.parameters,
544    TOK.ref_,
545    TOK.macro_,
546    TOK.pure_,
547    TOK.nothrow_,
548    TOK.gshared,
549    TOK.traits,
550    TOK.vector,
551    TOK.file,
552    TOK.fileFullPath,
553    TOK.line,
554    TOK.moduleString,
555    TOK.functionString,
556    TOK.prettyFunction,
557    TOK.shared_,
558    TOK.immutable_,
559
560    // C only keywords
561    TOK.inline,
562    TOK.register,
563    TOK.restrict,
564    TOK.signed,
565    TOK.sizeof_,
566    TOK.typedef_,
567    TOK.unsigned,
568    TOK.volatile,
569    TOK._Alignas,
570    TOK._Alignof,
571    TOK._Atomic,
572    TOK._Bool,
573    TOK._Complex,
574    TOK._Generic,
575    TOK._Imaginary,
576    TOK._Noreturn,
577    TOK._Static_assert,
578    TOK._Thread_local,
579
580    // C only extended keywords
581    TOK._import,
582    TOK.__cdecl,
583    TOK.__declspec,
584    TOK.__stdcall,
585    TOK.__attribute__,
586];
587
588// Initialize the identifier pool
589shared static this() nothrow
590{
591    Identifier.initTable();
592    foreach (kw; keywords)
593    {
594        //printf("keyword[%d] = '%s'\n",kw, Token.tochars[kw].ptr);
595        Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw);
596    }
597}
598
599/************************************
600 * This is used to pick the C keywords out of the tokens.
601 * If it's not a C keyword, then it's an identifier.
602 */
603static immutable TOK[TOK.max + 1] Ckeywords =
604() {
605    with (TOK)
606    {
607        TOK[TOK.max + 1] tab = identifier;  // default to identifier
608        enum Ckwds = [ auto_, break_, case_, char_, const_, continue_, default_, do_, float64, else_,
609                       enum_, extern_, float32, for_, goto_, if_, inline, int32, int64, register,
610                       restrict, return_, int16, signed, sizeof_, static_, struct_, switch_, typedef_,
611                       union_, unsigned, void_, volatile, while_, asm_,
612                       _Alignas, _Alignof, _Atomic, _Bool, _Complex, _Generic, _Imaginary, _Noreturn,
613                       _Static_assert, _Thread_local, _import, __cdecl, __declspec, __stdcall, __attribute__ ];
614
615        foreach (kw; Ckwds)
616            tab[kw] = cast(TOK) kw;
617
618        return tab;
619    }
620} ();
621
622
623/***********************************************************
624 */
625extern (C++) struct Token
626{
627    Token* next;
628    Loc loc;
629    const(char)* ptr; // pointer to first character of this token within buffer
630    TOK value;
631    const(char)[] blockComment; // doc comment string prior to this token
632    const(char)[] lineComment; // doc comment for previous token
633
634    union
635    {
636        // Integers
637        sinteger_t intvalue;
638        uinteger_t unsvalue;
639        // Floats
640        real_t floatvalue;
641
642        struct
643        {
644            const(char)* ustring; // UTF8 string
645            uint len;
646            ubyte postfix; // 'c', 'w', 'd'
647        }
648
649        Identifier ident;
650    }
651
652    extern (D) private static immutable string[TOK.max + 1] tochars =
653    [
654        // Keywords
655        TOK.this_: "this",
656        TOK.super_: "super",
657        TOK.assert_: "assert",
658        TOK.null_: "null",
659        TOK.true_: "true",
660        TOK.false_: "false",
661        TOK.cast_: "cast",
662        TOK.new_: "new",
663        TOK.delete_: "delete",
664        TOK.throw_: "throw",
665        TOK.module_: "module",
666        TOK.pragma_: "pragma",
667        TOK.typeof_: "typeof",
668        TOK.typeid_: "typeid",
669        TOK.template_: "template",
670        TOK.void_: "void",
671        TOK.int8: "byte",
672        TOK.uns8: "ubyte",
673        TOK.int16: "short",
674        TOK.uns16: "ushort",
675        TOK.int32: "int",
676        TOK.uns32: "uint",
677        TOK.int64: "long",
678        TOK.uns64: "ulong",
679        TOK.int128: "cent",
680        TOK.uns128: "ucent",
681        TOK.float32: "float",
682        TOK.float64: "double",
683        TOK.float80: "real",
684        TOK.bool_: "bool",
685        TOK.char_: "char",
686        TOK.wchar_: "wchar",
687        TOK.dchar_: "dchar",
688        TOK.imaginary32: "ifloat",
689        TOK.imaginary64: "idouble",
690        TOK.imaginary80: "ireal",
691        TOK.complex32: "cfloat",
692        TOK.complex64: "cdouble",
693        TOK.complex80: "creal",
694        TOK.delegate_: "delegate",
695        TOK.function_: "function",
696        TOK.is_: "is",
697        TOK.if_: "if",
698        TOK.else_: "else",
699        TOK.while_: "while",
700        TOK.for_: "for",
701        TOK.do_: "do",
702        TOK.switch_: "switch",
703        TOK.case_: "case",
704        TOK.default_: "default",
705        TOK.break_: "break",
706        TOK.continue_: "continue",
707        TOK.synchronized_: "synchronized",
708        TOK.return_: "return",
709        TOK.goto_: "goto",
710        TOK.try_: "try",
711        TOK.catch_: "catch",
712        TOK.finally_: "finally",
713        TOK.with_: "with",
714        TOK.asm_: "asm",
715        TOK.foreach_: "foreach",
716        TOK.foreach_reverse_: "foreach_reverse",
717        TOK.scope_: "scope",
718        TOK.struct_: "struct",
719        TOK.class_: "class",
720        TOK.interface_: "interface",
721        TOK.union_: "union",
722        TOK.enum_: "enum",
723        TOK.import_: "import",
724        TOK.mixin_: "mixin",
725        TOK.static_: "static",
726        TOK.final_: "final",
727        TOK.const_: "const",
728        TOK.alias_: "alias",
729        TOK.override_: "override",
730        TOK.abstract_: "abstract",
731        TOK.debug_: "debug",
732        TOK.deprecated_: "deprecated",
733        TOK.in_: "in",
734        TOK.out_: "out",
735        TOK.inout_: "inout",
736        TOK.lazy_: "lazy",
737        TOK.auto_: "auto",
738        TOK.align_: "align",
739        TOK.extern_: "extern",
740        TOK.private_: "private",
741        TOK.package_: "package",
742        TOK.protected_: "protected",
743        TOK.public_: "public",
744        TOK.export_: "export",
745        TOK.invariant_: "invariant",
746        TOK.unittest_: "unittest",
747        TOK.version_: "version",
748        TOK.argumentTypes: "__argTypes",
749        TOK.parameters: "__parameters",
750        TOK.ref_: "ref",
751        TOK.macro_: "macro",
752        TOK.pure_: "pure",
753        TOK.nothrow_: "nothrow",
754        TOK.gshared: "__gshared",
755        TOK.traits: "__traits",
756        TOK.vector: "__vector",
757        TOK.file: "__FILE__",
758        TOK.fileFullPath: "__FILE_FULL_PATH__",
759        TOK.line: "__LINE__",
760        TOK.moduleString: "__MODULE__",
761        TOK.functionString: "__FUNCTION__",
762        TOK.prettyFunction: "__PRETTY_FUNCTION__",
763        TOK.shared_: "shared",
764        TOK.immutable_: "immutable",
765
766        TOK.endOfFile: "End of File",
767        TOK.leftCurly: "{",
768        TOK.rightCurly: "}",
769        TOK.leftParenthesis: "(",
770        TOK.rightParenthesis: ")",
771        TOK.leftBracket: "[",
772        TOK.rightBracket: "]",
773        TOK.semicolon: ";",
774        TOK.colon: ":",
775        TOK.comma: ",",
776        TOK.dot: ".",
777        TOK.xor: "^",
778        TOK.xorAssign: "^=",
779        TOK.assign: "=",
780        TOK.lessThan: "<",
781        TOK.greaterThan: ">",
782        TOK.lessOrEqual: "<=",
783        TOK.greaterOrEqual: ">=",
784        TOK.equal: "==",
785        TOK.notEqual: "!=",
786        TOK.not: "!",
787        TOK.leftShift: "<<",
788        TOK.rightShift: ">>",
789        TOK.unsignedRightShift: ">>>",
790        TOK.add: "+",
791        TOK.min: "-",
792        TOK.mul: "*",
793        TOK.div: "/",
794        TOK.mod: "%",
795        TOK.slice: "..",
796        TOK.dotDotDot: "...",
797        TOK.and: "&",
798        TOK.andAnd: "&&",
799        TOK.or: "|",
800        TOK.orOr: "||",
801        TOK.tilde: "~",
802        TOK.dollar: "$",
803        TOK.plusPlus: "++",
804        TOK.minusMinus: "--",
805        TOK.question: "?",
806        TOK.variable: "var",
807        TOK.addAssign: "+=",
808        TOK.minAssign: "-=",
809        TOK.mulAssign: "*=",
810        TOK.divAssign: "/=",
811        TOK.modAssign: "%=",
812        TOK.leftShiftAssign: "<<=",
813        TOK.rightShiftAssign: ">>=",
814        TOK.unsignedRightShiftAssign: ">>>=",
815        TOK.andAssign: "&=",
816        TOK.orAssign: "|=",
817        TOK.concatenateAssign: "~=",
818        TOK.identity: "is",
819        TOK.notIdentity: "!is",
820        TOK.identifier: "identifier",
821        TOK.at: "@",
822        TOK.pow: "^^",
823        TOK.powAssign: "^^=",
824        TOK.goesTo: "=>",
825        TOK.pound: "#",
826        TOK.arrow: "->",
827        TOK.colonColon: "::",
828
829        // For debugging
830        TOK.error: "error",
831        TOK.string_: "string",
832        TOK.onScopeExit: "scope(exit)",
833        TOK.onScopeSuccess: "scope(success)",
834        TOK.onScopeFailure: "scope(failure)",
835
836        // Finish up
837        TOK.reserved: "reserved",
838        TOK.comment: "comment",
839        TOK.int32Literal: "int32v",
840        TOK.uns32Literal: "uns32v",
841        TOK.int64Literal: "int64v",
842        TOK.uns64Literal: "uns64v",
843        TOK.int128Literal: "int128v",
844        TOK.uns128Literal: "uns128v",
845        TOK.float32Literal: "float32v",
846        TOK.float64Literal: "float64v",
847        TOK.float80Literal: "float80v",
848        TOK.imaginary32Literal: "imaginary32v",
849        TOK.imaginary64Literal: "imaginary64v",
850        TOK.imaginary80Literal: "imaginary80v",
851        TOK.charLiteral: "charv",
852        TOK.wcharLiteral: "wcharv",
853        TOK.dcharLiteral: "dcharv",
854        TOK.wchar_tLiteral: "wchar_tv",
855        TOK.endOfLine: "\\n",
856        TOK.whitespace: "whitespace",
857
858        // C only keywords
859        TOK.inline    : "inline",
860        TOK.register  : "register",
861        TOK.restrict  : "restrict",
862        TOK.signed    : "signed",
863        TOK.sizeof_   : "sizeof",
864        TOK.typedef_  : "typedef",
865        TOK.unsigned  : "unsigned",
866        TOK.volatile  : "volatile",
867        TOK._Alignas  : "_Alignas",
868        TOK._Alignof  : "_Alignof",
869        TOK._Atomic   : "_Atomic",
870        TOK._Bool     : "_Bool",
871        TOK._Complex  : "_Complex",
872        TOK._Generic  : "_Generic",
873        TOK._Imaginary: "_Imaginary",
874        TOK._Noreturn : "_Noreturn",
875        TOK._Static_assert : "_Static_assert",
876        TOK._Thread_local  : "_Thread_local",
877
878        // C only extended keywords
879        TOK._import       : "__import",
880        TOK.__cdecl        : "__cdecl",
881        TOK.__declspec     : "__declspec",
882        TOK.__stdcall      : "__stdcall",
883        TOK.__attribute__  : "__attribute__",
884    ];
885
886    static assert(() {
887        foreach (s; tochars)
888            assert(s.length);
889        return true;
890    }());
891
892nothrow:
893
894    int isKeyword() const
895    {
896        foreach (kw; keywords)
897        {
898            if (kw == value)
899                return 1;
900        }
901        return 0;
902    }
903
904    /****
905     * Set to contents of ptr[0..length]
906     * Params:
907     *  ptr = pointer to string
908     *  length = length of string
909     */
910    void setString(const(char)* ptr, size_t length)
911    {
912        auto s = cast(char*)mem.xmalloc_noscan(length + 1);
913        memcpy(s, ptr, length);
914        s[length] = 0;
915        ustring = s;
916        len = cast(uint)length;
917        postfix = 0;
918    }
919
920    /****
921     * Set to contents of buf
922     * Params:
923     *  buf = string (not zero terminated)
924     */
925    void setString(const ref OutBuffer buf)
926    {
927        setString(cast(const(char)*)buf[].ptr, buf.length);
928    }
929
930    /****
931     * Set to empty string
932     */
933    void setString()
934    {
935        ustring = "";
936        len = 0;
937        postfix = 0;
938    }
939
940    extern (C++) const(char)* toChars() const
941    {
942        __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer;
943        const(char)* p = &buffer[0];
944        switch (value)
945        {
946        case TOK.int32Literal:
947            sprintf(&buffer[0], "%d", cast(int)intvalue);
948            break;
949        case TOK.uns32Literal:
950        case TOK.wchar_tLiteral:
951            sprintf(&buffer[0], "%uU", cast(uint)unsvalue);
952            break;
953        case TOK.wcharLiteral:
954        case TOK.dcharLiteral:
955        case TOK.charLiteral:
956            {
957                OutBuffer buf;
958                buf.writeSingleCharLiteral(cast(dchar) intvalue);
959                buf.writeByte('\0');
960                p = buf.extractSlice().ptr;
961            }
962            break;
963        case TOK.int64Literal:
964            sprintf(&buffer[0], "%lldL", cast(long)intvalue);
965            break;
966        case TOK.uns64Literal:
967            sprintf(&buffer[0], "%lluUL", cast(ulong)unsvalue);
968            break;
969        case TOK.float32Literal:
970            CTFloat.sprint(&buffer[0], 'g', floatvalue);
971            strcat(&buffer[0], "f");
972            break;
973        case TOK.float64Literal:
974            CTFloat.sprint(&buffer[0], 'g', floatvalue);
975            break;
976        case TOK.float80Literal:
977            CTFloat.sprint(&buffer[0], 'g', floatvalue);
978            strcat(&buffer[0], "L");
979            break;
980        case TOK.imaginary32Literal:
981            CTFloat.sprint(&buffer[0], 'g', floatvalue);
982            strcat(&buffer[0], "fi");
983            break;
984        case TOK.imaginary64Literal:
985            CTFloat.sprint(&buffer[0], 'g', floatvalue);
986            strcat(&buffer[0], "i");
987            break;
988        case TOK.imaginary80Literal:
989            CTFloat.sprint(&buffer[0], 'g', floatvalue);
990            strcat(&buffer[0], "Li");
991            break;
992        case TOK.string_:
993            {
994                OutBuffer buf;
995                buf.writeByte('"');
996                for (size_t i = 0; i < len;)
997                {
998                    dchar c;
999                    utf_decodeChar(ustring[0 .. len], i, c);
1000                    writeCharLiteral(buf, c);
1001                }
1002                buf.writeByte('"');
1003                if (postfix)
1004                    buf.writeByte(postfix);
1005                buf.writeByte(0);
1006                p = buf.extractSlice().ptr;
1007            }
1008            break;
1009        case TOK.identifier:
1010        case TOK.enum_:
1011        case TOK.struct_:
1012        case TOK.import_:
1013        case TOK.wchar_:
1014        case TOK.dchar_:
1015        case TOK.bool_:
1016        case TOK.char_:
1017        case TOK.int8:
1018        case TOK.uns8:
1019        case TOK.int16:
1020        case TOK.uns16:
1021        case TOK.int32:
1022        case TOK.uns32:
1023        case TOK.int64:
1024        case TOK.uns64:
1025        case TOK.int128:
1026        case TOK.uns128:
1027        case TOK.float32:
1028        case TOK.float64:
1029        case TOK.float80:
1030        case TOK.imaginary32:
1031        case TOK.imaginary64:
1032        case TOK.imaginary80:
1033        case TOK.complex32:
1034        case TOK.complex64:
1035        case TOK.complex80:
1036        case TOK.void_:
1037            p = ident.toChars();
1038            break;
1039        default:
1040            p = toChars(value);
1041            break;
1042        }
1043        return p;
1044    }
1045
1046    static const(char)* toChars(TOK value)
1047    {
1048        return toString(value).ptr;
1049    }
1050
1051    extern (D) static string toString(TOK value) pure nothrow @nogc @safe
1052    {
1053        return tochars[value];
1054    }
1055}
1056
1057/**
1058 * Write a character, using a readable escape sequence if needed
1059 *
1060 * Useful for printing "" string literals in e.g. error messages, ddoc, or the `.stringof` property
1061 *
1062 * Params:
1063 *   buf = buffer to append character in
1064 *   c = code point to write
1065 */
1066nothrow
1067void writeCharLiteral(ref OutBuffer buf, dchar c)
1068{
1069    switch (c)
1070    {
1071        case '\0':
1072            buf.writestring("\\0");
1073            break;
1074        case '\n':
1075            buf.writestring("\\n");
1076            break;
1077        case '\r':
1078            buf.writestring("\\r");
1079            break;
1080        case '\t':
1081            buf.writestring("\\t");
1082            break;
1083        case '\b':
1084            buf.writestring("\\b");
1085            break;
1086        case '\f':
1087            buf.writestring("\\f");
1088            break;
1089        case '"':
1090        case '\\':
1091            buf.writeByte('\\');
1092            goto default;
1093        default:
1094            if (c <= 0xFF)
1095            {
1096                if (isprint(c))
1097                    buf.writeByte(c);
1098                else
1099                    buf.printf("\\x%02x", c);
1100            }
1101            else if (c <= 0xFFFF)
1102                buf.printf("\\u%04x", c);
1103            else
1104                buf.printf("\\U%08x", c);
1105            break;
1106    }
1107}
1108
1109unittest
1110{
1111    OutBuffer buf;
1112    foreach(dchar d; "a\n\r\t\b\f\0\x11\u7233\U00017233"d)
1113    {
1114        writeCharLiteral(buf, d);
1115    }
1116    assert(buf.extractSlice() == `a\n\r\t\b\f\0\x11\u7233\U00017233`);
1117}
1118
1119/**
1120 * Write a single-quoted character literal
1121 *
1122 * Useful for printing '' char literals in e.g. error messages, ddoc, or the `.stringof` property
1123 *
1124 * Params:
1125 *   buf = buffer to append character in
1126 *   c = code point to write
1127 */
1128nothrow
1129void writeSingleCharLiteral(ref OutBuffer buf, dchar c)
1130{
1131    buf.writeByte('\'');
1132    if (c == '\'')
1133        buf.writeByte('\\');
1134
1135    if (c == '"')
1136        buf.writeByte('"');
1137    else
1138        writeCharLiteral(buf, c);
1139
1140    buf.writeByte('\'');
1141}
1142
1143unittest
1144{
1145    OutBuffer buf;
1146    writeSingleCharLiteral(buf, '\'');
1147    assert(buf.extractSlice() == `'\''`);
1148    buf.reset();
1149    writeSingleCharLiteral(buf, '"');
1150    assert(buf.extractSlice() == `'"'`);
1151    buf.reset();
1152    writeSingleCharLiteral(buf, '\n');
1153    assert(buf.extractSlice() == `'\n'`);
1154}
1155