Tokens.java revision 2571:10fc81ac75b4 
1/*
2 * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.  Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26package com.sun.tools.javac.parser;
27
28import java.util.Locale;
29
30import com.sun.tools.javac.api.Formattable;
31import com.sun.tools.javac.api.Messages;
32import com.sun.tools.javac.parser.Tokens.Token.Tag;
33import com.sun.tools.javac.util.List;
34import com.sun.tools.javac.util.Name;
35import com.sun.tools.javac.util.Context;
36import com.sun.tools.javac.util.Filter;
37import com.sun.tools.javac.util.ListBuffer;
38import com.sun.tools.javac.util.Names;
39
40/** A class that defines codes/utilities for Java source tokens
41 *  returned from lexical analysis.
42 *
43 *  <p><b>This is NOT part of any supported API.
44 *  If you write code that depends on this, you do so at your own risk.
45 *  This code and its internal interfaces are subject to change or
46 *  deletion without notice.</b>
47 */
48public class Tokens {
49
50    private final Names names;
51
52    /**
53     * Keyword array. Maps name indices to Token.
54     */
55    private final TokenKind[] key;
56
57    /**  The number of the last entered keyword.
58     */
59    private int maxKey = 0;
60
61    /** The names of all tokens.
62     */
63    private Name[] tokenName = new Name[TokenKind.values().length];
64
65    public static final Context.Key<Tokens> tokensKey = new Context.Key<>();
66
67    public static Tokens instance(Context context) {
68        Tokens instance = context.get(tokensKey);
69        if (instance == null)
70            instance = new Tokens(context);
71        return instance;
72    }
73
74    protected Tokens(Context context) {
75        context.put(tokensKey, this);
76        names = Names.instance(context);
77        for (TokenKind t : TokenKind.values()) {
78            if (t.name != null)
79                enterKeyword(t.name, t);
80            else
81                tokenName[t.ordinal()] = null;
82        }
83
84        key = new TokenKind[maxKey+1];
85        for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
86        for (TokenKind t : TokenKind.values()) {
87            if (t.name != null)
88            key[tokenName[t.ordinal()].getIndex()] = t;
89        }
90    }
91
92    private void enterKeyword(String s, TokenKind token) {
93        Name n = names.fromString(s);
94        tokenName[token.ordinal()] = n;
95        if (n.getIndex() > maxKey) maxKey = n.getIndex();
96    }
97
98    /**
99     * Create a new token given a name; if the name corresponds to a token name,
100     * a new token of the corresponding kind is returned; otherwise, an
101     * identifier token is returned.
102     */
103    TokenKind lookupKind(Name name) {
104        return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
105    }
106
107    TokenKind lookupKind(String name) {
108        return lookupKind(names.fromString(name));
109    }
110
111    /**
112     * This enum defines all tokens used by the javac scanner. A token is
113     * optionally associated with a name.
114     */
115    public enum TokenKind implements Formattable, Filter<TokenKind> {
116        EOF(),
117        ERROR(),
118        IDENTIFIER(Tag.NAMED),
119        ABSTRACT("abstract"),
120        ASSERT("assert", Tag.NAMED),
121        BOOLEAN("boolean", Tag.NAMED),
122        BREAK("break"),
123        BYTE("byte", Tag.NAMED),
124        CASE("case"),
125        CATCH("catch"),
126        CHAR("char", Tag.NAMED),
127        CLASS("class"),
128        CONST("const"),
129        CONTINUE("continue"),
130        DEFAULT("default"),
131        DO("do"),
132        DOUBLE("double", Tag.NAMED),
133        ELSE("else"),
134        ENUM("enum", Tag.NAMED),
135        EXTENDS("extends"),
136        FINAL("final"),
137        FINALLY("finally"),
138        FLOAT("float", Tag.NAMED),
139        FOR("for"),
140        GOTO("goto"),
141        IF("if"),
142        IMPLEMENTS("implements"),
143        IMPORT("import"),
144        INSTANCEOF("instanceof"),
145        INT("int", Tag.NAMED),
146        INTERFACE("interface"),
147        LONG("long", Tag.NAMED),
148        NATIVE("native"),
149        NEW("new"),
150        PACKAGE("package"),
151        PRIVATE("private"),
152        PROTECTED("protected"),
153        PUBLIC("public"),
154        RETURN("return"),
155        SHORT("short", Tag.NAMED),
156        STATIC("static"),
157        STRICTFP("strictfp"),
158        SUPER("super", Tag.NAMED),
159        SWITCH("switch"),
160        SYNCHRONIZED("synchronized"),
161        THIS("this", Tag.NAMED),
162        THROW("throw"),
163        THROWS("throws"),
164        TRANSIENT("transient"),
165        TRY("try"),
166        VOID("void", Tag.NAMED),
167        VOLATILE("volatile"),
168        WHILE("while"),
169        INTLITERAL(Tag.NUMERIC),
170        LONGLITERAL(Tag.NUMERIC),
171        FLOATLITERAL(Tag.NUMERIC),
172        DOUBLELITERAL(Tag.NUMERIC),
173        CHARLITERAL(Tag.NUMERIC),
174        STRINGLITERAL(Tag.STRING),
175        TRUE("true", Tag.NAMED),
176        FALSE("false", Tag.NAMED),
177        NULL("null", Tag.NAMED),
178        UNDERSCORE("_", Tag.NAMED),
179        ARROW("->"),
180        COLCOL("::"),
181        LPAREN("("),
182        RPAREN(")"),
183        LBRACE("{"),
184        RBRACE("}"),
185        LBRACKET("["),
186        RBRACKET("]"),
187        SEMI(";"),
188        COMMA(","),
189        DOT("."),
190        ELLIPSIS("..."),
191        EQ("="),
192        GT(">"),
193        LT("<"),
194        BANG("!"),
195        TILDE("~"),
196        QUES("?"),
197        COLON(":"),
198        EQEQ("=="),
199        LTEQ("<="),
200        GTEQ(">="),
201        BANGEQ("!="),
202        AMPAMP("&&"),
203        BARBAR("||"),
204        PLUSPLUS("++"),
205        SUBSUB("--"),
206        PLUS("+"),
207        SUB("-"),
208        STAR("*"),
209        SLASH("/"),
210        AMP("&"),
211        BAR("|"),
212        CARET("^"),
213        PERCENT("%"),
214        LTLT("<<"),
215        GTGT(">>"),
216        GTGTGT(">>>"),
217        PLUSEQ("+="),
218        SUBEQ("-="),
219        STAREQ("*="),
220        SLASHEQ("/="),
221        AMPEQ("&="),
222        BAREQ("|="),
223        CARETEQ("^="),
224        PERCENTEQ("%="),
225        LTLTEQ("<<="),
226        GTGTEQ(">>="),
227        GTGTGTEQ(">>>="),
228        MONKEYS_AT("@"),
229        CUSTOM;
230
231        public final String name;
232        public final Tag tag;
233
234        TokenKind() {
235            this(null, Tag.DEFAULT);
236        }
237
238        TokenKind(String name) {
239            this(name, Tag.DEFAULT);
240        }
241
242        TokenKind(Tag tag) {
243            this(null, tag);
244        }
245
246        TokenKind(String name, Tag tag) {
247            this.name = name;
248            this.tag = tag;
249        }
250
251        public String toString() {
252            switch (this) {
253            case IDENTIFIER:
254                return "token.identifier";
255            case CHARLITERAL:
256                return "token.character";
257            case STRINGLITERAL:
258                return "token.string";
259            case INTLITERAL:
260                return "token.integer";
261            case LONGLITERAL:
262                return "token.long-integer";
263            case FLOATLITERAL:
264                return "token.float";
265            case DOUBLELITERAL:
266                return "token.double";
267            case ERROR:
268                return "token.bad-symbol";
269            case EOF:
270                return "token.end-of-input";
271            case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
272            case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
273                return "'" + name + "'";
274            default:
275                return name;
276            }
277        }
278
279        public String getKind() {
280            return "Token";
281        }
282
283        public String toString(Locale locale, Messages messages) {
284            return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
285        }
286
287        @Override
288        public boolean accepts(TokenKind that) {
289            return this == that;
290        }
291    }
292
293    public interface Comment {
294
295        enum CommentStyle {
296            LINE,
297            BLOCK,
298            JAVADOC,
299        }
300
301        String getText();
302        int getSourcePos(int index);
303        CommentStyle getStyle();
304        boolean isDeprecated();
305    }
306
307    /**
308     * This is the class representing a javac token. Each token has several fields
309     * that are set by the javac lexer (i.e. start/end position, string value, etc).
310     */
311    public static class Token {
312
313        /** tags constants **/
314        enum Tag {
315            DEFAULT,
316            NAMED,
317            STRING,
318            NUMERIC
319        }
320
321        /** The token kind */
322        public final TokenKind kind;
323
324        /** The start position of this token */
325        public final int pos;
326
327        /** The end position of this token */
328        public final int endPos;
329
330        /** Comment reader associated with this token */
331        public final List<Comment> comments;
332
333        Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
334            this.kind = kind;
335            this.pos = pos;
336            this.endPos = endPos;
337            this.comments = comments;
338            checkKind();
339        }
340
341        Token[] split(Tokens tokens) {
342            if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
343                throw new AssertionError("Cant split" + kind);
344            }
345
346            TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
347            TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
348
349            if (t1 == null || t2 == null) {
350                throw new AssertionError("Cant split - bad subtokens");
351            }
352            return new Token[] {
353                new Token(t1, pos, pos + t1.name.length(), comments),
354                new Token(t2, pos + t1.name.length(), endPos, null)
355            };
356        }
357
358        protected void checkKind() {
359            if (kind.tag != Tag.DEFAULT) {
360                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
361            }
362        }
363
364        public Name name() {
365            throw new UnsupportedOperationException();
366        }
367
368        public String stringVal() {
369            throw new UnsupportedOperationException();
370        }
371
372        public int radix() {
373            throw new UnsupportedOperationException();
374        }
375
376        /**
377         * Preserve classic semantics - if multiple javadocs are found on the token
378         * the last one is returned
379         */
380        public Comment comment(Comment.CommentStyle style) {
381            List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
382            return comments.isEmpty() ?
383                    null :
384                    comments.head;
385        }
386
387        /**
388         * Preserve classic semantics - deprecated should be set if at least one
389         * javadoc comment attached to this token contains the '@deprecated' string
390         */
391        public boolean deprecatedFlag() {
392            for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
393                if (c.isDeprecated()) {
394                    return true;
395                }
396            }
397            return false;
398        }
399
400        private List<Comment> getComments(Comment.CommentStyle style) {
401            if (comments == null) {
402                return List.nil();
403            } else {
404                ListBuffer<Comment> buf = new ListBuffer<>();
405                for (Comment c : comments) {
406                    if (c.getStyle() == style) {
407                        buf.add(c);
408                    }
409                }
410                return buf.toList();
411            }
412        }
413    }
414
415    final static class NamedToken extends Token {
416        /** The name of this token */
417        public final Name name;
418
419        public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
420            super(kind, pos, endPos, comments);
421            this.name = name;
422        }
423
424        protected void checkKind() {
425            if (kind.tag != Tag.NAMED) {
426                throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
427            }
428        }
429
430        @Override
431        public Name name() {
432            return name;
433        }
434    }
435
436    static class StringToken extends Token {
437        /** The string value of this token */
438        public final String stringVal;
439
440        public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
441            super(kind, pos, endPos, comments);
442            this.stringVal = stringVal;
443        }
444
445        protected void checkKind() {
446            if (kind.tag != Tag.STRING) {
447                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
448            }
449        }
450
451        @Override
452        public String stringVal() {
453            return stringVal;
454        }
455    }
456
457    final static class NumericToken extends StringToken {
458        /** The 'radix' value of this token */
459        public final int radix;
460
461        public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
462            super(kind, pos, endPos, stringVal, comments);
463            this.radix = radix;
464        }
465
466        protected void checkKind() {
467            if (kind.tag != Tag.NUMERIC) {
468                throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
469            }
470        }
471
472        @Override
473        public int radix() {
474            return radix;
475        }
476    }
477
478    public static final Token DUMMY =
479                new Token(TokenKind.ERROR, 0, 0, null);
480}
481