1/**********************************************************************
2
3  parse.y -
4
5  $Author: nagachika $
6  created at: Fri May 28 18:02:42 JST 1993
7
8  Copyright (C) 1993-2007 Yukihiro Matsumoto
9
10**********************************************************************/
11
12%{
13
14#ifndef PARSER_DEBUG
15#define PARSER_DEBUG 0
16#endif
17#define YYDEBUG 1
18#define YYERROR_VERBOSE 1
19#define YYSTACK_USE_ALLOCA 0
20
21#include "ruby/ruby.h"
22#include "ruby/st.h"
23#include "ruby/encoding.h"
24#include "internal.h"
25#include "node.h"
26#include "parse.h"
27#include "id.h"
28#include "regenc.h"
29#include <stdio.h>
30#include <errno.h>
31#include <ctype.h>
32#include "probes.h"
33
34#define numberof(array) (int)(sizeof(array) / sizeof((array)[0]))
35
36#define YYMALLOC(size)		rb_parser_malloc(parser, (size))
37#define YYREALLOC(ptr, size)	rb_parser_realloc(parser, (ptr), (size))
38#define YYCALLOC(nelem, size)	rb_parser_calloc(parser, (nelem), (size))
39#define YYFREE(ptr)		rb_parser_free(parser, (ptr))
40#define malloc	YYMALLOC
41#define realloc	YYREALLOC
42#define calloc	YYCALLOC
43#define free	YYFREE
44
45#ifndef RIPPER
46static ID register_symid(ID, const char *, long, rb_encoding *);
47static ID register_symid_str(ID, VALUE);
48#define REGISTER_SYMID(id, name) register_symid((id), (name), strlen(name), enc)
49#include "id.c"
50#endif
51
52#define is_notop_id(id) ((id)>tLAST_OP_ID)
53#define is_local_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_LOCAL)
54#define is_global_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_GLOBAL)
55#define is_instance_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_INSTANCE)
56#define is_attrset_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_ATTRSET)
57#define is_const_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_CONST)
58#define is_class_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_CLASS)
59#define is_junk_id(id) (is_notop_id(id)&&((id)&ID_SCOPE_MASK)==ID_JUNK)
60#define id_type(id) (is_notop_id(id) ? (int)((id)&ID_SCOPE_MASK) : -1)
61
62#define is_asgn_or_id(id) ((is_notop_id(id)) && \
63	(((id)&ID_SCOPE_MASK) == ID_GLOBAL || \
64	 ((id)&ID_SCOPE_MASK) == ID_INSTANCE || \
65	 ((id)&ID_SCOPE_MASK) == ID_CLASS))
66
67enum lex_state_bits {
68    EXPR_BEG_bit,		/* ignore newline, +/- is a sign. */
69    EXPR_END_bit,		/* newline significant, +/- is an operator. */
70    EXPR_ENDARG_bit,		/* ditto, and unbound braces. */
71    EXPR_ENDFN_bit,		/* ditto, and unbound braces. */
72    EXPR_ARG_bit,		/* newline significant, +/- is an operator. */
73    EXPR_CMDARG_bit,		/* newline significant, +/- is an operator. */
74    EXPR_MID_bit,		/* newline significant, +/- is an operator. */
75    EXPR_FNAME_bit,		/* ignore newline, no reserved words. */
76    EXPR_DOT_bit,		/* right after `.' or `::', no reserved words. */
77    EXPR_CLASS_bit,		/* immediate after `class', no here document. */
78    EXPR_VALUE_bit,		/* alike EXPR_BEG but label is disallowed. */
79    EXPR_MAX_STATE
80};
81/* examine combinations */
82enum lex_state_e {
83#define DEF_EXPR(n) EXPR_##n = (1 << EXPR_##n##_bit)
84    DEF_EXPR(BEG),
85    DEF_EXPR(END),
86    DEF_EXPR(ENDARG),
87    DEF_EXPR(ENDFN),
88    DEF_EXPR(ARG),
89    DEF_EXPR(CMDARG),
90    DEF_EXPR(MID),
91    DEF_EXPR(FNAME),
92    DEF_EXPR(DOT),
93    DEF_EXPR(CLASS),
94    DEF_EXPR(VALUE),
95    EXPR_BEG_ANY  =  (EXPR_BEG | EXPR_VALUE | EXPR_MID | EXPR_CLASS),
96    EXPR_ARG_ANY  =  (EXPR_ARG | EXPR_CMDARG),
97    EXPR_END_ANY  =  (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
98};
99#define IS_lex_state_for(x, ls)	((x) & (ls))
100#define IS_lex_state(ls)	IS_lex_state_for(lex_state, (ls))
101
102#if PARSER_DEBUG
103static const char *lex_state_name(enum lex_state_e state);
104#endif
105
106typedef VALUE stack_type;
107
108# define BITSTACK_PUSH(stack, n)	((stack) = ((stack)<<1)|((n)&1))
109# define BITSTACK_POP(stack)	((stack) = (stack) >> 1)
110# define BITSTACK_LEXPOP(stack)	((stack) = ((stack) >> 1) | ((stack) & 1))
111# define BITSTACK_SET_P(stack)	((stack)&1)
112
113#define COND_PUSH(n)	BITSTACK_PUSH(cond_stack, (n))
114#define COND_POP()	BITSTACK_POP(cond_stack)
115#define COND_LEXPOP()	BITSTACK_LEXPOP(cond_stack)
116#define COND_P()	BITSTACK_SET_P(cond_stack)
117
118#define CMDARG_PUSH(n)	BITSTACK_PUSH(cmdarg_stack, (n))
119#define CMDARG_POP()	BITSTACK_POP(cmdarg_stack)
120#define CMDARG_LEXPOP()	BITSTACK_LEXPOP(cmdarg_stack)
121#define CMDARG_P()	BITSTACK_SET_P(cmdarg_stack)
122
123struct vtable {
124    ID *tbl;
125    int pos;
126    int capa;
127    struct vtable *prev;
128};
129
130struct local_vars {
131    struct vtable *args;
132    struct vtable *vars;
133    struct vtable *used;
134    struct local_vars *prev;
135    stack_type cmdargs;
136};
137
138#define DVARS_INHERIT ((void*)1)
139#define DVARS_TOPSCOPE NULL
140#define DVARS_SPECIAL_P(tbl) (!POINTER_P(tbl))
141#define POINTER_P(val) ((VALUE)(val) & ~(VALUE)3)
142
143static int
144vtable_size(const struct vtable *tbl)
145{
146    if (POINTER_P(tbl)) {
147        return tbl->pos;
148    }
149    else {
150        return 0;
151    }
152}
153
154#define VTBL_DEBUG 0
155
156static struct vtable *
157vtable_alloc(struct vtable *prev)
158{
159    struct vtable *tbl = ALLOC(struct vtable);
160    tbl->pos = 0;
161    tbl->capa = 8;
162    tbl->tbl = ALLOC_N(ID, tbl->capa);
163    tbl->prev = prev;
164    if (VTBL_DEBUG) printf("vtable_alloc: %p\n", (void *)tbl);
165    return tbl;
166}
167
168static void
169vtable_free(struct vtable *tbl)
170{
171    if (VTBL_DEBUG)printf("vtable_free: %p\n", (void *)tbl);
172    if (POINTER_P(tbl)) {
173        if (tbl->tbl) {
174            xfree(tbl->tbl);
175        }
176        xfree(tbl);
177    }
178}
179
180static void
181vtable_add(struct vtable *tbl, ID id)
182{
183    if (!POINTER_P(tbl)) {
184        rb_bug("vtable_add: vtable is not allocated (%p)", (void *)tbl);
185    }
186    if (VTBL_DEBUG) printf("vtable_add: %p, %s\n", (void *)tbl, rb_id2name(id));
187
188    if (tbl->pos == tbl->capa) {
189        tbl->capa = tbl->capa * 2;
190        REALLOC_N(tbl->tbl, ID, tbl->capa);
191    }
192    tbl->tbl[tbl->pos++] = id;
193}
194
195static int
196vtable_included(const struct vtable * tbl, ID id)
197{
198    int i;
199
200    if (POINTER_P(tbl)) {
201        for (i = 0; i < tbl->pos; i++) {
202            if (tbl->tbl[i] == id) {
203                return i+1;
204            }
205        }
206    }
207    return 0;
208}
209
210
211#ifndef RIPPER
212typedef struct token_info {
213    const char *token;
214    int linenum;
215    int column;
216    int nonspc;
217    struct token_info *next;
218} token_info;
219#endif
220
221/*
222    Structure of Lexer Buffer:
223
224 lex_pbeg      tokp         lex_p        lex_pend
225    |           |              |            |
226    |-----------+--------------+------------|
227                |<------------>|
228                     token
229*/
230struct parser_params {
231    int is_ripper;
232    NODE *heap;
233
234    YYSTYPE *parser_yylval;
235    VALUE eofp;
236
237    NODE *parser_lex_strterm;
238    enum lex_state_e parser_lex_state;
239    stack_type parser_cond_stack;
240    stack_type parser_cmdarg_stack;
241    int parser_class_nest;
242    int parser_paren_nest;
243    int parser_lpar_beg;
244    int parser_in_single;
245    int parser_in_def;
246    int parser_brace_nest;
247    int parser_compile_for_eval;
248    VALUE parser_cur_mid;
249    int parser_in_defined;
250    char *parser_tokenbuf;
251    int parser_tokidx;
252    int parser_toksiz;
253    int parser_tokline;
254    VALUE parser_lex_input;
255    VALUE parser_lex_lastline;
256    VALUE parser_lex_nextline;
257    const char *parser_lex_pbeg;
258    const char *parser_lex_p;
259    const char *parser_lex_pend;
260    int parser_heredoc_end;
261    int parser_command_start;
262    NODE *parser_deferred_nodes;
263    long parser_lex_gets_ptr;
264    VALUE (*parser_lex_gets)(struct parser_params*,VALUE);
265    struct local_vars *parser_lvtbl;
266    int parser_ruby__end__seen;
267    int line_count;
268    int has_shebang;
269    char *parser_ruby_sourcefile; /* current source file */
270    int parser_ruby_sourceline;	/* current line no. */
271    VALUE parser_ruby_sourcefile_string;
272    rb_encoding *enc;
273
274    int parser_yydebug;
275
276#ifndef RIPPER
277    /* Ruby core only */
278    NODE *parser_eval_tree_begin;
279    NODE *parser_eval_tree;
280    VALUE debug_lines;
281    VALUE coverage;
282    int nerr;
283
284    int parser_token_info_enabled;
285    token_info *parser_token_info;
286#else
287    /* Ripper only */
288    const char *tokp;
289    VALUE delayed;
290    int delayed_line;
291    int delayed_col;
292
293    VALUE value;
294    VALUE result;
295    VALUE parsing_thread;
296    int toplevel_p;
297#endif
298};
299
300#define STR_NEW(p,n) rb_enc_str_new((p),(n),current_enc)
301#define STR_NEW0() rb_enc_str_new(0,0,current_enc)
302#define STR_NEW2(p) rb_enc_str_new((p),strlen(p),current_enc)
303#define STR_NEW3(p,n,e,func) parser_str_new((p),(n),(e),(func),current_enc)
304#define ENC_SINGLE(cr) ((cr)==ENC_CODERANGE_7BIT)
305#define TOK_INTERN(mb) rb_intern3(tok(), toklen(), current_enc)
306
307static int parser_yyerror(struct parser_params*, const char*);
308#define yyerror(msg) parser_yyerror(parser, (msg))
309
310#define lex_strterm		(parser->parser_lex_strterm)
311#define lex_state		(parser->parser_lex_state)
312#define cond_stack		(parser->parser_cond_stack)
313#define cmdarg_stack		(parser->parser_cmdarg_stack)
314#define class_nest		(parser->parser_class_nest)
315#define paren_nest		(parser->parser_paren_nest)
316#define lpar_beg		(parser->parser_lpar_beg)
317#define brace_nest		(parser->parser_brace_nest)
318#define in_single		(parser->parser_in_single)
319#define in_def			(parser->parser_in_def)
320#define compile_for_eval	(parser->parser_compile_for_eval)
321#define cur_mid			(parser->parser_cur_mid)
322#define in_defined		(parser->parser_in_defined)
323#define tokenbuf		(parser->parser_tokenbuf)
324#define tokidx			(parser->parser_tokidx)
325#define toksiz			(parser->parser_toksiz)
326#define tokline			(parser->parser_tokline)
327#define lex_input		(parser->parser_lex_input)
328#define lex_lastline		(parser->parser_lex_lastline)
329#define lex_nextline		(parser->parser_lex_nextline)
330#define lex_pbeg		(parser->parser_lex_pbeg)
331#define lex_p			(parser->parser_lex_p)
332#define lex_pend		(parser->parser_lex_pend)
333#define heredoc_end		(parser->parser_heredoc_end)
334#define command_start		(parser->parser_command_start)
335#define deferred_nodes		(parser->parser_deferred_nodes)
336#define lex_gets_ptr		(parser->parser_lex_gets_ptr)
337#define lex_gets		(parser->parser_lex_gets)
338#define lvtbl			(parser->parser_lvtbl)
339#define ruby__end__seen		(parser->parser_ruby__end__seen)
340#define ruby_sourceline		(parser->parser_ruby_sourceline)
341#define ruby_sourcefile		(parser->parser_ruby_sourcefile)
342#define ruby_sourcefile_string	(parser->parser_ruby_sourcefile_string)
343#define current_enc		(parser->enc)
344#define yydebug			(parser->parser_yydebug)
345#ifdef RIPPER
346#else
347#define ruby_eval_tree		(parser->parser_eval_tree)
348#define ruby_eval_tree_begin	(parser->parser_eval_tree_begin)
349#define ruby_debug_lines	(parser->debug_lines)
350#define ruby_coverage		(parser->coverage)
351#endif
352
353#if YYPURE
354static int yylex(void*, void*);
355#else
356static int yylex(void*);
357#endif
358
359#ifndef RIPPER
360#define yyparse ruby_yyparse
361
362static NODE* node_newnode(struct parser_params *, enum node_type, VALUE, VALUE, VALUE);
363#define rb_node_newnode(type, a1, a2, a3) node_newnode(parser, (type), (a1), (a2), (a3))
364
365static NODE *cond_gen(struct parser_params*,NODE*);
366#define cond(node) cond_gen(parser, (node))
367static NODE *logop_gen(struct parser_params*,enum node_type,NODE*,NODE*);
368#define logop(type,node1,node2) logop_gen(parser, (type), (node1), (node2))
369
370static NODE *newline_node(NODE*);
371static void fixpos(NODE*,NODE*);
372
373static int value_expr_gen(struct parser_params*,NODE*);
374static void void_expr_gen(struct parser_params*,NODE*);
375static NODE *remove_begin(NODE*);
376#define value_expr(node) value_expr_gen(parser, (node) = remove_begin(node))
377#define void_expr0(node) void_expr_gen(parser, (node))
378#define void_expr(node) void_expr0((node) = remove_begin(node))
379static void void_stmts_gen(struct parser_params*,NODE*);
380#define void_stmts(node) void_stmts_gen(parser, (node))
381static void reduce_nodes_gen(struct parser_params*,NODE**);
382#define reduce_nodes(n) reduce_nodes_gen(parser,(n))
383static void block_dup_check_gen(struct parser_params*,NODE*,NODE*);
384#define block_dup_check(n1,n2) block_dup_check_gen(parser,(n1),(n2))
385
386static NODE *block_append_gen(struct parser_params*,NODE*,NODE*);
387#define block_append(h,t) block_append_gen(parser,(h),(t))
388static NODE *list_append_gen(struct parser_params*,NODE*,NODE*);
389#define list_append(l,i) list_append_gen(parser,(l),(i))
390static NODE *list_concat_gen(struct parser_params*,NODE*,NODE*);
391#define list_concat(h,t) list_concat_gen(parser,(h),(t))
392static NODE *arg_append_gen(struct parser_params*,NODE*,NODE*);
393#define arg_append(h,t) arg_append_gen(parser,(h),(t))
394static NODE *arg_concat_gen(struct parser_params*,NODE*,NODE*);
395#define arg_concat(h,t) arg_concat_gen(parser,(h),(t))
396static NODE *literal_concat_gen(struct parser_params*,NODE*,NODE*);
397#define literal_concat(h,t) literal_concat_gen(parser,(h),(t))
398static int literal_concat0(struct parser_params *, VALUE, VALUE);
399static NODE *new_evstr_gen(struct parser_params*,NODE*);
400#define new_evstr(n) new_evstr_gen(parser,(n))
401static NODE *evstr2dstr_gen(struct parser_params*,NODE*);
402#define evstr2dstr(n) evstr2dstr_gen(parser,(n))
403static NODE *splat_array(NODE*);
404
405static NODE *call_bin_op_gen(struct parser_params*,NODE*,ID,NODE*);
406#define call_bin_op(recv,id,arg1) call_bin_op_gen(parser, (recv),(id),(arg1))
407static NODE *call_uni_op_gen(struct parser_params*,NODE*,ID);
408#define call_uni_op(recv,id) call_uni_op_gen(parser, (recv),(id))
409
410static NODE *new_args_gen(struct parser_params*,NODE*,NODE*,ID,NODE*,NODE*);
411#define new_args(f,o,r,p,t) new_args_gen(parser, (f),(o),(r),(p),(t))
412static NODE *new_args_tail_gen(struct parser_params*,NODE*,ID,ID);
413#define new_args_tail(k,kr,b) new_args_tail_gen(parser, (k),(kr),(b))
414
415static NODE *negate_lit(NODE*);
416static NODE *ret_args_gen(struct parser_params*,NODE*);
417#define ret_args(node) ret_args_gen(parser, (node))
418static NODE *arg_blk_pass(NODE*,NODE*);
419static NODE *new_yield_gen(struct parser_params*,NODE*);
420#define new_yield(node) new_yield_gen(parser, (node))
421static NODE *dsym_node_gen(struct parser_params*,NODE*);
422#define dsym_node(node) dsym_node_gen(parser, (node))
423
424static NODE *gettable_gen(struct parser_params*,ID);
425#define gettable(id) gettable_gen(parser,(id))
426static NODE *assignable_gen(struct parser_params*,ID,NODE*);
427#define assignable(id,node) assignable_gen(parser, (id), (node))
428
429static NODE *aryset_gen(struct parser_params*,NODE*,NODE*);
430#define aryset(node1,node2) aryset_gen(parser, (node1), (node2))
431static NODE *attrset_gen(struct parser_params*,NODE*,ID);
432#define attrset(node,id) attrset_gen(parser, (node), (id))
433
434static void rb_backref_error_gen(struct parser_params*,NODE*);
435#define rb_backref_error(n) rb_backref_error_gen(parser,(n))
436static NODE *node_assign_gen(struct parser_params*,NODE*,NODE*);
437#define node_assign(node1, node2) node_assign_gen(parser, (node1), (node2))
438
439static NODE *new_op_assign_gen(struct parser_params *parser, NODE *lhs, ID op, NODE *rhs);
440static NODE *new_attr_op_assign_gen(struct parser_params *parser, NODE *lhs, ID attr, ID op, NODE *rhs);
441#define new_attr_op_assign(lhs, type, attr, op, rhs) new_attr_op_assign_gen(parser, (lhs), (attr), (op), (rhs))
442static NODE *new_const_op_assign_gen(struct parser_params *parser, NODE *lhs, ID op, NODE *rhs);
443#define new_const_op_assign(lhs, op, rhs) new_const_op_assign_gen(parser, (lhs), (op), (rhs))
444
445static NODE *match_op_gen(struct parser_params*,NODE*,NODE*);
446#define match_op(node1,node2) match_op_gen(parser, (node1), (node2))
447
448static ID  *local_tbl_gen(struct parser_params*);
449#define local_tbl() local_tbl_gen(parser)
450
451static void fixup_nodes(NODE **);
452
453static VALUE reg_compile_gen(struct parser_params*, VALUE, int);
454#define reg_compile(str,options) reg_compile_gen(parser, (str), (options))
455static void reg_fragment_setenc_gen(struct parser_params*, VALUE, int);
456#define reg_fragment_setenc(str,options) reg_fragment_setenc_gen(parser, (str), (options))
457static int reg_fragment_check_gen(struct parser_params*, VALUE, int);
458#define reg_fragment_check(str,options) reg_fragment_check_gen(parser, (str), (options))
459static NODE *reg_named_capture_assign_gen(struct parser_params* parser, VALUE regexp, NODE *match);
460#define reg_named_capture_assign(regexp,match) reg_named_capture_assign_gen(parser,(regexp),(match))
461
462#define get_id(id) (id)
463#define get_value(val) (val)
464#else
465#define value_expr(node) ((void)(node))
466#define remove_begin(node) (node)
467#define rb_dvar_defined(id) 0
468#define rb_local_defined(id) 0
469static ID ripper_get_id(VALUE);
470#define get_id(id) ripper_get_id(id)
471static VALUE ripper_get_value(VALUE);
472#define get_value(val) ripper_get_value(val)
473static VALUE assignable_gen(struct parser_params*,VALUE);
474#define assignable(lhs,node) assignable_gen(parser, (lhs))
475static int id_is_var_gen(struct parser_params *parser, ID id);
476#define id_is_var(id) id_is_var_gen(parser, (id))
477
478#define node_assign(node1, node2) dispatch2(assign, (node1), (node2))
479
480static VALUE new_op_assign_gen(struct parser_params *parser, VALUE lhs, VALUE op, VALUE rhs);
481static VALUE new_attr_op_assign_gen(struct parser_params *parser, VALUE lhs, VALUE type, VALUE attr, VALUE op, VALUE rhs);
482#define new_attr_op_assign(lhs, type, attr, op, rhs) new_attr_op_assign_gen(parser, (lhs), (type), (attr), (op), (rhs))
483
484#endif /* !RIPPER */
485
486#define new_op_assign(lhs, op, rhs) new_op_assign_gen(parser, (lhs), (op), (rhs))
487
488static ID formal_argument_gen(struct parser_params*, ID);
489#define formal_argument(id) formal_argument_gen(parser, (id))
490static ID shadowing_lvar_gen(struct parser_params*,ID);
491#define shadowing_lvar(name) shadowing_lvar_gen(parser, (name))
492static void new_bv_gen(struct parser_params*,ID);
493#define new_bv(id) new_bv_gen(parser, (id))
494
495static void local_push_gen(struct parser_params*,int);
496#define local_push(top) local_push_gen(parser,(top))
497static void local_pop_gen(struct parser_params*);
498#define local_pop() local_pop_gen(parser)
499static int local_var_gen(struct parser_params*, ID);
500#define local_var(id) local_var_gen(parser, (id))
501static int arg_var_gen(struct parser_params*, ID);
502#define arg_var(id) arg_var_gen(parser, (id))
503static int  local_id_gen(struct parser_params*, ID);
504#define local_id(id) local_id_gen(parser, (id))
505static ID   internal_id_gen(struct parser_params*);
506#define internal_id() internal_id_gen(parser)
507
508static const struct vtable *dyna_push_gen(struct parser_params *);
509#define dyna_push() dyna_push_gen(parser)
510static void dyna_pop_gen(struct parser_params*, const struct vtable *);
511#define dyna_pop(node) dyna_pop_gen(parser, (node))
512static int dyna_in_block_gen(struct parser_params*);
513#define dyna_in_block() dyna_in_block_gen(parser)
514#define dyna_var(id) local_var(id)
515static int dvar_defined_gen(struct parser_params*,ID,int);
516#define dvar_defined(id) dvar_defined_gen(parser, (id), 0)
517#define dvar_defined_get(id) dvar_defined_gen(parser, (id), 1)
518static int dvar_curr_gen(struct parser_params*,ID);
519#define dvar_curr(id) dvar_curr_gen(parser, (id))
520
521static int lvar_defined_gen(struct parser_params*, ID);
522#define lvar_defined(id) lvar_defined_gen(parser, (id))
523
524#define RE_OPTION_ONCE (1<<16)
525#define RE_OPTION_ENCODING_SHIFT 8
526#define RE_OPTION_ENCODING(e) (((e)&0xff)<<RE_OPTION_ENCODING_SHIFT)
527#define RE_OPTION_ENCODING_IDX(o) (((o)>>RE_OPTION_ENCODING_SHIFT)&0xff)
528#define RE_OPTION_ENCODING_NONE(o) ((o)&RE_OPTION_ARG_ENCODING_NONE)
529#define RE_OPTION_MASK  0xff
530#define RE_OPTION_ARG_ENCODING_NONE 32
531
532#define NODE_STRTERM NODE_ZARRAY	/* nothing to gc */
533#define NODE_HEREDOC NODE_ARRAY 	/* 1, 3 to gc */
534#define SIGN_EXTEND(x,n) (((1<<(n)-1)^((x)&~(~0<<(n))))-(1<<(n)-1))
535#define nd_func u1.id
536#if SIZEOF_SHORT == 2
537#define nd_term(node) ((signed short)(node)->u2.id)
538#else
539#define nd_term(node) SIGN_EXTEND((node)->u2.id, CHAR_BIT*2)
540#endif
541#define nd_paren(node) (char)((node)->u2.id >> CHAR_BIT*2)
542#define nd_nest u3.cnt
543
544/****** Ripper *******/
545
546#ifdef RIPPER
547#define RIPPER_VERSION "0.1.0"
548
549#include "eventids1.c"
550#include "eventids2.c"
551
552static VALUE ripper_dispatch0(struct parser_params*,ID);
553static VALUE ripper_dispatch1(struct parser_params*,ID,VALUE);
554static VALUE ripper_dispatch2(struct parser_params*,ID,VALUE,VALUE);
555static VALUE ripper_dispatch3(struct parser_params*,ID,VALUE,VALUE,VALUE);
556static VALUE ripper_dispatch4(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE);
557static VALUE ripper_dispatch5(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE);
558static VALUE ripper_dispatch7(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE);
559
560#define dispatch0(n)            ripper_dispatch0(parser, TOKEN_PASTE(ripper_id_, n))
561#define dispatch1(n,a)          ripper_dispatch1(parser, TOKEN_PASTE(ripper_id_, n), (a))
562#define dispatch2(n,a,b)        ripper_dispatch2(parser, TOKEN_PASTE(ripper_id_, n), (a), (b))
563#define dispatch3(n,a,b,c)      ripper_dispatch3(parser, TOKEN_PASTE(ripper_id_, n), (a), (b), (c))
564#define dispatch4(n,a,b,c,d)    ripper_dispatch4(parser, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d))
565#define dispatch5(n,a,b,c,d,e)  ripper_dispatch5(parser, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d), (e))
566#define dispatch7(n,a,b,c,d,e,f,g) ripper_dispatch7(parser, TOKEN_PASTE(ripper_id_, n), (a), (b), (c), (d), (e), (f), (g))
567
568#define yyparse ripper_yyparse
569
570#define ripper_intern(s) ID2SYM(rb_intern(s))
571static VALUE ripper_id2sym(ID);
572#ifdef __GNUC__
573#define ripper_id2sym(id) ((id) < 256 && rb_ispunct(id) ? \
574			   ID2SYM(id) : ripper_id2sym(id))
575#endif
576
577#define arg_new() dispatch0(args_new)
578#define arg_add(l,a) dispatch2(args_add, (l), (a))
579#define arg_add_star(l,a) dispatch2(args_add_star, (l), (a))
580#define arg_add_block(l,b) dispatch2(args_add_block, (l), (b))
581#define arg_add_optblock(l,b) ((b)==Qundef? (l) : dispatch2(args_add_block, (l), (b)))
582#define bare_assoc(v) dispatch1(bare_assoc_hash, (v))
583#define arg_add_assocs(l,b) arg_add((l), bare_assoc(b))
584
585#define args2mrhs(a) dispatch1(mrhs_new_from_args, (a))
586#define mrhs_new() dispatch0(mrhs_new)
587#define mrhs_add(l,a) dispatch2(mrhs_add, (l), (a))
588#define mrhs_add_star(l,a) dispatch2(mrhs_add_star, (l), (a))
589
590#define mlhs_new() dispatch0(mlhs_new)
591#define mlhs_add(l,a) dispatch2(mlhs_add, (l), (a))
592#define mlhs_add_star(l,a) dispatch2(mlhs_add_star, (l), (a))
593
594#define params_new(pars, opts, rest, pars2, kws, kwrest, blk) \
595        dispatch7(params, (pars), (opts), (rest), (pars2), (kws), (kwrest), (blk))
596
597#define blockvar_new(p,v) dispatch2(block_var, (p), (v))
598#define blockvar_add_star(l,a) dispatch2(block_var_add_star, (l), (a))
599#define blockvar_add_block(l,a) dispatch2(block_var_add_block, (l), (a))
600
601#define method_optarg(m,a) ((a)==Qundef ? (m) : dispatch2(method_add_arg,(m),(a)))
602#define method_arg(m,a) dispatch2(method_add_arg,(m),(a))
603#define method_add_block(m,b) dispatch2(method_add_block, (m), (b))
604
605#define escape_Qundef(x) ((x)==Qundef ? Qnil : (x))
606
607static inline VALUE
608new_args_gen(struct parser_params *parser, VALUE f, VALUE o, VALUE r, VALUE p, VALUE tail)
609{
610    NODE *t = (NODE *)tail;
611    VALUE k = t->u1.value, kr = t->u2.value, b = t->u3.value;
612    return params_new(f, o, r, p, k, kr, escape_Qundef(b));
613}
614#define new_args(f,o,r,p,t) new_args_gen(parser, (f),(o),(r),(p),(t))
615
616static inline VALUE
617new_args_tail_gen(struct parser_params *parser, VALUE k, VALUE kr, VALUE b)
618{
619    return (VALUE)rb_node_newnode(NODE_MEMO, k, kr, b);
620}
621#define new_args_tail(k,kr,b) new_args_tail_gen(parser, (k),(kr),(b))
622
623#define FIXME 0
624
625#endif /* RIPPER */
626
627#ifndef RIPPER
628# define Qnone 0
629# define ifndef_ripper(x) (x)
630#else
631# define Qnone Qnil
632# define ifndef_ripper(x)
633#endif
634
635#ifndef RIPPER
636# define rb_warn0(fmt)    rb_compile_warn(ruby_sourcefile, ruby_sourceline, (fmt))
637# define rb_warnI(fmt,a)  rb_compile_warn(ruby_sourcefile, ruby_sourceline, (fmt), (a))
638# define rb_warnS(fmt,a)  rb_compile_warn(ruby_sourcefile, ruby_sourceline, (fmt), (a))
639# define rb_warn4S(file,line,fmt,a)  rb_compile_warn((file), (line), (fmt), (a))
640# define rb_warning0(fmt) rb_compile_warning(ruby_sourcefile, ruby_sourceline, (fmt))
641# define rb_warningS(fmt,a) rb_compile_warning(ruby_sourcefile, ruby_sourceline, (fmt), (a))
642#else
643# define rb_warn0(fmt)    ripper_warn0(parser, (fmt))
644# define rb_warnI(fmt,a)  ripper_warnI(parser, (fmt), (a))
645# define rb_warnS(fmt,a)  ripper_warnS(parser, (fmt), (a))
646# define rb_warn4S(file,line,fmt,a)  ripper_warnS(parser, (fmt), (a))
647# define rb_warning0(fmt) ripper_warning0(parser, (fmt))
648# define rb_warningS(fmt,a) ripper_warningS(parser, (fmt), (a))
649static void ripper_warn0(struct parser_params*, const char*);
650static void ripper_warnI(struct parser_params*, const char*, int);
651static void ripper_warnS(struct parser_params*, const char*, const char*);
652static void ripper_warning0(struct parser_params*, const char*);
653static void ripper_warningS(struct parser_params*, const char*, const char*);
654#endif
655
656#ifdef RIPPER
657static void ripper_compile_error(struct parser_params*, const char *fmt, ...);
658# define rb_compile_error ripper_compile_error
659# define compile_error ripper_compile_error
660# define PARSER_ARG parser,
661#else
662# define rb_compile_error rb_compile_error_with_enc
663# define compile_error parser->nerr++,rb_compile_error_with_enc
664# define PARSER_ARG ruby_sourcefile, ruby_sourceline, current_enc,
665#endif
666
667/* Older versions of Yacc set YYMAXDEPTH to a very low value by default (150,
668   for instance).  This is too low for Ruby to parse some files, such as
669   date/format.rb, therefore bump the value up to at least Bison's default. */
670#ifdef OLD_YACC
671#ifndef YYMAXDEPTH
672#define YYMAXDEPTH 10000
673#endif
674#endif
675
676#ifndef RIPPER
677static void token_info_push(struct parser_params*, const char *token);
678static void token_info_pop(struct parser_params*, const char *token);
679#define token_info_push(token) (RTEST(ruby_verbose) ? token_info_push(parser, (token)) : (void)0)
680#define token_info_pop(token) (RTEST(ruby_verbose) ? token_info_pop(parser, (token)) : (void)0)
681#else
682#define token_info_push(token) /* nothing */
683#define token_info_pop(token) /* nothing */
684#endif
685%}
686
687%pure-parser
688%lex-param {struct parser_params *parser}
689%parse-param {struct parser_params *parser}
690
691%union {
692    VALUE val;
693    NODE *node;
694    ID id;
695    int num;
696    const struct vtable *vars;
697}
698
699/*%%%*/
700%token
701/*%
702%token <val>
703%*/
704	keyword_class
705	keyword_module
706	keyword_def
707	keyword_undef
708	keyword_begin
709	keyword_rescue
710	keyword_ensure
711	keyword_end
712	keyword_if
713	keyword_unless
714	keyword_then
715	keyword_elsif
716	keyword_else
717	keyword_case
718	keyword_when
719	keyword_while
720	keyword_until
721	keyword_for
722	keyword_break
723	keyword_next
724	keyword_redo
725	keyword_retry
726	keyword_in
727	keyword_do
728	keyword_do_cond
729	keyword_do_block
730	keyword_do_LAMBDA
731	keyword_return
732	keyword_yield
733	keyword_super
734	keyword_self
735	keyword_nil
736	keyword_true
737	keyword_false
738	keyword_and
739	keyword_or
740	keyword_not
741	modifier_if
742	modifier_unless
743	modifier_while
744	modifier_until
745	modifier_rescue
746	keyword_alias
747	keyword_defined
748	keyword_BEGIN
749	keyword_END
750	keyword__LINE__
751	keyword__FILE__
752	keyword__ENCODING__
753
754%token <id>   tIDENTIFIER tFID tGVAR tIVAR tCONSTANT tCVAR tLABEL
755%token <node> tINTEGER tFLOAT tSTRING_CONTENT tCHAR
756%token <node> tNTH_REF tBACK_REF
757%token <num>  tREGEXP_END
758
759%type <node> singleton strings string string1 xstring regexp
760%type <node> string_contents xstring_contents regexp_contents string_content
761%type <node> words symbols symbol_list qwords qsymbols word_list qword_list qsym_list word
762%type <node> literal numeric dsym cpath
763%type <node> top_compstmt top_stmts top_stmt
764%type <node> bodystmt compstmt stmts stmt_or_begin stmt expr arg primary command command_call method_call
765%type <node> expr_value arg_value primary_value fcall
766%type <node> if_tail opt_else case_body cases opt_rescue exc_list exc_var opt_ensure
767%type <node> args call_args opt_call_args
768%type <node> paren_args opt_paren_args args_tail opt_args_tail block_args_tail opt_block_args_tail
769%type <node> command_args aref_args opt_block_arg block_arg var_ref var_lhs
770%type <node> command_asgn mrhs superclass block_call block_command
771%type <node> f_block_optarg f_block_opt
772%type <node> f_arglist f_args f_arg f_arg_item f_optarg f_marg f_marg_list f_margs
773%type <node> assoc_list assocs assoc undef_list backref string_dvar for_var
774%type <node> block_param opt_block_param block_param_def f_opt
775%type <node> f_kwarg f_kw f_block_kwarg f_block_kw
776%type <node> bv_decls opt_bv_decl bvar
777%type <node> lambda f_larglist lambda_body
778%type <node> brace_block cmd_brace_block do_block lhs none fitem
779%type <node> mlhs mlhs_head mlhs_basic mlhs_item mlhs_node mlhs_post mlhs_inner
780%type <id>   fsym keyword_variable user_variable sym symbol operation operation2 operation3
781%type <id>   cname fname op f_rest_arg f_block_arg opt_f_block_arg f_norm_arg f_bad_arg
782%type <id>   f_kwrest
783/*%%%*/
784/*%
785%type <val> program reswords then do dot_or_colon
786%*/
787%token END_OF_INPUT 0	"end-of-input"
788%token tUPLUS		RUBY_TOKEN(UPLUS)  "unary+"
789%token tUMINUS		RUBY_TOKEN(UMINUS) "unary-"
790%token tPOW		RUBY_TOKEN(POW)    "**"
791%token tCMP		RUBY_TOKEN(CMP)    "<=>"
792%token tEQ		RUBY_TOKEN(EQ)     "=="
793%token tEQQ		RUBY_TOKEN(EQQ)    "==="
794%token tNEQ		RUBY_TOKEN(NEQ)    "!="
795%token tGEQ		RUBY_TOKEN(GEQ)    ">="
796%token tLEQ		RUBY_TOKEN(LEQ)    "<="
797%token tANDOP		"&&"
798%token tOROP		"||"
799%token tMATCH		RUBY_TOKEN(MATCH)  "=~"
800%token tNMATCH		RUBY_TOKEN(NMATCH) "!~"
801%token tDOT2		RUBY_TOKEN(DOT2)   ".."
802%token tDOT3		RUBY_TOKEN(DOT3)   "..."
803%token tAREF		RUBY_TOKEN(AREF)   "[]"
804%token tASET		RUBY_TOKEN(ASET)   "[]="
805%token tLSHFT		RUBY_TOKEN(LSHFT)  "<<"
806%token tRSHFT		RUBY_TOKEN(RSHFT)  ">>"
807%token tCOLON2		"::"
808%token tCOLON3		":: at EXPR_BEG"
809%token <id> tOP_ASGN	/* +=, -=  etc. */
810%token tASSOC		"=>"
811%token tLPAREN		"("
812%token tLPAREN_ARG	"( arg"
813%token tRPAREN		")"
814%token tLBRACK		"["
815%token tLBRACE		"{"
816%token tLBRACE_ARG	"{ arg"
817%token tSTAR		"*"
818%token tDSTAR		"**arg"
819%token tAMPER		"&"
820%token tLAMBDA		"->"
821%token tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG tWORDS_BEG tQWORDS_BEG tSYMBOLS_BEG tQSYMBOLS_BEG
822%token tSTRING_DBEG tSTRING_DEND tSTRING_DVAR tSTRING_END tLAMBEG
823
824/*
825 *	precedence table
826 */
827
828%nonassoc tLOWEST
829%nonassoc tLBRACE_ARG
830
831%nonassoc  modifier_if modifier_unless modifier_while modifier_until
832%left  keyword_or keyword_and
833%right keyword_not
834%nonassoc keyword_defined
835%right '=' tOP_ASGN
836%left modifier_rescue
837%right '?' ':'
838%nonassoc tDOT2 tDOT3
839%left  tOROP
840%left  tANDOP
841%nonassoc  tCMP tEQ tEQQ tNEQ tMATCH tNMATCH
842%left  '>' tGEQ '<' tLEQ
843%left  '|' '^'
844%left  '&'
845%left  tLSHFT tRSHFT
846%left  '+' '-'
847%left  '*' '/' '%'
848%right tUMINUS_NUM tUMINUS
849%right tPOW
850%right '!' '~' tUPLUS
851
852%token tLAST_TOKEN
853
854%%
855program		:  {
856			lex_state = EXPR_BEG;
857		    /*%%%*/
858			local_push(compile_for_eval || rb_parse_in_main());
859		    /*%
860			local_push(0);
861		    %*/
862		    }
863		  top_compstmt
864		    {
865		    /*%%%*/
866			if ($2 && !compile_for_eval) {
867			    /* last expression should not be void */
868			    if (nd_type($2) != NODE_BLOCK) void_expr($2);
869			    else {
870				NODE *node = $2;
871				while (node->nd_next) {
872				    node = node->nd_next;
873				}
874				void_expr(node->nd_head);
875			    }
876			}
877			ruby_eval_tree = NEW_SCOPE(0, block_append(ruby_eval_tree, $2));
878		    /*%
879			$$ = $2;
880			parser->result = dispatch1(program, $$);
881		    %*/
882			local_pop();
883		    }
884		;
885
886top_compstmt	: top_stmts opt_terms
887		    {
888		    /*%%%*/
889			void_stmts($1);
890			fixup_nodes(&deferred_nodes);
891		    /*%
892		    %*/
893			$$ = $1;
894		    }
895		;
896
897top_stmts	: none
898                    {
899		    /*%%%*/
900			$$ = NEW_BEGIN(0);
901		    /*%
902			$$ = dispatch2(stmts_add, dispatch0(stmts_new),
903						  dispatch0(void_stmt));
904		    %*/
905		    }
906		| top_stmt
907		    {
908		    /*%%%*/
909			$$ = newline_node($1);
910		    /*%
911			$$ = dispatch2(stmts_add, dispatch0(stmts_new), $1);
912		    %*/
913		    }
914		| top_stmts terms top_stmt
915		    {
916		    /*%%%*/
917			$$ = block_append($1, newline_node($3));
918		    /*%
919			$$ = dispatch2(stmts_add, $1, $3);
920		    %*/
921		    }
922		| error top_stmt
923		    {
924			$$ = remove_begin($2);
925		    }
926		;
927
928top_stmt	: stmt
929		| keyword_BEGIN
930		    {
931		    /*%%%*/
932			/* local_push(0); */
933		    /*%
934		    %*/
935		    }
936		  '{' top_compstmt '}'
937		    {
938		    /*%%%*/
939			ruby_eval_tree_begin = block_append(ruby_eval_tree_begin,
940							    $4);
941			/* NEW_PREEXE($4)); */
942			/* local_pop(); */
943			$$ = NEW_BEGIN(0);
944		    /*%
945			$$ = dispatch1(BEGIN, $4);
946		    %*/
947		    }
948		;
949
950bodystmt	: compstmt
951		  opt_rescue
952		  opt_else
953		  opt_ensure
954		    {
955		    /*%%%*/
956			$$ = $1;
957			if ($2) {
958			    $$ = NEW_RESCUE($1, $2, $3);
959			}
960			else if ($3) {
961			    rb_warn0("else without rescue is useless");
962			    $$ = block_append($$, $3);
963			}
964			if ($4) {
965			    if ($$) {
966				$$ = NEW_ENSURE($$, $4);
967			    }
968			    else {
969				$$ = block_append($4, NEW_NIL());
970			    }
971			}
972			fixpos($$, $1);
973		    /*%
974			$$ = dispatch4(bodystmt,
975				       escape_Qundef($1),
976				       escape_Qundef($2),
977				       escape_Qundef($3),
978				       escape_Qundef($4));
979		    %*/
980		    }
981		;
982
983compstmt	: stmts opt_terms
984		    {
985		    /*%%%*/
986			void_stmts($1);
987			fixup_nodes(&deferred_nodes);
988		    /*%
989		    %*/
990			$$ = $1;
991		    }
992		;
993
994stmts		: none
995                    {
996		    /*%%%*/
997			$$ = NEW_BEGIN(0);
998		    /*%
999			$$ = dispatch2(stmts_add, dispatch0(stmts_new),
1000						  dispatch0(void_stmt));
1001		    %*/
1002		    }
1003		| stmt_or_begin
1004		    {
1005		    /*%%%*/
1006			$$ = newline_node($1);
1007		    /*%
1008			$$ = dispatch2(stmts_add, dispatch0(stmts_new), $1);
1009		    %*/
1010		    }
1011		| stmts terms stmt_or_begin
1012		    {
1013		    /*%%%*/
1014			$$ = block_append($1, newline_node($3));
1015		    /*%
1016			$$ = dispatch2(stmts_add, $1, $3);
1017		    %*/
1018		    }
1019		| error stmt
1020		    {
1021			$$ = remove_begin($2);
1022		    }
1023		;
1024
1025stmt_or_begin	: stmt
1026                    {
1027			$$ = $1;
1028		    }
1029                | keyword_BEGIN
1030		    {
1031			yyerror("BEGIN is permitted only at toplevel");
1032		    /*%%%*/
1033			/* local_push(0); */
1034		    /*%
1035		    %*/
1036		    }
1037		  '{' top_compstmt '}'
1038		    {
1039		    /*%%%*/
1040			ruby_eval_tree_begin = block_append(ruby_eval_tree_begin,
1041							    $4);
1042			/* NEW_PREEXE($4)); */
1043			/* local_pop(); */
1044			$$ = NEW_BEGIN(0);
1045		    /*%
1046			$$ = dispatch1(BEGIN, $4);
1047		    %*/
1048		    }
1049
1050stmt		: keyword_alias fitem {lex_state = EXPR_FNAME;} fitem
1051		    {
1052		    /*%%%*/
1053			$$ = NEW_ALIAS($2, $4);
1054		    /*%
1055			$$ = dispatch2(alias, $2, $4);
1056		    %*/
1057		    }
1058		| keyword_alias tGVAR tGVAR
1059		    {
1060		    /*%%%*/
1061			$$ = NEW_VALIAS($2, $3);
1062		    /*%
1063			$$ = dispatch2(var_alias, $2, $3);
1064		    %*/
1065		    }
1066		| keyword_alias tGVAR tBACK_REF
1067		    {
1068		    /*%%%*/
1069			char buf[2];
1070			buf[0] = '$';
1071			buf[1] = (char)$3->nd_nth;
1072			$$ = NEW_VALIAS($2, rb_intern2(buf, 2));
1073		    /*%
1074			$$ = dispatch2(var_alias, $2, $3);
1075		    %*/
1076		    }
1077		| keyword_alias tGVAR tNTH_REF
1078		    {
1079		    /*%%%*/
1080			yyerror("can't make alias for the number variables");
1081			$$ = NEW_BEGIN(0);
1082		    /*%
1083			$$ = dispatch2(var_alias, $2, $3);
1084			$$ = dispatch1(alias_error, $$);
1085		    %*/
1086		    }
1087		| keyword_undef undef_list
1088		    {
1089		    /*%%%*/
1090			$$ = $2;
1091		    /*%
1092			$$ = dispatch1(undef, $2);
1093		    %*/
1094		    }
1095		| stmt modifier_if expr_value
1096		    {
1097		    /*%%%*/
1098			$$ = NEW_IF(cond($3), remove_begin($1), 0);
1099			fixpos($$, $3);
1100		    /*%
1101			$$ = dispatch2(if_mod, $3, $1);
1102		    %*/
1103		    }
1104		| stmt modifier_unless expr_value
1105		    {
1106		    /*%%%*/
1107			$$ = NEW_UNLESS(cond($3), remove_begin($1), 0);
1108			fixpos($$, $3);
1109		    /*%
1110			$$ = dispatch2(unless_mod, $3, $1);
1111		    %*/
1112		    }
1113		| stmt modifier_while expr_value
1114		    {
1115		    /*%%%*/
1116			if ($1 && nd_type($1) == NODE_BEGIN) {
1117			    $$ = NEW_WHILE(cond($3), $1->nd_body, 0);
1118			}
1119			else {
1120			    $$ = NEW_WHILE(cond($3), $1, 1);
1121			}
1122		    /*%
1123			$$ = dispatch2(while_mod, $3, $1);
1124		    %*/
1125		    }
1126		| stmt modifier_until expr_value
1127		    {
1128		    /*%%%*/
1129			if ($1 && nd_type($1) == NODE_BEGIN) {
1130			    $$ = NEW_UNTIL(cond($3), $1->nd_body, 0);
1131			}
1132			else {
1133			    $$ = NEW_UNTIL(cond($3), $1, 1);
1134			}
1135		    /*%
1136			$$ = dispatch2(until_mod, $3, $1);
1137		    %*/
1138		    }
1139		| stmt modifier_rescue stmt
1140		    {
1141		    /*%%%*/
1142			NODE *resq = NEW_RESBODY(0, remove_begin($3), 0);
1143			$$ = NEW_RESCUE(remove_begin($1), resq, 0);
1144		    /*%
1145			$$ = dispatch2(rescue_mod, $1, $3);
1146		    %*/
1147		    }
1148		| keyword_END '{' compstmt '}'
1149		    {
1150			if (in_def || in_single) {
1151			    rb_warn0("END in method; use at_exit");
1152			}
1153		    /*%%%*/
1154			$$ = NEW_POSTEXE(NEW_NODE(
1155			    NODE_SCOPE, 0 /* tbl */, $3 /* body */, 0 /* args */));
1156		    /*%
1157			$$ = dispatch1(END, $3);
1158		    %*/
1159		    }
1160		| command_asgn
1161		| mlhs '=' command_call
1162		    {
1163		    /*%%%*/
1164			value_expr($3);
1165			$1->nd_value = $3;
1166			$$ = $1;
1167		    /*%
1168			$$ = dispatch2(massign, $1, $3);
1169		    %*/
1170		    }
1171		| var_lhs tOP_ASGN command_call
1172		    {
1173			value_expr($3);
1174			$$ = new_op_assign($1, $2, $3);
1175		    }
1176		| primary_value '[' opt_call_args rbracket tOP_ASGN command_call
1177		    {
1178		    /*%%%*/
1179			NODE *args;
1180
1181			value_expr($6);
1182			if (!$3) $3 = NEW_ZARRAY();
1183			args = arg_concat($3, $6);
1184			if ($5 == tOROP) {
1185			    $5 = 0;
1186			}
1187			else if ($5 == tANDOP) {
1188			    $5 = 1;
1189			}
1190			$$ = NEW_OP_ASGN1($1, $5, args);
1191			fixpos($$, $1);
1192		    /*%
1193			$$ = dispatch2(aref_field, $1, escape_Qundef($3));
1194			$$ = dispatch3(opassign, $$, $5, $6);
1195		    %*/
1196		    }
1197		| primary_value '.' tIDENTIFIER tOP_ASGN command_call
1198		    {
1199			value_expr($5);
1200			$$ = new_attr_op_assign($1, ripper_id2sym('.'), $3, $4, $5);
1201		    }
1202		| primary_value '.' tCONSTANT tOP_ASGN command_call
1203		    {
1204			value_expr($5);
1205			$$ = new_attr_op_assign($1, ripper_id2sym('.'), $3, $4, $5);
1206		    }
1207		| primary_value tCOLON2 tCONSTANT tOP_ASGN command_call
1208		    {
1209		    /*%%%*/
1210			$$ = NEW_COLON2($1, $3);
1211			$$ = new_const_op_assign($$, $4, $5);
1212		    /*%
1213			$$ = dispatch2(const_path_field, $1, $3);
1214			$$ = dispatch3(opassign, $$, $4, $5);
1215		    %*/
1216		    }
1217		| primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_call
1218		    {
1219			value_expr($5);
1220			$$ = new_attr_op_assign($1, ripper_intern("::"), $3, $4, $5);
1221		    }
1222		| backref tOP_ASGN command_call
1223		    {
1224		    /*%%%*/
1225			rb_backref_error($1);
1226			$$ = NEW_BEGIN(0);
1227		    /*%
1228			$$ = dispatch2(assign, dispatch1(var_field, $1), $3);
1229			$$ = dispatch1(assign_error, $$);
1230		    %*/
1231		    }
1232		| lhs '=' mrhs
1233		    {
1234		    /*%%%*/
1235			value_expr($3);
1236			$$ = node_assign($1, $3);
1237		    /*%
1238			$$ = dispatch2(assign, $1, $3);
1239		    %*/
1240		    }
1241		| mlhs '=' arg_value
1242		    {
1243		    /*%%%*/
1244			$1->nd_value = $3;
1245			$$ = $1;
1246		    /*%
1247			$$ = dispatch2(massign, $1, $3);
1248		    %*/
1249		    }
1250		| mlhs '=' mrhs
1251		    {
1252		    /*%%%*/
1253			$1->nd_value = $3;
1254			$$ = $1;
1255		    /*%
1256			$$ = dispatch2(massign, $1, $3);
1257		    %*/
1258		    }
1259		| expr
1260		;
1261
1262command_asgn	: lhs '=' command_call
1263		    {
1264		    /*%%%*/
1265			value_expr($3);
1266			$$ = node_assign($1, $3);
1267		    /*%
1268			$$ = dispatch2(assign, $1, $3);
1269		    %*/
1270		    }
1271		| lhs '=' command_asgn
1272		    {
1273		    /*%%%*/
1274			value_expr($3);
1275			$$ = node_assign($1, $3);
1276		    /*%
1277			$$ = dispatch2(assign, $1, $3);
1278		    %*/
1279		    }
1280		;
1281
1282
1283expr		: command_call
1284		| expr keyword_and expr
1285		    {
1286		    /*%%%*/
1287			$$ = logop(NODE_AND, $1, $3);
1288		    /*%
1289			$$ = dispatch3(binary, $1, ripper_intern("and"), $3);
1290		    %*/
1291		    }
1292		| expr keyword_or expr
1293		    {
1294		    /*%%%*/
1295			$$ = logop(NODE_OR, $1, $3);
1296		    /*%
1297			$$ = dispatch3(binary, $1, ripper_intern("or"), $3);
1298		    %*/
1299		    }
1300		| keyword_not opt_nl expr
1301		    {
1302		    /*%%%*/
1303			$$ = call_uni_op(cond($3), '!');
1304		    /*%
1305			$$ = dispatch2(unary, ripper_intern("not"), $3);
1306		    %*/
1307		    }
1308		| '!' command_call
1309		    {
1310		    /*%%%*/
1311			$$ = call_uni_op(cond($2), '!');
1312		    /*%
1313			$$ = dispatch2(unary, ripper_id2sym('!'), $2);
1314		    %*/
1315		    }
1316		| arg
1317		;
1318
1319expr_value	: expr
1320		    {
1321		    /*%%%*/
1322			value_expr($1);
1323			$$ = $1;
1324		        if (!$$) $$ = NEW_NIL();
1325		    /*%
1326			$$ = $1;
1327		    %*/
1328		    }
1329		;
1330
1331command_call	: command
1332		| block_command
1333		;
1334
1335block_command	: block_call
1336		| block_call dot_or_colon operation2 command_args
1337		    {
1338		    /*%%%*/
1339			$$ = NEW_CALL($1, $3, $4);
1340		    /*%
1341			$$ = dispatch3(call, $1, $2, $3);
1342			$$ = method_arg($$, $4);
1343		    %*/
1344		    }
1345		;
1346
1347cmd_brace_block	: tLBRACE_ARG
1348		    {
1349			$<vars>1 = dyna_push();
1350		    /*%%%*/
1351			$<num>$ = ruby_sourceline;
1352		    /*%
1353		    %*/
1354		    }
1355		  opt_block_param
1356		  compstmt
1357		  '}'
1358		    {
1359		    /*%%%*/
1360			$$ = NEW_ITER($3,$4);
1361			nd_set_line($$, $<num>2);
1362		    /*%
1363			$$ = dispatch2(brace_block, escape_Qundef($3), $4);
1364		    %*/
1365			dyna_pop($<vars>1);
1366		    }
1367		;
1368
1369fcall		: operation
1370		    {
1371		    /*%%%*/
1372			$$ = NEW_FCALL($1, 0);
1373			nd_set_line($$, tokline);
1374		    /*%
1375		    %*/
1376		    }
1377		;
1378
1379command		: fcall command_args       %prec tLOWEST
1380		    {
1381		    /*%%%*/
1382			$$ = $1;
1383			$$->nd_args = $2;
1384		    /*%
1385			$$ = dispatch2(command, $1, $2);
1386		    %*/
1387		    }
1388		| fcall command_args cmd_brace_block
1389		    {
1390		    /*%%%*/
1391			block_dup_check($2,$3);
1392			$1->nd_args = $2;
1393		        $3->nd_iter = $1;
1394			$$ = $3;
1395			fixpos($$, $1);
1396		    /*%
1397			$$ = dispatch2(command, $1, $2);
1398			$$ = method_add_block($$, $3);
1399		    %*/
1400		    }
1401		| primary_value '.' operation2 command_args	%prec tLOWEST
1402		    {
1403		    /*%%%*/
1404			$$ = NEW_CALL($1, $3, $4);
1405			fixpos($$, $1);
1406		    /*%
1407			$$ = dispatch4(command_call, $1, ripper_id2sym('.'), $3, $4);
1408		    %*/
1409		    }
1410		| primary_value '.' operation2 command_args cmd_brace_block
1411		    {
1412		    /*%%%*/
1413			block_dup_check($4,$5);
1414		        $5->nd_iter = NEW_CALL($1, $3, $4);
1415			$$ = $5;
1416			fixpos($$, $1);
1417		    /*%
1418			$$ = dispatch4(command_call, $1, ripper_id2sym('.'), $3, $4);
1419			$$ = method_add_block($$, $5);
1420		    %*/
1421		   }
1422		| primary_value tCOLON2 operation2 command_args	%prec tLOWEST
1423		    {
1424		    /*%%%*/
1425			$$ = NEW_CALL($1, $3, $4);
1426			fixpos($$, $1);
1427		    /*%
1428			$$ = dispatch4(command_call, $1, ripper_intern("::"), $3, $4);
1429		    %*/
1430		    }
1431		| primary_value tCOLON2 operation2 command_args cmd_brace_block
1432		    {
1433		    /*%%%*/
1434			block_dup_check($4,$5);
1435		        $5->nd_iter = NEW_CALL($1, $3, $4);
1436			$$ = $5;
1437			fixpos($$, $1);
1438		    /*%
1439			$$ = dispatch4(command_call, $1, ripper_intern("::"), $3, $4);
1440			$$ = method_add_block($$, $5);
1441		    %*/
1442		   }
1443		| keyword_super command_args
1444		    {
1445		    /*%%%*/
1446			$$ = NEW_SUPER($2);
1447			fixpos($$, $2);
1448		    /*%
1449			$$ = dispatch1(super, $2);
1450		    %*/
1451		    }
1452		| keyword_yield command_args
1453		    {
1454		    /*%%%*/
1455			$$ = new_yield($2);
1456			fixpos($$, $2);
1457		    /*%
1458			$$ = dispatch1(yield, $2);
1459		    %*/
1460		    }
1461		| keyword_return call_args
1462		    {
1463		    /*%%%*/
1464			$$ = NEW_RETURN(ret_args($2));
1465		    /*%
1466			$$ = dispatch1(return, $2);
1467		    %*/
1468		    }
1469		| keyword_break call_args
1470		    {
1471		    /*%%%*/
1472			$$ = NEW_BREAK(ret_args($2));
1473		    /*%
1474			$$ = dispatch1(break, $2);
1475		    %*/
1476		    }
1477		| keyword_next call_args
1478		    {
1479		    /*%%%*/
1480			$$ = NEW_NEXT(ret_args($2));
1481		    /*%
1482			$$ = dispatch1(next, $2);
1483		    %*/
1484		    }
1485		;
1486
1487mlhs		: mlhs_basic
1488		| tLPAREN mlhs_inner rparen
1489		    {
1490		    /*%%%*/
1491			$$ = $2;
1492		    /*%
1493			$$ = dispatch1(mlhs_paren, $2);
1494		    %*/
1495		    }
1496		;
1497
1498mlhs_inner	: mlhs_basic
1499		| tLPAREN mlhs_inner rparen
1500		    {
1501		    /*%%%*/
1502			$$ = NEW_MASGN(NEW_LIST($2), 0);
1503		    /*%
1504			$$ = dispatch1(mlhs_paren, $2);
1505		    %*/
1506		    }
1507		;
1508
1509mlhs_basic	: mlhs_head
1510		    {
1511		    /*%%%*/
1512			$$ = NEW_MASGN($1, 0);
1513		    /*%
1514			$$ = $1;
1515		    %*/
1516		    }
1517		| mlhs_head mlhs_item
1518		    {
1519		    /*%%%*/
1520			$$ = NEW_MASGN(list_append($1,$2), 0);
1521		    /*%
1522			$$ = mlhs_add($1, $2);
1523		    %*/
1524		    }
1525		| mlhs_head tSTAR mlhs_node
1526		    {
1527		    /*%%%*/
1528			$$ = NEW_MASGN($1, $3);
1529		    /*%
1530			$$ = mlhs_add_star($1, $3);
1531		    %*/
1532		    }
1533		| mlhs_head tSTAR mlhs_node ',' mlhs_post
1534		    {
1535		    /*%%%*/
1536			$$ = NEW_MASGN($1, NEW_POSTARG($3,$5));
1537		    /*%
1538			$1 = mlhs_add_star($1, $3);
1539			$$ = mlhs_add($1, $5);
1540		    %*/
1541		    }
1542		| mlhs_head tSTAR
1543		    {
1544		    /*%%%*/
1545			$$ = NEW_MASGN($1, -1);
1546		    /*%
1547			$$ = mlhs_add_star($1, Qnil);
1548		    %*/
1549		    }
1550		| mlhs_head tSTAR ',' mlhs_post
1551		    {
1552		    /*%%%*/
1553			$$ = NEW_MASGN($1, NEW_POSTARG(-1, $4));
1554		    /*%
1555			$1 = mlhs_add_star($1, Qnil);
1556			$$ = mlhs_add($1, $4);
1557		    %*/
1558		    }
1559		| tSTAR mlhs_node
1560		    {
1561		    /*%%%*/
1562			$$ = NEW_MASGN(0, $2);
1563		    /*%
1564			$$ = mlhs_add_star(mlhs_new(), $2);
1565		    %*/
1566		    }
1567		| tSTAR mlhs_node ',' mlhs_post
1568		    {
1569		    /*%%%*/
1570			$$ = NEW_MASGN(0, NEW_POSTARG($2,$4));
1571		    /*%
1572			$2 = mlhs_add_star(mlhs_new(), $2);
1573			$$ = mlhs_add($2, $4);
1574		    %*/
1575		    }
1576		| tSTAR
1577		    {
1578		    /*%%%*/
1579			$$ = NEW_MASGN(0, -1);
1580		    /*%
1581			$$ = mlhs_add_star(mlhs_new(), Qnil);
1582		    %*/
1583		    }
1584		| tSTAR ',' mlhs_post
1585		    {
1586		    /*%%%*/
1587			$$ = NEW_MASGN(0, NEW_POSTARG(-1, $3));
1588		    /*%
1589			$$ = mlhs_add_star(mlhs_new(), Qnil);
1590			$$ = mlhs_add($$, $3);
1591		    %*/
1592		    }
1593		;
1594
1595mlhs_item	: mlhs_node
1596		| tLPAREN mlhs_inner rparen
1597		    {
1598		    /*%%%*/
1599			$$ = $2;
1600		    /*%
1601			$$ = dispatch1(mlhs_paren, $2);
1602		    %*/
1603		    }
1604		;
1605
1606mlhs_head	: mlhs_item ','
1607		    {
1608		    /*%%%*/
1609			$$ = NEW_LIST($1);
1610		    /*%
1611			$$ = mlhs_add(mlhs_new(), $1);
1612		    %*/
1613		    }
1614		| mlhs_head mlhs_item ','
1615		    {
1616		    /*%%%*/
1617			$$ = list_append($1, $2);
1618		    /*%
1619			$$ = mlhs_add($1, $2);
1620		    %*/
1621		    }
1622		;
1623
1624mlhs_post	: mlhs_item
1625		    {
1626		    /*%%%*/
1627			$$ = NEW_LIST($1);
1628		    /*%
1629			$$ = mlhs_add(mlhs_new(), $1);
1630		    %*/
1631		    }
1632		| mlhs_post ',' mlhs_item
1633		    {
1634		    /*%%%*/
1635			$$ = list_append($1, $3);
1636		    /*%
1637			$$ = mlhs_add($1, $3);
1638		    %*/
1639		    }
1640		;
1641
1642mlhs_node	: user_variable
1643		    {
1644			$$ = assignable($1, 0);
1645		    }
1646		| keyword_variable
1647		    {
1648		        $$ = assignable($1, 0);
1649		    }
1650		| primary_value '[' opt_call_args rbracket
1651		    {
1652		    /*%%%*/
1653			$$ = aryset($1, $3);
1654		    /*%
1655			$$ = dispatch2(aref_field, $1, escape_Qundef($3));
1656		    %*/
1657		    }
1658		| primary_value '.' tIDENTIFIER
1659		    {
1660		    /*%%%*/
1661			$$ = attrset($1, $3);
1662		    /*%
1663			$$ = dispatch3(field, $1, ripper_id2sym('.'), $3);
1664		    %*/
1665		    }
1666		| primary_value tCOLON2 tIDENTIFIER
1667		    {
1668		    /*%%%*/
1669			$$ = attrset($1, $3);
1670		    /*%
1671			$$ = dispatch2(const_path_field, $1, $3);
1672		    %*/
1673		    }
1674		| primary_value '.' tCONSTANT
1675		    {
1676		    /*%%%*/
1677			$$ = attrset($1, $3);
1678		    /*%
1679			$$ = dispatch3(field, $1, ripper_id2sym('.'), $3);
1680		    %*/
1681		    }
1682		| primary_value tCOLON2 tCONSTANT
1683		    {
1684		    /*%%%*/
1685			if (in_def || in_single)
1686			    yyerror("dynamic constant assignment");
1687			$$ = NEW_CDECL(0, 0, NEW_COLON2($1, $3));
1688		    /*%
1689			if (in_def || in_single)
1690			    yyerror("dynamic constant assignment");
1691			$$ = dispatch2(const_path_field, $1, $3);
1692		    %*/
1693		    }
1694		| tCOLON3 tCONSTANT
1695		    {
1696		    /*%%%*/
1697			if (in_def || in_single)
1698			    yyerror("dynamic constant assignment");
1699			$$ = NEW_CDECL(0, 0, NEW_COLON3($2));
1700		    /*%
1701			$$ = dispatch1(top_const_field, $2);
1702		    %*/
1703		    }
1704		| backref
1705		    {
1706		    /*%%%*/
1707			rb_backref_error($1);
1708			$$ = NEW_BEGIN(0);
1709		    /*%
1710			$$ = dispatch1(var_field, $1);
1711			$$ = dispatch1(assign_error, $$);
1712		    %*/
1713		    }
1714		;
1715
1716lhs		: user_variable
1717		    {
1718			$$ = assignable($1, 0);
1719		    /*%%%*/
1720			if (!$$) $$ = NEW_BEGIN(0);
1721		    /*%
1722			$$ = dispatch1(var_field, $$);
1723		    %*/
1724		    }
1725		| keyword_variable
1726		    {
1727		        $$ = assignable($1, 0);
1728		    /*%%%*/
1729		        if (!$$) $$ = NEW_BEGIN(0);
1730		    /*%
1731		        $$ = dispatch1(var_field, $$);
1732		    %*/
1733		    }
1734		| primary_value '[' opt_call_args rbracket
1735		    {
1736		    /*%%%*/
1737			$$ = aryset($1, $3);
1738		    /*%
1739			$$ = dispatch2(aref_field, $1, escape_Qundef($3));
1740		    %*/
1741		    }
1742		| primary_value '.' tIDENTIFIER
1743		    {
1744		    /*%%%*/
1745			$$ = attrset($1, $3);
1746		    /*%
1747			$$ = dispatch3(field, $1, ripper_id2sym('.'), $3);
1748		    %*/
1749		    }
1750		| primary_value tCOLON2 tIDENTIFIER
1751		    {
1752		    /*%%%*/
1753			$$ = attrset($1, $3);
1754		    /*%
1755			$$ = dispatch3(field, $1, ripper_intern("::"), $3);
1756		    %*/
1757		    }
1758		| primary_value '.' tCONSTANT
1759		    {
1760		    /*%%%*/
1761			$$ = attrset($1, $3);
1762		    /*%
1763			$$ = dispatch3(field, $1, ripper_id2sym('.'), $3);
1764		    %*/
1765		    }
1766		| primary_value tCOLON2 tCONSTANT
1767		    {
1768		    /*%%%*/
1769			if (in_def || in_single)
1770			    yyerror("dynamic constant assignment");
1771			$$ = NEW_CDECL(0, 0, NEW_COLON2($1, $3));
1772		    /*%
1773			$$ = dispatch2(const_path_field, $1, $3);
1774			if (in_def || in_single) {
1775			    $$ = dispatch1(assign_error, $$);
1776			}
1777		    %*/
1778		    }
1779		| tCOLON3 tCONSTANT
1780		    {
1781		    /*%%%*/
1782			if (in_def || in_single)
1783			    yyerror("dynamic constant assignment");
1784			$$ = NEW_CDECL(0, 0, NEW_COLON3($2));
1785		    /*%
1786			$$ = dispatch1(top_const_field, $2);
1787			if (in_def || in_single) {
1788			    $$ = dispatch1(assign_error, $$);
1789			}
1790		    %*/
1791		    }
1792		| backref
1793		    {
1794		    /*%%%*/
1795			rb_backref_error($1);
1796			$$ = NEW_BEGIN(0);
1797		    /*%
1798			$$ = dispatch1(assign_error, $1);
1799		    %*/
1800		    }
1801		;
1802
1803cname		: tIDENTIFIER
1804		    {
1805		    /*%%%*/
1806			yyerror("class/module name must be CONSTANT");
1807		    /*%
1808			$$ = dispatch1(class_name_error, $1);
1809		    %*/
1810		    }
1811		| tCONSTANT
1812		;
1813
1814cpath		: tCOLON3 cname
1815		    {
1816		    /*%%%*/
1817			$$ = NEW_COLON3($2);
1818		    /*%
1819			$$ = dispatch1(top_const_ref, $2);
1820		    %*/
1821		    }
1822		| cname
1823		    {
1824		    /*%%%*/
1825			$$ = NEW_COLON2(0, $$);
1826		    /*%
1827			$$ = dispatch1(const_ref, $1);
1828		    %*/
1829		    }
1830		| primary_value tCOLON2 cname
1831		    {
1832		    /*%%%*/
1833			$$ = NEW_COLON2($1, $3);
1834		    /*%
1835			$$ = dispatch2(const_path_ref, $1, $3);
1836		    %*/
1837		    }
1838		;
1839
1840fname		: tIDENTIFIER
1841		| tCONSTANT
1842		| tFID
1843		| op
1844		    {
1845			lex_state = EXPR_ENDFN;
1846			$$ = $1;
1847		    }
1848		| reswords
1849		    {
1850			lex_state = EXPR_ENDFN;
1851		    /*%%%*/
1852			$$ = $<id>1;
1853		    /*%
1854			$$ = $1;
1855		    %*/
1856		    }
1857		;
1858
1859fsym		: fname
1860		| symbol
1861		;
1862
1863fitem		: fsym
1864		    {
1865		    /*%%%*/
1866			$$ = NEW_LIT(ID2SYM($1));
1867		    /*%
1868			$$ = dispatch1(symbol_literal, $1);
1869		    %*/
1870		    }
1871		| dsym
1872		;
1873
1874undef_list	: fitem
1875		    {
1876		    /*%%%*/
1877			$$ = NEW_UNDEF($1);
1878		    /*%
1879			$$ = rb_ary_new3(1, $1);
1880		    %*/
1881		    }
1882		| undef_list ',' {lex_state = EXPR_FNAME;} fitem
1883		    {
1884		    /*%%%*/
1885			$$ = block_append($1, NEW_UNDEF($4));
1886		    /*%
1887			rb_ary_push($1, $4);
1888		    %*/
1889		    }
1890		;
1891
1892op		: '|'		{ ifndef_ripper($$ = '|'); }
1893		| '^'		{ ifndef_ripper($$ = '^'); }
1894		| '&'		{ ifndef_ripper($$ = '&'); }
1895		| tCMP		{ ifndef_ripper($$ = tCMP); }
1896		| tEQ		{ ifndef_ripper($$ = tEQ); }
1897		| tEQQ		{ ifndef_ripper($$ = tEQQ); }
1898		| tMATCH	{ ifndef_ripper($$ = tMATCH); }
1899		| tNMATCH	{ ifndef_ripper($$ = tNMATCH); }
1900		| '>'		{ ifndef_ripper($$ = '>'); }
1901		| tGEQ		{ ifndef_ripper($$ = tGEQ); }
1902		| '<'		{ ifndef_ripper($$ = '<'); }
1903		| tLEQ		{ ifndef_ripper($$ = tLEQ); }
1904		| tNEQ		{ ifndef_ripper($$ = tNEQ); }
1905		| tLSHFT	{ ifndef_ripper($$ = tLSHFT); }
1906		| tRSHFT	{ ifndef_ripper($$ = tRSHFT); }
1907		| '+'		{ ifndef_ripper($$ = '+'); }
1908		| '-'		{ ifndef_ripper($$ = '-'); }
1909		| '*'		{ ifndef_ripper($$ = '*'); }
1910		| tSTAR		{ ifndef_ripper($$ = '*'); }
1911		| '/'		{ ifndef_ripper($$ = '/'); }
1912		| '%'		{ ifndef_ripper($$ = '%'); }
1913		| tPOW		{ ifndef_ripper($$ = tPOW); }
1914		| tDSTAR	{ ifndef_ripper($$ = tDSTAR); }
1915		| '!'		{ ifndef_ripper($$ = '!'); }
1916		| '~'		{ ifndef_ripper($$ = '~'); }
1917		| tUPLUS	{ ifndef_ripper($$ = tUPLUS); }
1918		| tUMINUS	{ ifndef_ripper($$ = tUMINUS); }
1919		| tAREF		{ ifndef_ripper($$ = tAREF); }
1920		| tASET		{ ifndef_ripper($$ = tASET); }
1921		| '`'		{ ifndef_ripper($$ = '`'); }
1922		;
1923
1924reswords	: keyword__LINE__ | keyword__FILE__ | keyword__ENCODING__
1925		| keyword_BEGIN | keyword_END
1926		| keyword_alias | keyword_and | keyword_begin
1927		| keyword_break | keyword_case | keyword_class | keyword_def
1928		| keyword_defined | keyword_do | keyword_else | keyword_elsif
1929		| keyword_end | keyword_ensure | keyword_false
1930		| keyword_for | keyword_in | keyword_module | keyword_next
1931		| keyword_nil | keyword_not | keyword_or | keyword_redo
1932		| keyword_rescue | keyword_retry | keyword_return | keyword_self
1933		| keyword_super | keyword_then | keyword_true | keyword_undef
1934		| keyword_when | keyword_yield | keyword_if | keyword_unless
1935		| keyword_while | keyword_until
1936		;
1937
1938arg		: lhs '=' arg
1939		    {
1940		    /*%%%*/
1941			value_expr($3);
1942			$$ = node_assign($1, $3);
1943		    /*%
1944			$$ = dispatch2(assign, $1, $3);
1945		    %*/
1946		    }
1947		| lhs '=' arg modifier_rescue arg
1948		    {
1949		    /*%%%*/
1950			value_expr($3);
1951		        $3 = NEW_RESCUE($3, NEW_RESBODY(0,$5,0), 0);
1952			$$ = node_assign($1, $3);
1953		    /*%
1954			$$ = dispatch2(assign, $1, dispatch2(rescue_mod, $3, $5));
1955		    %*/
1956		    }
1957		| var_lhs tOP_ASGN arg
1958		    {
1959			value_expr($3);
1960			$$ = new_op_assign($1, $2, $3);
1961		    }
1962		| var_lhs tOP_ASGN arg modifier_rescue arg
1963		    {
1964		    /*%%%*/
1965			value_expr($3);
1966		        $3 = NEW_RESCUE($3, NEW_RESBODY(0,$5,0), 0);
1967		    /*%
1968			$3 = dispatch2(rescue_mod, $3, $5);
1969		    %*/
1970			$$ = new_op_assign($1, $2, $3);
1971		    }
1972		| primary_value '[' opt_call_args rbracket tOP_ASGN arg
1973		    {
1974		    /*%%%*/
1975			NODE *args;
1976
1977			value_expr($6);
1978			if (!$3) $3 = NEW_ZARRAY();
1979			if (nd_type($3) == NODE_BLOCK_PASS) {
1980			    args = NEW_ARGSCAT($3, $6);
1981			}
1982		        else {
1983			    args = arg_concat($3, $6);
1984		        }
1985			if ($5 == tOROP) {
1986			    $5 = 0;
1987			}
1988			else if ($5 == tANDOP) {
1989			    $5 = 1;
1990			}
1991			$$ = NEW_OP_ASGN1($1, $5, args);
1992			fixpos($$, $1);
1993		    /*%
1994			$1 = dispatch2(aref_field, $1, escape_Qundef($3));
1995			$$ = dispatch3(opassign, $1, $5, $6);
1996		    %*/
1997		    }
1998		| primary_value '.' tIDENTIFIER tOP_ASGN arg
1999		    {
2000			value_expr($5);
2001			$$ = new_attr_op_assign($1, ripper_id2sym('.'), $3, $4, $5);
2002		    }
2003		| primary_value '.' tCONSTANT tOP_ASGN arg
2004		    {
2005			value_expr($5);
2006			$$ = new_attr_op_assign($1, ripper_id2sym('.'), $3, $4, $5);
2007		    }
2008		| primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg
2009		    {
2010			value_expr($5);
2011			$$ = new_attr_op_assign($1, ripper_intern("::"), $3, $4, $5);
2012		    }
2013		| primary_value tCOLON2 tCONSTANT tOP_ASGN arg
2014		    {
2015		    /*%%%*/
2016			$$ = NEW_COLON2($1, $3);
2017			$$ = new_const_op_assign($$, $4, $5);
2018		    /*%
2019			$$ = dispatch2(const_path_field, $1, $3);
2020			$$ = dispatch3(opassign, $$, $4, $5);
2021		    %*/
2022		    }
2023		| tCOLON3 tCONSTANT tOP_ASGN arg
2024		    {
2025		    /*%%%*/
2026			$$ = NEW_COLON3($2);
2027			$$ = new_const_op_assign($$, $3, $4);
2028		    /*%
2029			$$ = dispatch1(top_const_field, $2);
2030			$$ = dispatch3(opassign, $$, $3, $4);
2031		    %*/
2032		    }
2033		| backref tOP_ASGN arg
2034		    {
2035		    /*%%%*/
2036			rb_backref_error($1);
2037			$$ = NEW_BEGIN(0);
2038		    /*%
2039			$$ = dispatch1(var_field, $1);
2040			$$ = dispatch3(opassign, $$, $2, $3);
2041			$$ = dispatch1(assign_error, $$);
2042		    %*/
2043		    }
2044		| arg tDOT2 arg
2045		    {
2046		    /*%%%*/
2047			value_expr($1);
2048			value_expr($3);
2049			$$ = NEW_DOT2($1, $3);
2050			if (nd_type($1) == NODE_LIT && FIXNUM_P($1->nd_lit) &&
2051			    nd_type($3) == NODE_LIT && FIXNUM_P($3->nd_lit)) {
2052			    deferred_nodes = list_append(deferred_nodes, $$);
2053			}
2054		    /*%
2055			$$ = dispatch2(dot2, $1, $3);
2056		    %*/
2057		    }
2058		| arg tDOT3 arg
2059		    {
2060		    /*%%%*/
2061			value_expr($1);
2062			value_expr($3);
2063			$$ = NEW_DOT3($1, $3);
2064			if (nd_type($1) == NODE_LIT && FIXNUM_P($1->nd_lit) &&
2065			    nd_type($3) == NODE_LIT && FIXNUM_P($3->nd_lit)) {
2066			    deferred_nodes = list_append(deferred_nodes, $$);
2067			}
2068		    /*%
2069			$$ = dispatch2(dot3, $1, $3);
2070		    %*/
2071		    }
2072		| arg '+' arg
2073		    {
2074		    /*%%%*/
2075			$$ = call_bin_op($1, '+', $3);
2076		    /*%
2077			$$ = dispatch3(binary, $1, ID2SYM('+'), $3);
2078		    %*/
2079		    }
2080		| arg '-' arg
2081		    {
2082		    /*%%%*/
2083			$$ = call_bin_op($1, '-', $3);
2084		    /*%
2085			$$ = dispatch3(binary, $1, ID2SYM('-'), $3);
2086		    %*/
2087		    }
2088		| arg '*' arg
2089		    {
2090		    /*%%%*/
2091			$$ = call_bin_op($1, '*', $3);
2092		    /*%
2093			$$ = dispatch3(binary, $1, ID2SYM('*'), $3);
2094		    %*/
2095		    }
2096		| arg '/' arg
2097		    {
2098		    /*%%%*/
2099			$$ = call_bin_op($1, '/', $3);
2100		    /*%
2101			$$ = dispatch3(binary, $1, ID2SYM('/'), $3);
2102		    %*/
2103		    }
2104		| arg '%' arg
2105		    {
2106		    /*%%%*/
2107			$$ = call_bin_op($1, '%', $3);
2108		    /*%
2109			$$ = dispatch3(binary, $1, ID2SYM('%'), $3);
2110		    %*/
2111		    }
2112		| arg tPOW arg
2113		    {
2114		    /*%%%*/
2115			$$ = call_bin_op($1, tPOW, $3);
2116		    /*%
2117			$$ = dispatch3(binary, $1, ripper_intern("**"), $3);
2118		    %*/
2119		    }
2120		| tUMINUS_NUM tINTEGER tPOW arg
2121		    {
2122		    /*%%%*/
2123			$$ = NEW_CALL(call_bin_op($2, tPOW, $4), tUMINUS, 0);
2124		    /*%
2125			$$ = dispatch3(binary, $2, ripper_intern("**"), $4);
2126			$$ = dispatch2(unary, ripper_intern("-@"), $$);
2127		    %*/
2128		    }
2129		| tUMINUS_NUM tFLOAT tPOW arg
2130		    {
2131		    /*%%%*/
2132			$$ = NEW_CALL(call_bin_op($2, tPOW, $4), tUMINUS, 0);
2133		    /*%
2134			$$ = dispatch3(binary, $2, ripper_intern("**"), $4);
2135			$$ = dispatch2(unary, ripper_intern("-@"), $$);
2136		    %*/
2137		    }
2138		| tUPLUS arg
2139		    {
2140		    /*%%%*/
2141			$$ = call_uni_op($2, tUPLUS);
2142		    /*%
2143			$$ = dispatch2(unary, ripper_intern("+@"), $2);
2144		    %*/
2145		    }
2146		| tUMINUS arg
2147		    {
2148		    /*%%%*/
2149			$$ = call_uni_op($2, tUMINUS);
2150		    /*%
2151			$$ = dispatch2(unary, ripper_intern("-@"), $2);
2152		    %*/
2153		    }
2154		| arg '|' arg
2155		    {
2156		    /*%%%*/
2157			$$ = call_bin_op($1, '|', $3);
2158		    /*%
2159			$$ = dispatch3(binary, $1, ID2SYM('|'), $3);
2160		    %*/
2161		    }
2162		| arg '^' arg
2163		    {
2164		    /*%%%*/
2165			$$ = call_bin_op($1, '^', $3);
2166		    /*%
2167			$$ = dispatch3(binary, $1, ID2SYM('^'), $3);
2168		    %*/
2169		    }
2170		| arg '&' arg
2171		    {
2172		    /*%%%*/
2173			$$ = call_bin_op($1, '&', $3);
2174		    /*%
2175			$$ = dispatch3(binary, $1, ID2SYM('&'), $3);
2176		    %*/
2177		    }
2178		| arg tCMP arg
2179		    {
2180		    /*%%%*/
2181			$$ = call_bin_op($1, tCMP, $3);
2182		    /*%
2183			$$ = dispatch3(binary, $1, ripper_intern("<=>"), $3);
2184		    %*/
2185		    }
2186		| arg '>' arg
2187		    {
2188		    /*%%%*/
2189			$$ = call_bin_op($1, '>', $3);
2190		    /*%
2191			$$ = dispatch3(binary, $1, ID2SYM('>'), $3);
2192		    %*/
2193		    }
2194		| arg tGEQ arg
2195		    {
2196		    /*%%%*/
2197			$$ = call_bin_op($1, tGEQ, $3);
2198		    /*%
2199			$$ = dispatch3(binary, $1, ripper_intern(">="), $3);
2200		    %*/
2201		    }
2202		| arg '<' arg
2203		    {
2204		    /*%%%*/
2205			$$ = call_bin_op($1, '<', $3);
2206		    /*%
2207			$$ = dispatch3(binary, $1, ID2SYM('<'), $3);
2208		    %*/
2209		    }
2210		| arg tLEQ arg
2211		    {
2212		    /*%%%*/
2213			$$ = call_bin_op($1, tLEQ, $3);
2214		    /*%
2215			$$ = dispatch3(binary, $1, ripper_intern("<="), $3);
2216		    %*/
2217		    }
2218		| arg tEQ arg
2219		    {
2220		    /*%%%*/
2221			$$ = call_bin_op($1, tEQ, $3);
2222		    /*%
2223			$$ = dispatch3(binary, $1, ripper_intern("=="), $3);
2224		    %*/
2225		    }
2226		| arg tEQQ arg
2227		    {
2228		    /*%%%*/
2229			$$ = call_bin_op($1, tEQQ, $3);
2230		    /*%
2231			$$ = dispatch3(binary, $1, ripper_intern("==="), $3);
2232		    %*/
2233		    }
2234		| arg tNEQ arg
2235		    {
2236		    /*%%%*/
2237			$$ = call_bin_op($1, tNEQ, $3);
2238		    /*%
2239			$$ = dispatch3(binary, $1, ripper_intern("!="), $3);
2240		    %*/
2241		    }
2242		| arg tMATCH arg
2243		    {
2244		    /*%%%*/
2245			$$ = match_op($1, $3);
2246                        if (nd_type($1) == NODE_LIT && RB_TYPE_P($1->nd_lit, T_REGEXP)) {
2247                            $$ = reg_named_capture_assign($1->nd_lit, $$);
2248                        }
2249		    /*%
2250			$$ = dispatch3(binary, $1, ripper_intern("=~"), $3);
2251		    %*/
2252		    }
2253		| arg tNMATCH arg
2254		    {
2255		    /*%%%*/
2256			$$ = call_bin_op($1, tNMATCH, $3);
2257		    /*%
2258			$$ = dispatch3(binary, $1, ripper_intern("!~"), $3);
2259		    %*/
2260		    }
2261		| '!' arg
2262		    {
2263		    /*%%%*/
2264			$$ = call_uni_op(cond($2), '!');
2265		    /*%
2266			$$ = dispatch2(unary, ID2SYM('!'), $2);
2267		    %*/
2268		    }
2269		| '~' arg
2270		    {
2271		    /*%%%*/
2272			$$ = call_uni_op($2, '~');
2273		    /*%
2274			$$ = dispatch2(unary, ID2SYM('~'), $2);
2275		    %*/
2276		    }
2277		| arg tLSHFT arg
2278		    {
2279		    /*%%%*/
2280			$$ = call_bin_op($1, tLSHFT, $3);
2281		    /*%
2282			$$ = dispatch3(binary, $1, ripper_intern("<<"), $3);
2283		    %*/
2284		    }
2285		| arg tRSHFT arg
2286		    {
2287		    /*%%%*/
2288			$$ = call_bin_op($1, tRSHFT, $3);
2289		    /*%
2290			$$ = dispatch3(binary, $1, ripper_intern(">>"), $3);
2291		    %*/
2292		    }
2293		| arg tANDOP arg
2294		    {
2295		    /*%%%*/
2296			$$ = logop(NODE_AND, $1, $3);
2297		    /*%
2298			$$ = dispatch3(binary, $1, ripper_intern("&&"), $3);
2299		    %*/
2300		    }
2301		| arg tOROP arg
2302		    {
2303		    /*%%%*/
2304			$$ = logop(NODE_OR, $1, $3);
2305		    /*%
2306			$$ = dispatch3(binary, $1, ripper_intern("||"), $3);
2307		    %*/
2308		    }
2309		| keyword_defined opt_nl {in_defined = 1;} arg
2310		    {
2311		    /*%%%*/
2312			in_defined = 0;
2313			$$ = NEW_DEFINED($4);
2314		    /*%
2315			in_defined = 0;
2316			$$ = dispatch1(defined, $4);
2317		    %*/
2318		    }
2319		| arg '?' arg opt_nl ':' arg
2320		    {
2321		    /*%%%*/
2322			value_expr($1);
2323			$$ = NEW_IF(cond($1), $3, $6);
2324			fixpos($$, $1);
2325		    /*%
2326			$$ = dispatch3(ifop, $1, $3, $6);
2327		    %*/
2328		    }
2329		| primary
2330		    {
2331			$$ = $1;
2332		    }
2333		;
2334
2335arg_value	: arg
2336		    {
2337		    /*%%%*/
2338			value_expr($1);
2339			$$ = $1;
2340		        if (!$$) $$ = NEW_NIL();
2341		    /*%
2342			$$ = $1;
2343		    %*/
2344		    }
2345		;
2346
2347aref_args	: none
2348		| args trailer
2349		    {
2350			$$ = $1;
2351		    }
2352		| args ',' assocs trailer
2353		    {
2354		    /*%%%*/
2355			$$ = arg_append($1, NEW_HASH($3));
2356		    /*%
2357			$$ = arg_add_assocs($1, $3);
2358		    %*/
2359		    }
2360		| assocs trailer
2361		    {
2362		    /*%%%*/
2363			$$ = NEW_LIST(NEW_HASH($1));
2364		    /*%
2365			$$ = arg_add_assocs(arg_new(), $1);
2366		    %*/
2367		    }
2368		;
2369
2370paren_args	: '(' opt_call_args rparen
2371		    {
2372		    /*%%%*/
2373			$$ = $2;
2374		    /*%
2375			$$ = dispatch1(arg_paren, escape_Qundef($2));
2376		    %*/
2377		    }
2378		;
2379
2380opt_paren_args	: none
2381		| paren_args
2382		;
2383
2384opt_call_args	: none
2385		| call_args
2386		| args ','
2387		    {
2388		      $$ = $1;
2389		    }
2390		| args ',' assocs ','
2391		    {
2392		    /*%%%*/
2393			$$ = arg_append($1, NEW_HASH($3));
2394		    /*%
2395			$$ = arg_add_assocs($1, $3);
2396		    %*/
2397		    }
2398		| assocs ','
2399		    {
2400		    /*%%%*/
2401			$$ = NEW_LIST(NEW_HASH($1));
2402		    /*%
2403			$$ = arg_add_assocs(arg_new(), $1);
2404		    %*/
2405		    }
2406		;
2407
2408call_args	: command
2409		    {
2410		    /*%%%*/
2411			value_expr($1);
2412			$$ = NEW_LIST($1);
2413		    /*%
2414			$$ = arg_add(arg_new(), $1);
2415		    %*/
2416		    }
2417		| args opt_block_arg
2418		    {
2419		    /*%%%*/
2420			$$ = arg_blk_pass($1, $2);
2421		    /*%
2422			$$ = arg_add_optblock($1, $2);
2423		    %*/
2424		    }
2425		| assocs opt_block_arg
2426		    {
2427		    /*%%%*/
2428			$$ = NEW_LIST(NEW_HASH($1));
2429			$$ = arg_blk_pass($$, $2);
2430		    /*%
2431			$$ = arg_add_assocs(arg_new(), $1);
2432			$$ = arg_add_optblock($$, $2);
2433		    %*/
2434		    }
2435		| args ',' assocs opt_block_arg
2436		    {
2437		    /*%%%*/
2438			$$ = arg_append($1, NEW_HASH($3));
2439			$$ = arg_blk_pass($$, $4);
2440		    /*%
2441			$$ = arg_add_optblock(arg_add_assocs($1, $3), $4);
2442		    %*/
2443		    }
2444		| block_arg
2445		    /*%c%*/
2446		    /*%c
2447		    {
2448			$$ = arg_add_block(arg_new(), $1);
2449		    }
2450		    %*/
2451		;
2452
2453command_args	:  {
2454			$<val>$ = cmdarg_stack;
2455			CMDARG_PUSH(1);
2456		    }
2457		  call_args
2458		    {
2459			/* CMDARG_POP() */
2460			cmdarg_stack = $<val>1;
2461			$$ = $2;
2462		    }
2463		;
2464
2465block_arg	: tAMPER arg_value
2466		    {
2467		    /*%%%*/
2468			$$ = NEW_BLOCK_PASS($2);
2469		    /*%
2470			$$ = $2;
2471		    %*/
2472		    }
2473		;
2474
2475opt_block_arg	: ',' block_arg
2476		    {
2477			$$ = $2;
2478		    }
2479		| none
2480		    {
2481			$$ = 0;
2482		    }
2483		;
2484
2485args		: arg_value
2486		    {
2487		    /*%%%*/
2488			$$ = NEW_LIST($1);
2489		    /*%
2490			$$ = arg_add(arg_new(), $1);
2491		    %*/
2492		    }
2493		| tSTAR arg_value
2494		    {
2495		    /*%%%*/
2496			$$ = NEW_SPLAT($2);
2497		    /*%
2498			$$ = arg_add_star(arg_new(), $2);
2499		    %*/
2500		    }
2501		| args ',' arg_value
2502		    {
2503		    /*%%%*/
2504			NODE *n1;
2505			if ((n1 = splat_array($1)) != 0) {
2506			    $$ = list_append(n1, $3);
2507			}
2508			else {
2509			    $$ = arg_append($1, $3);
2510			}
2511		    /*%
2512			$$ = arg_add($1, $3);
2513		    %*/
2514		    }
2515		| args ',' tSTAR arg_value
2516		    {
2517		    /*%%%*/
2518			NODE *n1;
2519			if ((nd_type($4) == NODE_ARRAY) && (n1 = splat_array($1)) != 0) {
2520			    $$ = list_concat(n1, $4);
2521			}
2522			else {
2523			    $$ = arg_concat($1, $4);
2524			}
2525		    /*%
2526			$$ = arg_add_star($1, $4);
2527		    %*/
2528		    }
2529		;
2530
2531mrhs		: args ',' arg_value
2532		    {
2533		    /*%%%*/
2534			NODE *n1;
2535			if ((n1 = splat_array($1)) != 0) {
2536			    $$ = list_append(n1, $3);
2537			}
2538			else {
2539			    $$ = arg_append($1, $3);
2540			}
2541		    /*%
2542			$$ = mrhs_add(args2mrhs($1), $3);
2543		    %*/
2544		    }
2545		| args ',' tSTAR arg_value
2546		    {
2547		    /*%%%*/
2548			NODE *n1;
2549			if (nd_type($4) == NODE_ARRAY &&
2550			    (n1 = splat_array($1)) != 0) {
2551			    $$ = list_concat(n1, $4);
2552			}
2553			else {
2554			    $$ = arg_concat($1, $4);
2555			}
2556		    /*%
2557			$$ = mrhs_add_star(args2mrhs($1), $4);
2558		    %*/
2559		    }
2560		| tSTAR arg_value
2561		    {
2562		    /*%%%*/
2563			$$ = NEW_SPLAT($2);
2564		    /*%
2565			$$ = mrhs_add_star(mrhs_new(), $2);
2566		    %*/
2567		    }
2568		;
2569
2570primary		: literal
2571		| strings
2572		| xstring
2573		| regexp
2574		| words
2575		| qwords
2576		| symbols
2577		| qsymbols
2578		| var_ref
2579		| backref
2580		| tFID
2581		    {
2582		    /*%%%*/
2583			$$ = NEW_FCALL($1, 0);
2584		    /*%
2585			$$ = method_arg(dispatch1(fcall, $1), arg_new());
2586		    %*/
2587		    }
2588		| k_begin
2589		    {
2590			$<val>1 = cmdarg_stack;
2591			cmdarg_stack = 0;
2592		    /*%%%*/
2593			$<num>$ = ruby_sourceline;
2594		    /*%
2595		    %*/
2596		    }
2597		  bodystmt
2598		  k_end
2599		    {
2600			cmdarg_stack = $<val>1;
2601		    /*%%%*/
2602			if ($3 == NULL) {
2603			    $$ = NEW_NIL();
2604			}
2605			else {
2606			    if (nd_type($3) == NODE_RESCUE ||
2607				nd_type($3) == NODE_ENSURE)
2608				nd_set_line($3, $<num>2);
2609			    $$ = NEW_BEGIN($3);
2610			}
2611			nd_set_line($$, $<num>2);
2612		    /*%
2613			$$ = dispatch1(begin, $3);
2614		    %*/
2615		    }
2616		| tLPAREN_ARG {lex_state = EXPR_ENDARG;} rparen
2617		    {
2618		    /*%%%*/
2619			$$ = 0;
2620		    /*%
2621			$$ = dispatch1(paren, 0);
2622		    %*/
2623		    }
2624		| tLPAREN_ARG expr {lex_state = EXPR_ENDARG;} rparen
2625		    {
2626		    /*%%%*/
2627			$$ = $2;
2628		    /*%
2629			$$ = dispatch1(paren, $2);
2630		    %*/
2631		    }
2632		| tLPAREN compstmt ')'
2633		    {
2634		    /*%%%*/
2635			$$ = $2;
2636		    /*%
2637			$$ = dispatch1(paren, $2);
2638		    %*/
2639		    }
2640		| primary_value tCOLON2 tCONSTANT
2641		    {
2642		    /*%%%*/
2643			$$ = NEW_COLON2($1, $3);
2644		    /*%
2645			$$ = dispatch2(const_path_ref, $1, $3);
2646		    %*/
2647		    }
2648		| tCOLON3 tCONSTANT
2649		    {
2650		    /*%%%*/
2651			$$ = NEW_COLON3($2);
2652		    /*%
2653			$$ = dispatch1(top_const_ref, $2);
2654		    %*/
2655		    }
2656		| tLBRACK aref_args ']'
2657		    {
2658		    /*%%%*/
2659			if ($2 == 0) {
2660			    $$ = NEW_ZARRAY(); /* zero length array*/
2661			}
2662			else {
2663			    $$ = $2;
2664			}
2665		    /*%
2666			$$ = dispatch1(array, escape_Qundef($2));
2667		    %*/
2668		    }
2669		| tLBRACE assoc_list '}'
2670		    {
2671		    /*%%%*/
2672			$$ = NEW_HASH($2);
2673		    /*%
2674			$$ = dispatch1(hash, escape_Qundef($2));
2675		    %*/
2676		    }
2677		| keyword_return
2678		    {
2679		    /*%%%*/
2680			$$ = NEW_RETURN(0);
2681		    /*%
2682			$$ = dispatch0(return0);
2683		    %*/
2684		    }
2685		| keyword_yield '(' call_args rparen
2686		    {
2687		    /*%%%*/
2688			$$ = new_yield($3);
2689		    /*%
2690			$$ = dispatch1(yield, dispatch1(paren, $3));
2691		    %*/
2692		    }
2693		| keyword_yield '(' rparen
2694		    {
2695		    /*%%%*/
2696			$$ = NEW_YIELD(0);
2697		    /*%
2698			$$ = dispatch1(yield, dispatch1(paren, arg_new()));
2699		    %*/
2700		    }
2701		| keyword_yield
2702		    {
2703		    /*%%%*/
2704			$$ = NEW_YIELD(0);
2705		    /*%
2706			$$ = dispatch0(yield0);
2707		    %*/
2708		    }
2709		| keyword_defined opt_nl '(' {in_defined = 1;} expr rparen
2710		    {
2711		    /*%%%*/
2712			in_defined = 0;
2713			$$ = NEW_DEFINED($5);
2714		    /*%
2715			in_defined = 0;
2716			$$ = dispatch1(defined, $5);
2717		    %*/
2718		    }
2719		| keyword_not '(' expr rparen
2720		    {
2721		    /*%%%*/
2722			$$ = call_uni_op(cond($3), '!');
2723		    /*%
2724			$$ = dispatch2(unary, ripper_intern("not"), $3);
2725		    %*/
2726		    }
2727		| keyword_not '(' rparen
2728		    {
2729		    /*%%%*/
2730			$$ = call_uni_op(cond(NEW_NIL()), '!');
2731		    /*%
2732			$$ = dispatch2(unary, ripper_intern("not"), Qnil);
2733		    %*/
2734		    }
2735		| fcall brace_block
2736		    {
2737		    /*%%%*/
2738			$2->nd_iter = $1;
2739			$$ = $2;
2740		    /*%
2741			$$ = method_arg(dispatch1(fcall, $1), arg_new());
2742			$$ = method_add_block($$, $2);
2743		    %*/
2744		    }
2745		| method_call
2746		| method_call brace_block
2747		    {
2748		    /*%%%*/
2749			block_dup_check($1->nd_args, $2);
2750			$2->nd_iter = $1;
2751			$$ = $2;
2752		    /*%
2753			$$ = method_add_block($1, $2);
2754		    %*/
2755		    }
2756		| tLAMBDA lambda
2757		    {
2758			$$ = $2;
2759		    }
2760		| k_if expr_value then
2761		  compstmt
2762		  if_tail
2763		  k_end
2764		    {
2765		    /*%%%*/
2766			$$ = NEW_IF(cond($2), $4, $5);
2767			fixpos($$, $2);
2768		    /*%
2769			$$ = dispatch3(if, $2, $4, escape_Qundef($5));
2770		    %*/
2771		    }
2772		| k_unless expr_value then
2773		  compstmt
2774		  opt_else
2775		  k_end
2776		    {
2777		    /*%%%*/
2778			$$ = NEW_UNLESS(cond($2), $4, $5);
2779			fixpos($$, $2);
2780		    /*%
2781			$$ = dispatch3(unless, $2, $4, escape_Qundef($5));
2782		    %*/
2783		    }
2784		| k_while {COND_PUSH(1);} expr_value do {COND_POP();}
2785		  compstmt
2786		  k_end
2787		    {
2788		    /*%%%*/
2789			$$ = NEW_WHILE(cond($3), $6, 1);
2790			fixpos($$, $3);
2791		    /*%
2792			$$ = dispatch2(while, $3, $6);
2793		    %*/
2794		    }
2795		| k_until {COND_PUSH(1);} expr_value do {COND_POP();}
2796		  compstmt
2797		  k_end
2798		    {
2799		    /*%%%*/
2800			$$ = NEW_UNTIL(cond($3), $6, 1);
2801			fixpos($$, $3);
2802		    /*%
2803			$$ = dispatch2(until, $3, $6);
2804		    %*/
2805		    }
2806		| k_case expr_value opt_terms
2807		  case_body
2808		  k_end
2809		    {
2810		    /*%%%*/
2811			$$ = NEW_CASE($2, $4);
2812			fixpos($$, $2);
2813		    /*%
2814			$$ = dispatch2(case, $2, $4);
2815		    %*/
2816		    }
2817		| k_case opt_terms case_body k_end
2818		    {
2819		    /*%%%*/
2820			$$ = NEW_CASE(0, $3);
2821		    /*%
2822			$$ = dispatch2(case, Qnil, $3);
2823		    %*/
2824		    }
2825		| k_for for_var keyword_in
2826		  {COND_PUSH(1);}
2827		  expr_value do
2828		  {COND_POP();}
2829		  compstmt
2830		  k_end
2831		    {
2832		    /*%%%*/
2833			/*
2834			 *  for a, b, c in e
2835			 *  #=>
2836			 *  e.each{|*x| a, b, c = x
2837			 *
2838			 *  for a in e
2839			 *  #=>
2840			 *  e.each{|x| a, = x}
2841			 */
2842			ID id = internal_id();
2843			ID *tbl = ALLOC_N(ID, 2);
2844			NODE *m = NEW_ARGS_AUX(0, 0);
2845			NODE *args, *scope;
2846
2847			if (nd_type($2) == NODE_MASGN) {
2848			    /* if args.length == 1 && args[0].kind_of?(Array)
2849			     *   args = args[0]
2850			     * end
2851			     */
2852			    NODE *one = NEW_LIST(NEW_LIT(INT2FIX(1)));
2853			    NODE *zero = NEW_LIST(NEW_LIT(INT2FIX(0)));
2854			    m->nd_next = block_append(
2855				NEW_IF(
2856				    NEW_NODE(NODE_AND,
2857					     NEW_CALL(NEW_CALL(NEW_DVAR(id), idLength, 0),
2858						      idEq, one),
2859					     NEW_CALL(NEW_CALL(NEW_DVAR(id), idAREF, zero),
2860						      rb_intern("kind_of?"), NEW_LIST(NEW_LIT(rb_cArray))),
2861					     0),
2862				    NEW_DASGN_CURR(id,
2863						   NEW_CALL(NEW_DVAR(id), idAREF, zero)),
2864				    0),
2865				node_assign($2, NEW_DVAR(id)));
2866
2867			    args = new_args(m, 0, id, 0, new_args_tail(0, 0, 0));
2868			}
2869			else {
2870			    if (nd_type($2) == NODE_LASGN ||
2871				nd_type($2) == NODE_DASGN ||
2872				nd_type($2) == NODE_DASGN_CURR) {
2873				$2->nd_value = NEW_DVAR(id);
2874				m->nd_plen = 1;
2875				m->nd_next = $2;
2876				args = new_args(m, 0, 0, 0, new_args_tail(0, 0, 0));
2877			    }
2878			    else {
2879				m->nd_next = node_assign(NEW_MASGN(NEW_LIST($2), 0), NEW_DVAR(id));
2880				args = new_args(m, 0, id, 0, new_args_tail(0, 0, 0));
2881			    }
2882			}
2883			scope = NEW_NODE(NODE_SCOPE, tbl, $8, args);
2884			tbl[0] = 1; tbl[1] = id;
2885			$$ = NEW_FOR(0, $5, scope);
2886			fixpos($$, $2);
2887		    /*%
2888			$$ = dispatch3(for, $2, $5, $8);
2889		    %*/
2890		    }
2891		| k_class cpath superclass
2892		    {
2893			if (in_def || in_single)
2894			    yyerror("class definition in method body");
2895			local_push(0);
2896		    /*%%%*/
2897			$<num>$ = ruby_sourceline;
2898		    /*%
2899		    %*/
2900		    }
2901		  bodystmt
2902		  k_end
2903		    {
2904		    /*%%%*/
2905			$$ = NEW_CLASS($2, $5, $3);
2906			nd_set_line($$, $<num>4);
2907		    /*%
2908			$$ = dispatch3(class, $2, $3, $5);
2909		    %*/
2910			local_pop();
2911		    }
2912		| k_class tLSHFT expr
2913		    {
2914			$<num>$ = in_def;
2915			in_def = 0;
2916		    }
2917		  term
2918		    {
2919			$<num>$ = in_single;
2920			in_single = 0;
2921			local_push(0);
2922		    }
2923		  bodystmt
2924		  k_end
2925		    {
2926		    /*%%%*/
2927			$$ = NEW_SCLASS($3, $7);
2928			fixpos($$, $3);
2929		    /*%
2930			$$ = dispatch2(sclass, $3, $7);
2931		    %*/
2932			local_pop();
2933			in_def = $<num>4;
2934			in_single = $<num>6;
2935		    }
2936		| k_module cpath
2937		    {
2938			if (in_def || in_single)
2939			    yyerror("module definition in method body");
2940			local_push(0);
2941		    /*%%%*/
2942			$<num>$ = ruby_sourceline;
2943		    /*%
2944		    %*/
2945		    }
2946		  bodystmt
2947		  k_end
2948		    {
2949		    /*%%%*/
2950			$$ = NEW_MODULE($2, $4);
2951			nd_set_line($$, $<num>3);
2952		    /*%
2953			$$ = dispatch2(module, $2, $4);
2954		    %*/
2955			local_pop();
2956		    }
2957		| k_def fname
2958		    {
2959			$<id>$ = cur_mid;
2960			cur_mid = $2;
2961			in_def++;
2962			local_push(0);
2963		    }
2964		  f_arglist
2965		  bodystmt
2966		  k_end
2967		    {
2968		    /*%%%*/
2969			NODE *body = remove_begin($5);
2970			reduce_nodes(&body);
2971			$$ = NEW_DEFN($2, $4, body, NOEX_PRIVATE);
2972			nd_set_line($$, $<num>1);
2973		    /*%
2974			$$ = dispatch3(def, $2, $4, $5);
2975		    %*/
2976			local_pop();
2977			in_def--;
2978			cur_mid = $<id>3;
2979		    }
2980		| k_def singleton dot_or_colon {lex_state = EXPR_FNAME;} fname
2981		    {
2982			in_single++;
2983			lex_state = EXPR_ENDFN; /* force for args */
2984			local_push(0);
2985		    }
2986		  f_arglist
2987		  bodystmt
2988		  k_end
2989		    {
2990		    /*%%%*/
2991			NODE *body = remove_begin($8);
2992			reduce_nodes(&body);
2993			$$ = NEW_DEFS($2, $5, $7, body);
2994			nd_set_line($$, $<num>1);
2995		    /*%
2996			$$ = dispatch5(defs, $2, $3, $5, $7, $8);
2997		    %*/
2998			local_pop();
2999			in_single--;
3000		    }
3001		| keyword_break
3002		    {
3003		    /*%%%*/
3004			$$ = NEW_BREAK(0);
3005		    /*%
3006			$$ = dispatch1(break, arg_new());
3007		    %*/
3008		    }
3009		| keyword_next
3010		    {
3011		    /*%%%*/
3012			$$ = NEW_NEXT(0);
3013		    /*%
3014			$$ = dispatch1(next, arg_new());
3015		    %*/
3016		    }
3017		| keyword_redo
3018		    {
3019		    /*%%%*/
3020			$$ = NEW_REDO();
3021		    /*%
3022			$$ = dispatch0(redo);
3023		    %*/
3024		    }
3025		| keyword_retry
3026		    {
3027		    /*%%%*/
3028			$$ = NEW_RETRY();
3029		    /*%
3030			$$ = dispatch0(retry);
3031		    %*/
3032		    }
3033		;
3034
3035primary_value	: primary
3036		    {
3037		    /*%%%*/
3038			value_expr($1);
3039			$$ = $1;
3040		        if (!$$) $$ = NEW_NIL();
3041		    /*%
3042			$$ = $1;
3043		    %*/
3044		    }
3045		;
3046
3047k_begin		: keyword_begin
3048		    {
3049			token_info_push("begin");
3050		    }
3051		;
3052
3053k_if		: keyword_if
3054		    {
3055			token_info_push("if");
3056		    }
3057		;
3058
3059k_unless	: keyword_unless
3060		    {
3061			token_info_push("unless");
3062		    }
3063		;
3064
3065k_while		: keyword_while
3066		    {
3067			token_info_push("while");
3068		    }
3069		;
3070
3071k_until		: keyword_until
3072		    {
3073			token_info_push("until");
3074		    }
3075		;
3076
3077k_case		: keyword_case
3078		    {
3079			token_info_push("case");
3080		    }
3081		;
3082
3083k_for		: keyword_for
3084		    {
3085			token_info_push("for");
3086		    }
3087		;
3088
3089k_class		: keyword_class
3090		    {
3091			token_info_push("class");
3092		    }
3093		;
3094
3095k_module	: keyword_module
3096		    {
3097			token_info_push("module");
3098		    }
3099		;
3100
3101k_def		: keyword_def
3102		    {
3103			token_info_push("def");
3104		    /*%%%*/
3105			$<num>$ = ruby_sourceline;
3106		    /*%
3107		    %*/
3108		    }
3109		;
3110
3111k_end		: keyword_end
3112		    {
3113			token_info_pop("end");
3114		    }
3115		;
3116
3117then		: term
3118		    /*%c%*/
3119		    /*%c
3120		    { $$ = Qnil; }
3121		    %*/
3122		| keyword_then
3123		| term keyword_then
3124		    /*%c%*/
3125		    /*%c
3126		    { $$ = $2; }
3127		    %*/
3128		;
3129
3130do		: term
3131		    /*%c%*/
3132		    /*%c
3133		    { $$ = Qnil; }
3134		    %*/
3135		| keyword_do_cond
3136		;
3137
3138if_tail		: opt_else
3139		| keyword_elsif expr_value then
3140		  compstmt
3141		  if_tail
3142		    {
3143		    /*%%%*/
3144			$$ = NEW_IF(cond($2), $4, $5);
3145			fixpos($$, $2);
3146		    /*%
3147			$$ = dispatch3(elsif, $2, $4, escape_Qundef($5));
3148		    %*/
3149		    }
3150		;
3151
3152opt_else	: none
3153		| keyword_else compstmt
3154		    {
3155		    /*%%%*/
3156			$$ = $2;
3157		    /*%
3158			$$ = dispatch1(else, $2);
3159		    %*/
3160		    }
3161		;
3162
3163for_var		: lhs
3164		| mlhs
3165		;
3166
3167f_marg		: f_norm_arg
3168		    {
3169			$$ = assignable($1, 0);
3170		    /*%%%*/
3171		    /*%
3172			$$ = dispatch1(mlhs_paren, $$);
3173		    %*/
3174		    }
3175		| tLPAREN f_margs rparen
3176		    {
3177		    /*%%%*/
3178			$$ = $2;
3179		    /*%
3180			$$ = dispatch1(mlhs_paren, $2);
3181		    %*/
3182		    }
3183		;
3184
3185f_marg_list	: f_marg
3186		    {
3187		    /*%%%*/
3188			$$ = NEW_LIST($1);
3189		    /*%
3190			$$ = mlhs_add(mlhs_new(), $1);
3191		    %*/
3192		    }
3193		| f_marg_list ',' f_marg
3194		    {
3195		    /*%%%*/
3196			$$ = list_append($1, $3);
3197		    /*%
3198			$$ = mlhs_add($1, $3);
3199		    %*/
3200		    }
3201		;
3202
3203f_margs		: f_marg_list
3204		    {
3205		    /*%%%*/
3206			$$ = NEW_MASGN($1, 0);
3207		    /*%
3208			$$ = $1;
3209		    %*/
3210		    }
3211		| f_marg_list ',' tSTAR f_norm_arg
3212		    {
3213			$$ = assignable($4, 0);
3214		    /*%%%*/
3215			$$ = NEW_MASGN($1, $$);
3216		    /*%
3217			$$ = mlhs_add_star($1, $$);
3218		    %*/
3219		    }
3220		| f_marg_list ',' tSTAR f_norm_arg ',' f_marg_list
3221		    {
3222			$$ = assignable($4, 0);
3223		    /*%%%*/
3224			$$ = NEW_MASGN($1, NEW_POSTARG($$, $6));
3225		    /*%
3226			$$ = mlhs_add_star($1, $$);
3227		    %*/
3228		    }
3229		| f_marg_list ',' tSTAR
3230		    {
3231		    /*%%%*/
3232			$$ = NEW_MASGN($1, -1);
3233		    /*%
3234			$$ = mlhs_add_star($1, Qnil);
3235		    %*/
3236		    }
3237		| f_marg_list ',' tSTAR ',' f_marg_list
3238		    {
3239		    /*%%%*/
3240			$$ = NEW_MASGN($1, NEW_POSTARG(-1, $5));
3241		    /*%
3242			$$ = mlhs_add_star($1, $5);
3243		    %*/
3244		    }
3245		| tSTAR f_norm_arg
3246		    {
3247			$$ = assignable($2, 0);
3248		    /*%%%*/
3249			$$ = NEW_MASGN(0, $$);
3250		    /*%
3251			$$ = mlhs_add_star(mlhs_new(), $$);
3252		    %*/
3253		    }
3254		| tSTAR f_norm_arg ',' f_marg_list
3255		    {
3256			$$ = assignable($2, 0);
3257		    /*%%%*/
3258			$$ = NEW_MASGN(0, NEW_POSTARG($$, $4));
3259		    /*%
3260		      #if 0
3261		      TODO: Check me
3262		      #endif
3263			$$ = mlhs_add_star($$, $4);
3264		    %*/
3265		    }
3266		| tSTAR
3267		    {
3268		    /*%%%*/
3269			$$ = NEW_MASGN(0, -1);
3270		    /*%
3271			$$ = mlhs_add_star(mlhs_new(), Qnil);
3272		    %*/
3273		    }
3274		| tSTAR ',' f_marg_list
3275		    {
3276		    /*%%%*/
3277			$$ = NEW_MASGN(0, NEW_POSTARG(-1, $3));
3278		    /*%
3279			$$ = mlhs_add_star(mlhs_new(), Qnil);
3280		    %*/
3281		    }
3282		;
3283
3284
3285block_args_tail	: f_block_kwarg ',' f_kwrest opt_f_block_arg
3286		    {
3287			$$ = new_args_tail($1, $3, $4);
3288		    }
3289		| f_block_kwarg opt_f_block_arg
3290		    {
3291			$$ = new_args_tail($1, Qnone, $2);
3292		    }
3293		| f_kwrest opt_f_block_arg
3294		    {
3295			$$ = new_args_tail(Qnone, $1, $2);
3296		    }
3297		| f_block_arg
3298		    {
3299			$$ = new_args_tail(Qnone, Qnone, $1);
3300		    }
3301		;
3302
3303opt_block_args_tail : ',' block_args_tail
3304		    {
3305			$$ = $2;
3306		    }
3307		| /* none */
3308		    {
3309			$$ = new_args_tail(Qnone, Qnone, Qnone);
3310		    }
3311		;
3312
3313block_param	: f_arg ',' f_block_optarg ',' f_rest_arg opt_block_args_tail
3314		    {
3315			$$ = new_args($1, $3, $5, Qnone, $6);
3316		    }
3317		| f_arg ',' f_block_optarg ',' f_rest_arg ',' f_arg opt_block_args_tail
3318		    {
3319			$$ = new_args($1, $3, $5, $7, $8);
3320		    }
3321		| f_arg ',' f_block_optarg opt_block_args_tail
3322		    {
3323			$$ = new_args($1, $3, Qnone, Qnone, $4);
3324		    }
3325		| f_arg ',' f_block_optarg ',' f_arg opt_block_args_tail
3326		    {
3327			$$ = new_args($1, $3, Qnone, $5, $6);
3328		    }
3329                | f_arg ',' f_rest_arg opt_block_args_tail
3330		    {
3331			$$ = new_args($1, Qnone, $3, Qnone, $4);
3332		    }
3333		| f_arg ','
3334		    {
3335			$$ = new_args($1, Qnone, 1, Qnone, new_args_tail(Qnone, Qnone, Qnone));
3336		    /*%%%*/
3337		    /*%
3338                        dispatch1(excessed_comma, $$);
3339		    %*/
3340		    }
3341		| f_arg ',' f_rest_arg ',' f_arg opt_block_args_tail
3342		    {
3343			$$ = new_args($1, Qnone, $3, $5, $6);
3344		    }
3345		| f_arg opt_block_args_tail
3346		    {
3347			$$ = new_args($1, Qnone, Qnone, Qnone, $2);
3348		    }
3349		| f_block_optarg ',' f_rest_arg opt_block_args_tail
3350		    {
3351			$$ = new_args(Qnone, $1, $3, Qnone, $4);
3352		    }
3353		| f_block_optarg ',' f_rest_arg ',' f_arg opt_block_args_tail
3354		    {
3355			$$ = new_args(Qnone, $1, $3, $5, $6);
3356		    }
3357		| f_block_optarg opt_block_args_tail
3358		    {
3359			$$ = new_args(Qnone, $1, Qnone, Qnone, $2);
3360		    }
3361		| f_block_optarg ',' f_arg opt_block_args_tail
3362		    {
3363			$$ = new_args(Qnone, $1, Qnone, $3, $4);
3364		    }
3365		| f_rest_arg opt_block_args_tail
3366		    {
3367			$$ = new_args(Qnone, Qnone, $1, Qnone, $2);
3368		    }
3369		| f_rest_arg ',' f_arg opt_block_args_tail
3370		    {
3371			$$ = new_args(Qnone, Qnone, $1, $3, $4);
3372		    }
3373		| block_args_tail
3374		    {
3375			$$ = new_args(Qnone, Qnone, Qnone, Qnone, $1);
3376		    }
3377		;
3378
3379opt_block_param	: none
3380		| block_param_def
3381		    {
3382			command_start = TRUE;
3383		    }
3384		;
3385
3386block_param_def	: '|' opt_bv_decl '|'
3387		    {
3388		    /*%%%*/
3389			$$ = 0;
3390		    /*%
3391			$$ = blockvar_new(params_new(Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil),
3392                                          escape_Qundef($2));
3393		    %*/
3394		    }
3395		| tOROP
3396		    {
3397		    /*%%%*/
3398			$$ = 0;
3399		    /*%
3400			$$ = blockvar_new(params_new(Qnil,Qnil,Qnil,Qnil,Qnil,Qnil,Qnil),
3401                                          Qnil);
3402		    %*/
3403		    }
3404		| '|' block_param opt_bv_decl '|'
3405		    {
3406		    /*%%%*/
3407			$$ = $2;
3408		    /*%
3409			$$ = blockvar_new(escape_Qundef($2), escape_Qundef($3));
3410		    %*/
3411		    }
3412		;
3413
3414
3415opt_bv_decl	: opt_nl
3416		    {
3417		      $$ = 0;
3418		    }
3419		| opt_nl ';' bv_decls opt_nl
3420		    {
3421		    /*%%%*/
3422			$$ = 0;
3423		    /*%
3424			$$ = $3;
3425		    %*/
3426		    }
3427		;
3428
3429bv_decls	: bvar
3430		    /*%c%*/
3431		    /*%c
3432		    {
3433			$$ = rb_ary_new3(1, $1);
3434		    }
3435		    %*/
3436		| bv_decls ',' bvar
3437		    /*%c%*/
3438		    /*%c
3439		    {
3440			rb_ary_push($1, $3);
3441		    }
3442		    %*/
3443		;
3444
3445bvar		: tIDENTIFIER
3446		    {
3447			new_bv(get_id($1));
3448		    /*%%%*/
3449		    /*%
3450			$$ = get_value($1);
3451		    %*/
3452		    }
3453		| f_bad_arg
3454		    {
3455			$$ = 0;
3456		    }
3457		;
3458
3459lambda		:   {
3460			$<vars>$ = dyna_push();
3461		    }
3462		    {
3463			$<num>$ = lpar_beg;
3464			lpar_beg = ++paren_nest;
3465		    }
3466		  f_larglist
3467		    {
3468			$<num>$ = ruby_sourceline;
3469		    }
3470		  lambda_body
3471		    {
3472			lpar_beg = $<num>2;
3473		    /*%%%*/
3474			$$ = NEW_LAMBDA($3, $5);
3475			nd_set_line($$, $<num>4);
3476		    /*%
3477			$$ = dispatch2(lambda, $3, $5);
3478		    %*/
3479			dyna_pop($<vars>1);
3480		    }
3481		;
3482
3483f_larglist	: '(' f_args opt_bv_decl ')'
3484		    {
3485		    /*%%%*/
3486			$$ = $2;
3487		    /*%
3488			$$ = dispatch1(paren, $2);
3489		    %*/
3490		    }
3491		| f_args
3492		    {
3493		    /*%%%*/
3494			$$ = $1;
3495		    /*%
3496			$$ = $1;
3497		    %*/
3498		    }
3499		;
3500
3501lambda_body	: tLAMBEG compstmt '}'
3502		    {
3503			$$ = $2;
3504		    }
3505		| keyword_do_LAMBDA compstmt keyword_end
3506		    {
3507			$$ = $2;
3508		    }
3509		;
3510
3511do_block	: keyword_do_block
3512		    {
3513			$<vars>1 = dyna_push();
3514		    /*%%%*/
3515			$<num>$ = ruby_sourceline;
3516		    /*% %*/
3517		    }
3518		  opt_block_param
3519		  compstmt
3520		  keyword_end
3521		    {
3522		    /*%%%*/
3523			$$ = NEW_ITER($3,$4);
3524			nd_set_line($$, $<num>2);
3525		    /*%
3526			$$ = dispatch2(do_block, escape_Qundef($3), $4);
3527		    %*/
3528			dyna_pop($<vars>1);
3529		    }
3530		;
3531
3532block_call	: command do_block
3533		    {
3534		    /*%%%*/
3535			if (nd_type($1) == NODE_YIELD) {
3536			    compile_error(PARSER_ARG "block given to yield");
3537			}
3538			else {
3539			    block_dup_check($1->nd_args, $2);
3540			}
3541			$2->nd_iter = $1;
3542			$$ = $2;
3543			fixpos($$, $1);
3544		    /*%
3545			$$ = method_add_block($1, $2);
3546		    %*/
3547		    }
3548		| block_call dot_or_colon operation2 opt_paren_args
3549		    {
3550		    /*%%%*/
3551			$$ = NEW_CALL($1, $3, $4);
3552		    /*%
3553			$$ = dispatch3(call, $1, $2, $3);
3554			$$ = method_optarg($$, $4);
3555		    %*/
3556		    }
3557		| block_call dot_or_colon operation2 opt_paren_args brace_block
3558		    {
3559		    /*%%%*/
3560			block_dup_check($4, $5);
3561			$5->nd_iter = NEW_CALL($1, $3, $4);
3562			$$ = $5;
3563			fixpos($$, $1);
3564		    /*%
3565			$$ = dispatch4(command_call, $1, $2, $3, $4);
3566			$$ = method_add_block($$, $5);
3567		    %*/
3568		    }
3569		| block_call dot_or_colon operation2 command_args do_block
3570		    {
3571		    /*%%%*/
3572			block_dup_check($4, $5);
3573			$5->nd_iter = NEW_CALL($1, $3, $4);
3574			$$ = $5;
3575			fixpos($$, $1);
3576		    /*%
3577			$$ = dispatch4(command_call, $1, $2, $3, $4);
3578			$$ = method_add_block($$, $5);
3579		    %*/
3580		    }
3581		;
3582
3583method_call	: fcall paren_args
3584		    {
3585		    /*%%%*/
3586			$$ = $1;
3587			$$->nd_args = $2;
3588		    /*%
3589			$$ = method_arg(dispatch1(fcall, $1), $2);
3590		    %*/
3591		    }
3592		| primary_value '.' operation2
3593		    {
3594		    /*%%%*/
3595			$<num>$ = ruby_sourceline;
3596		    /*% %*/
3597		    }
3598		  opt_paren_args
3599		    {
3600		    /*%%%*/
3601			$$ = NEW_CALL($1, $3, $5);
3602			nd_set_line($$, $<num>4);
3603		    /*%
3604			$$ = dispatch3(call, $1, ripper_id2sym('.'), $3);
3605			$$ = method_optarg($$, $5);
3606		    %*/
3607		    }
3608		| primary_value tCOLON2 operation2
3609		    {
3610		    /*%%%*/
3611			$<num>$ = ruby_sourceline;
3612		    /*% %*/
3613		    }
3614		  paren_args
3615		    {
3616		    /*%%%*/
3617			$$ = NEW_CALL($1, $3, $5);
3618			nd_set_line($$, $<num>4);
3619		    /*%
3620			$$ = dispatch3(call, $1, ripper_id2sym('.'), $3);
3621			$$ = method_optarg($$, $5);
3622		    %*/
3623		    }
3624		| primary_value tCOLON2 operation3
3625		    {
3626		    /*%%%*/
3627			$$ = NEW_CALL($1, $3, 0);
3628		    /*%
3629			$$ = dispatch3(call, $1, ripper_intern("::"), $3);
3630		    %*/
3631		    }
3632		| primary_value '.'
3633		    {
3634		    /*%%%*/
3635			$<num>$ = ruby_sourceline;
3636		    /*% %*/
3637		    }
3638		  paren_args
3639		    {
3640		    /*%%%*/
3641			$$ = NEW_CALL($1, rb_intern("call"), $4);
3642			nd_set_line($$, $<num>3);
3643		    /*%
3644			$$ = dispatch3(call, $1, ripper_id2sym('.'),
3645				       ripper_intern("call"));
3646			$$ = method_optarg($$, $4);
3647		    %*/
3648		    }
3649		| primary_value tCOLON2
3650		    {
3651		    /*%%%*/
3652			$<num>$ = ruby_sourceline;
3653		    /*% %*/
3654		    }
3655		  paren_args
3656		    {
3657		    /*%%%*/
3658			$$ = NEW_CALL($1, rb_intern("call"), $4);
3659			nd_set_line($$, $<num>3);
3660		    /*%
3661			$$ = dispatch3(call, $1, ripper_intern("::"),
3662				       ripper_intern("call"));
3663			$$ = method_optarg($$, $4);
3664		    %*/
3665		    }
3666		| keyword_super paren_args
3667		    {
3668		    /*%%%*/
3669			$$ = NEW_SUPER($2);
3670		    /*%
3671			$$ = dispatch1(super, $2);
3672		    %*/
3673		    }
3674		| keyword_super
3675		    {
3676		    /*%%%*/
3677			$$ = NEW_ZSUPER();
3678		    /*%
3679			$$ = dispatch0(zsuper);
3680		    %*/
3681		    }
3682		| primary_value '[' opt_call_args rbracket
3683		    {
3684		    /*%%%*/
3685			if ($1 && nd_type($1) == NODE_SELF)
3686			    $$ = NEW_FCALL(tAREF, $3);
3687			else
3688			    $$ = NEW_CALL($1, tAREF, $3);
3689			fixpos($$, $1);
3690		    /*%
3691			$$ = dispatch2(aref, $1, escape_Qundef($3));
3692		    %*/
3693		    }
3694		;
3695
3696brace_block	: '{'
3697		    {
3698			$<vars>1 = dyna_push();
3699		    /*%%%*/
3700			$<num>$ = ruby_sourceline;
3701		    /*%
3702                    %*/
3703		    }
3704		  opt_block_param
3705		  compstmt '}'
3706		    {
3707		    /*%%%*/
3708			$$ = NEW_ITER($3,$4);
3709			nd_set_line($$, $<num>2);
3710		    /*%
3711			$$ = dispatch2(brace_block, escape_Qundef($3), $4);
3712		    %*/
3713			dyna_pop($<vars>1);
3714		    }
3715		| keyword_do
3716		    {
3717			$<vars>1 = dyna_push();
3718		    /*%%%*/
3719			$<num>$ = ruby_sourceline;
3720		    /*%
3721                    %*/
3722		    }
3723		  opt_block_param
3724		  compstmt keyword_end
3725		    {
3726		    /*%%%*/
3727			$$ = NEW_ITER($3,$4);
3728			nd_set_line($$, $<num>2);
3729		    /*%
3730			$$ = dispatch2(do_block, escape_Qundef($3), $4);
3731		    %*/
3732			dyna_pop($<vars>1);
3733		    }
3734		;
3735
3736case_body	: keyword_when args then
3737		  compstmt
3738		  cases
3739		    {
3740		    /*%%%*/
3741			$$ = NEW_WHEN($2, $4, $5);
3742		    /*%
3743			$$ = dispatch3(when, $2, $4, escape_Qundef($5));
3744		    %*/
3745		    }
3746		;
3747
3748cases		: opt_else
3749		| case_body
3750		;
3751
3752opt_rescue	: keyword_rescue exc_list exc_var then
3753		  compstmt
3754		  opt_rescue
3755		    {
3756		    /*%%%*/
3757			if ($3) {
3758			    $3 = node_assign($3, NEW_ERRINFO());
3759			    $5 = block_append($3, $5);
3760			}
3761			$$ = NEW_RESBODY($2, $5, $6);
3762			fixpos($$, $2?$2:$5);
3763		    /*%
3764			$$ = dispatch4(rescue,
3765				       escape_Qundef($2),
3766				       escape_Qundef($3),
3767				       escape_Qundef($5),
3768				       escape_Qundef($6));
3769		    %*/
3770		    }
3771		| none
3772		;
3773
3774exc_list	: arg_value
3775		    {
3776		    /*%%%*/
3777			$$ = NEW_LIST($1);
3778		    /*%
3779			$$ = rb_ary_new3(1, $1);
3780		    %*/
3781		    }
3782		| mrhs
3783		    {
3784		    /*%%%*/
3785			if (!($$ = splat_array($1))) $$ = $1;
3786		    /*%
3787			$$ = $1;
3788		    %*/
3789		    }
3790		| none
3791		;
3792
3793exc_var		: tASSOC lhs
3794		    {
3795			$$ = $2;
3796		    }
3797		| none
3798		;
3799
3800opt_ensure	: keyword_ensure compstmt
3801		    {
3802		    /*%%%*/
3803			$$ = $2;
3804		    /*%
3805			$$ = dispatch1(ensure, $2);
3806		    %*/
3807		    }
3808		| none
3809		;
3810
3811literal		: numeric
3812		| symbol
3813		    {
3814		    /*%%%*/
3815			$$ = NEW_LIT(ID2SYM($1));
3816		    /*%
3817			$$ = dispatch1(symbol_literal, $1);
3818		    %*/
3819		    }
3820		| dsym
3821		;
3822
3823strings		: string
3824		    {
3825		    /*%%%*/
3826			NODE *node = $1;
3827			if (!node) {
3828			    node = NEW_STR(STR_NEW0());
3829			}
3830			else {
3831			    node = evstr2dstr(node);
3832			}
3833			$$ = node;
3834		    /*%
3835			$$ = $1;
3836		    %*/
3837		    }
3838		;
3839
3840string		: tCHAR
3841		| string1
3842		| string string1
3843		    {
3844		    /*%%%*/
3845			$$ = literal_concat($1, $2);
3846		    /*%
3847			$$ = dispatch2(string_concat, $1, $2);
3848		    %*/
3849		    }
3850		;
3851
3852string1		: tSTRING_BEG string_contents tSTRING_END
3853		    {
3854		    /*%%%*/
3855			$$ = $2;
3856		    /*%
3857			$$ = dispatch1(string_literal, $2);
3858		    %*/
3859		    }
3860		;
3861
3862xstring		: tXSTRING_BEG xstring_contents tSTRING_END
3863		    {
3864		    /*%%%*/
3865			NODE *node = $2;
3866			if (!node) {
3867			    node = NEW_XSTR(STR_NEW0());
3868			}
3869			else {
3870			    switch (nd_type(node)) {
3871			      case NODE_STR:
3872				nd_set_type(node, NODE_XSTR);
3873				break;
3874			      case NODE_DSTR:
3875				nd_set_type(node, NODE_DXSTR);
3876				break;
3877			      default:
3878				node = NEW_NODE(NODE_DXSTR, Qnil, 1, NEW_LIST(node));
3879				break;
3880			    }
3881			}
3882			$$ = node;
3883		    /*%
3884			$$ = dispatch1(xstring_literal, $2);
3885		    %*/
3886		    }
3887		;
3888
3889regexp		: tREGEXP_BEG regexp_contents tREGEXP_END
3890		    {
3891		    /*%%%*/
3892			int options = $3;
3893			NODE *node = $2;
3894			NODE *list, *prev;
3895			if (!node) {
3896			    node = NEW_LIT(reg_compile(STR_NEW0(), options));
3897			}
3898			else switch (nd_type(node)) {
3899			  case NODE_STR:
3900			    {
3901				VALUE src = node->nd_lit;
3902				nd_set_type(node, NODE_LIT);
3903				node->nd_lit = reg_compile(src, options);
3904			    }
3905			    break;
3906			  default:
3907			    node = NEW_NODE(NODE_DSTR, STR_NEW0(), 1, NEW_LIST(node));
3908			  case NODE_DSTR:
3909			    if (options & RE_OPTION_ONCE) {
3910				nd_set_type(node, NODE_DREGX_ONCE);
3911			    }
3912			    else {
3913				nd_set_type(node, NODE_DREGX);
3914			    }
3915			    node->nd_cflag = options & RE_OPTION_MASK;
3916			    if (!NIL_P(node->nd_lit)) reg_fragment_check(node->nd_lit, options);
3917			    for (list = (prev = node)->nd_next; list; list = list->nd_next) {
3918				if (nd_type(list->nd_head) == NODE_STR) {
3919				    VALUE tail = list->nd_head->nd_lit;
3920				    if (reg_fragment_check(tail, options) && prev && !NIL_P(prev->nd_lit)) {
3921					VALUE lit = prev == node ? prev->nd_lit : prev->nd_head->nd_lit;
3922					if (!literal_concat0(parser, lit, tail)) {
3923					    node = 0;
3924					    break;
3925					}
3926					rb_str_resize(tail, 0);
3927					prev->nd_next = list->nd_next;
3928					rb_gc_force_recycle((VALUE)list->nd_head);
3929					rb_gc_force_recycle((VALUE)list);
3930					list = prev;
3931				    }
3932				    else {
3933					prev = list;
3934				    }
3935                                }
3936				else {
3937				    prev = 0;
3938				}
3939                            }
3940			    if (!node->nd_next) {
3941				VALUE src = node->nd_lit;
3942				nd_set_type(node, NODE_LIT);
3943				node->nd_lit = reg_compile(src, options);
3944			    }
3945			    break;
3946			}
3947			$$ = node;
3948		    /*%
3949			$$ = dispatch2(regexp_literal, $2, $3);
3950		    %*/
3951		    }
3952		;
3953
3954words		: tWORDS_BEG ' ' tSTRING_END
3955		    {
3956		    /*%%%*/
3957			$$ = NEW_ZARRAY();
3958		    /*%
3959			$$ = dispatch0(words_new);
3960			$$ = dispatch1(array, $$);
3961		    %*/
3962		    }
3963		| tWORDS_BEG word_list tSTRING_END
3964		    {
3965		    /*%%%*/
3966			$$ = $2;
3967		    /*%
3968			$$ = dispatch1(array, $2);
3969		    %*/
3970		    }
3971		;
3972
3973word_list	: /* none */
3974		    {
3975		    /*%%%*/
3976			$$ = 0;
3977		    /*%
3978			$$ = dispatch0(words_new);
3979		    %*/
3980		    }
3981		| word_list word ' '
3982		    {
3983		    /*%%%*/
3984			$$ = list_append($1, evstr2dstr($2));
3985		    /*%
3986			$$ = dispatch2(words_add, $1, $2);
3987		    %*/
3988		    }
3989		;
3990
3991word		: string_content
3992		    /*%c%*/
3993		    /*%c
3994		    {
3995			$$ = dispatch0(word_new);
3996			$$ = dispatch2(word_add, $$, $1);
3997		    }
3998		    %*/
3999		| word string_content
4000		    {
4001		    /*%%%*/
4002			$$ = literal_concat($1, $2);
4003		    /*%
4004			$$ = dispatch2(word_add, $1, $2);
4005		    %*/
4006		    }
4007		;
4008
4009symbols	        : tSYMBOLS_BEG ' ' tSTRING_END
4010		    {
4011		    /*%%%*/
4012			$$ = NEW_ZARRAY();
4013		    /*%
4014			$$ = dispatch0(symbols_new);
4015			$$ = dispatch1(array, $$);
4016		    %*/
4017		    }
4018		| tSYMBOLS_BEG symbol_list tSTRING_END
4019		    {
4020		    /*%%%*/
4021			$$ = $2;
4022		    /*%
4023			$$ = dispatch1(array, $2);
4024		    %*/
4025		    }
4026		;
4027
4028symbol_list	: /* none */
4029		    {
4030		    /*%%%*/
4031			$$ = 0;
4032		    /*%
4033			$$ = dispatch0(symbols_new);
4034		    %*/
4035		    }
4036		| symbol_list word ' '
4037		    {
4038		    /*%%%*/
4039			$2 = evstr2dstr($2);
4040			nd_set_type($2, NODE_DSYM);
4041			$$ = list_append($1, $2);
4042		    /*%
4043			$$ = dispatch2(symbols_add, $1, $2);
4044		    %*/
4045		    }
4046		;
4047
4048qwords		: tQWORDS_BEG ' ' tSTRING_END
4049		    {
4050		    /*%%%*/
4051			$$ = NEW_ZARRAY();
4052		    /*%
4053			$$ = dispatch0(qwords_new);
4054			$$ = dispatch1(array, $$);
4055		    %*/
4056		    }
4057		| tQWORDS_BEG qword_list tSTRING_END
4058		    {
4059		    /*%%%*/
4060			$$ = $2;
4061		    /*%
4062			$$ = dispatch1(array, $2);
4063		    %*/
4064		    }
4065		;
4066
4067qsymbols	: tQSYMBOLS_BEG ' ' tSTRING_END
4068		    {
4069		    /*%%%*/
4070			$$ = NEW_ZARRAY();
4071		    /*%
4072			$$ = dispatch0(qsymbols_new);
4073			$$ = dispatch1(array, $$);
4074		    %*/
4075		    }
4076		| tQSYMBOLS_BEG qsym_list tSTRING_END
4077		    {
4078		    /*%%%*/
4079			$$ = $2;
4080		    /*%
4081			$$ = dispatch1(array, $2);
4082		    %*/
4083		    }
4084		;
4085
4086qword_list	: /* none */
4087		    {
4088		    /*%%%*/
4089			$$ = 0;
4090		    /*%
4091			$$ = dispatch0(qwords_new);
4092		    %*/
4093		    }
4094		| qword_list tSTRING_CONTENT ' '
4095		    {
4096		    /*%%%*/
4097			$$ = list_append($1, $2);
4098		    /*%
4099			$$ = dispatch2(qwords_add, $1, $2);
4100		    %*/
4101		    }
4102		;
4103
4104qsym_list	: /* none */
4105		    {
4106		    /*%%%*/
4107			$$ = 0;
4108		    /*%
4109			$$ = dispatch0(qsymbols_new);
4110		    %*/
4111		    }
4112		| qsym_list tSTRING_CONTENT ' '
4113		    {
4114		    /*%%%*/
4115			VALUE lit;
4116			lit = $2->nd_lit;
4117			$2->nd_lit = ID2SYM(rb_intern_str(lit));
4118			nd_set_type($2, NODE_LIT);
4119			$$ = list_append($1, $2);
4120		    /*%
4121			$$ = dispatch2(qsymbols_add, $1, $2);
4122		    %*/
4123		    }
4124		;
4125
4126string_contents : /* none */
4127		    {
4128		    /*%%%*/
4129			$$ = 0;
4130		    /*%
4131			$$ = dispatch0(string_content);
4132		    %*/
4133		    }
4134		| string_contents string_content
4135		    {
4136		    /*%%%*/
4137			$$ = literal_concat($1, $2);
4138		    /*%
4139			$$ = dispatch2(string_add, $1, $2);
4140		    %*/
4141		    }
4142		;
4143
4144xstring_contents: /* none */
4145		    {
4146		    /*%%%*/
4147			$$ = 0;
4148		    /*%
4149			$$ = dispatch0(xstring_new);
4150		    %*/
4151		    }
4152		| xstring_contents string_content
4153		    {
4154		    /*%%%*/
4155			$$ = literal_concat($1, $2);
4156		    /*%
4157			$$ = dispatch2(xstring_add, $1, $2);
4158		    %*/
4159		    }
4160		;
4161
4162regexp_contents: /* none */
4163		    {
4164		    /*%%%*/
4165			$$ = 0;
4166		    /*%
4167			$$ = dispatch0(regexp_new);
4168		    %*/
4169		    }
4170		| regexp_contents string_content
4171		    {
4172		    /*%%%*/
4173			NODE *head = $1, *tail = $2;
4174			if (!head) {
4175			    $$ = tail;
4176			}
4177			else if (!tail) {
4178			    $$ = head;
4179			}
4180			else {
4181			    switch (nd_type(head)) {
4182			      case NODE_STR:
4183				nd_set_type(head, NODE_DSTR);
4184				break;
4185			      case NODE_DSTR:
4186				break;
4187			      default:
4188				head = list_append(NEW_DSTR(Qnil), head);
4189				break;
4190			    }
4191			    $$ = list_append(head, tail);
4192			}
4193		    /*%
4194			$$ = dispatch2(regexp_add, $1, $2);
4195		    %*/
4196		    }
4197		;
4198
4199string_content	: tSTRING_CONTENT
4200		| tSTRING_DVAR
4201		    {
4202			$<node>$ = lex_strterm;
4203			lex_strterm = 0;
4204			lex_state = EXPR_BEG;
4205		    }
4206		  string_dvar
4207		    {
4208		    /*%%%*/
4209			lex_strterm = $<node>2;
4210			$$ = NEW_EVSTR($3);
4211		    /*%
4212			lex_strterm = $<node>2;
4213			$$ = dispatch1(string_dvar, $3);
4214		    %*/
4215		    }
4216		| tSTRING_DBEG
4217		    {
4218			$<val>1 = cond_stack;
4219			$<val>$ = cmdarg_stack;
4220			cond_stack = 0;
4221			cmdarg_stack = 0;
4222		    }
4223		    {
4224			$<node>$ = lex_strterm;
4225			lex_strterm = 0;
4226			lex_state = EXPR_BEG;
4227		    }
4228		    {
4229			$<num>$ = brace_nest;
4230			brace_nest = 0;
4231		    }
4232		  compstmt tSTRING_DEND
4233		    {
4234			cond_stack = $<val>1;
4235			cmdarg_stack = $<val>2;
4236			lex_strterm = $<node>3;
4237			brace_nest = $<num>4;
4238		    /*%%%*/
4239			if ($5) $5->flags &= ~NODE_FL_NEWLINE;
4240			$$ = new_evstr($5);
4241		    /*%
4242			$$ = dispatch1(string_embexpr, $5);
4243		    %*/
4244		    }
4245		;
4246
4247string_dvar	: tGVAR
4248		    {
4249		    /*%%%*/
4250			$$ = NEW_GVAR($1);
4251		    /*%
4252			$$ = dispatch1(var_ref, $1);
4253		    %*/
4254		    }
4255		| tIVAR
4256		    {
4257		    /*%%%*/
4258			$$ = NEW_IVAR($1);
4259		    /*%
4260			$$ = dispatch1(var_ref, $1);
4261		    %*/
4262		    }
4263		| tCVAR
4264		    {
4265		    /*%%%*/
4266			$$ = NEW_CVAR($1);
4267		    /*%
4268			$$ = dispatch1(var_ref, $1);
4269		    %*/
4270		    }
4271		| backref
4272		;
4273
4274symbol		: tSYMBEG sym
4275		    {
4276			lex_state = EXPR_END;
4277		    /*%%%*/
4278			$$ = $2;
4279		    /*%
4280			$$ = dispatch1(symbol, $2);
4281		    %*/
4282		    }
4283		;
4284
4285sym		: fname
4286		| tIVAR
4287		| tGVAR
4288		| tCVAR
4289		;
4290
4291dsym		: tSYMBEG xstring_contents tSTRING_END
4292		    {
4293			lex_state = EXPR_END;
4294		    /*%%%*/
4295			$$ = dsym_node($2);
4296		    /*%
4297			$$ = dispatch1(dyna_symbol, $2);
4298		    %*/
4299		    }
4300		;
4301
4302numeric 	: tINTEGER
4303		| tFLOAT
4304		| tUMINUS_NUM tINTEGER	       %prec tLOWEST
4305		    {
4306		    /*%%%*/
4307			$$ = negate_lit($2);
4308		    /*%
4309			$$ = dispatch2(unary, ripper_intern("-@"), $2);
4310		    %*/
4311		    }
4312		| tUMINUS_NUM tFLOAT	       %prec tLOWEST
4313		    {
4314		    /*%%%*/
4315			$$ = negate_lit($2);
4316		    /*%
4317			$$ = dispatch2(unary, ripper_intern("-@"), $2);
4318		    %*/
4319		    }
4320		;
4321
4322user_variable	: tIDENTIFIER
4323		| tIVAR
4324		| tGVAR
4325		| tCONSTANT
4326		| tCVAR
4327		;
4328
4329keyword_variable: keyword_nil {ifndef_ripper($$ = keyword_nil);}
4330		| keyword_self {ifndef_ripper($$ = keyword_self);}
4331		| keyword_true {ifndef_ripper($$ = keyword_true);}
4332		| keyword_false {ifndef_ripper($$ = keyword_false);}
4333		| keyword__FILE__ {ifndef_ripper($$ = keyword__FILE__);}
4334		| keyword__LINE__ {ifndef_ripper($$ = keyword__LINE__);}
4335		| keyword__ENCODING__ {ifndef_ripper($$ = keyword__ENCODING__);}
4336		;
4337
4338var_ref		: user_variable
4339		    {
4340		    /*%%%*/
4341			if (!($$ = gettable($1))) $$ = NEW_BEGIN(0);
4342		    /*%
4343			if (id_is_var(get_id($1))) {
4344			    $$ = dispatch1(var_ref, $1);
4345			}
4346			else {
4347			    $$ = dispatch1(vcall, $1);
4348			}
4349		    %*/
4350		    }
4351		| keyword_variable
4352		    {
4353		    /*%%%*/
4354			if (!($$ = gettable($1))) $$ = NEW_BEGIN(0);
4355		    /*%
4356			$$ = dispatch1(var_ref, $1);
4357		    %*/
4358		    }
4359		;
4360
4361var_lhs		: user_variable
4362		    {
4363			$$ = assignable($1, 0);
4364		    /*%%%*/
4365		    /*%
4366			$$ = dispatch1(var_field, $$);
4367		    %*/
4368		    }
4369		| keyword_variable
4370		    {
4371		        $$ = assignable($1, 0);
4372		    /*%%%*/
4373		    /*%
4374			$$ = dispatch1(var_field, $$);
4375		    %*/
4376		    }
4377		;
4378
4379backref		: tNTH_REF
4380		| tBACK_REF
4381		;
4382
4383superclass	: term
4384		    {
4385		    /*%%%*/
4386			$$ = 0;
4387		    /*%
4388			$$ = Qnil;
4389		    %*/
4390		    }
4391		| '<'
4392		    {
4393			lex_state = EXPR_BEG;
4394			command_start = TRUE;
4395		    }
4396		  expr_value term
4397		    {
4398			$$ = $3;
4399		    }
4400		| error term
4401		    {
4402		    /*%%%*/
4403			yyerrok;
4404			$$ = 0;
4405		    /*%
4406			yyerrok;
4407			$$ = Qnil;
4408		    %*/
4409		    }
4410		;
4411
4412f_arglist	: '(' f_args rparen
4413		    {
4414		    /*%%%*/
4415			$$ = $2;
4416		    /*%
4417			$$ = dispatch1(paren, $2);
4418		    %*/
4419			lex_state = EXPR_BEG;
4420			command_start = TRUE;
4421		    }
4422		| f_args term
4423		    {
4424			$$ = $1;
4425			lex_state = EXPR_BEG;
4426			command_start = TRUE;
4427		    }
4428		;
4429
4430args_tail	: f_kwarg ',' f_kwrest opt_f_block_arg
4431		    {
4432			$$ = new_args_tail($1, $3, $4);
4433		    }
4434		| f_kwarg opt_f_block_arg
4435		    {
4436			$$ = new_args_tail($1, Qnone, $2);
4437		    }
4438		| f_kwrest opt_f_block_arg
4439		    {
4440			$$ = new_args_tail(Qnone, $1, $2);
4441		    }
4442		| f_block_arg
4443		    {
4444			$$ = new_args_tail(Qnone, Qnone, $1);
4445		    }
4446		;
4447
4448opt_args_tail	: ',' args_tail
4449		    {
4450			$$ = $2;
4451		    }
4452		| /* none */
4453		    {
4454			$$ = new_args_tail(Qnone, Qnone, Qnone);
4455		    }
4456		;
4457
4458f_args		: f_arg ',' f_optarg ',' f_rest_arg opt_args_tail
4459		    {
4460			$$ = new_args($1, $3, $5, Qnone, $6);
4461		    }
4462		| f_arg ',' f_optarg ',' f_rest_arg ',' f_arg opt_args_tail
4463		    {
4464			$$ = new_args($1, $3, $5, $7, $8);
4465		    }
4466		| f_arg ',' f_optarg opt_args_tail
4467		    {
4468			$$ = new_args($1, $3, Qnone, Qnone, $4);
4469		    }
4470		| f_arg ',' f_optarg ',' f_arg opt_args_tail
4471		    {
4472			$$ = new_args($1, $3, Qnone, $5, $6);
4473		    }
4474		| f_arg ',' f_rest_arg opt_args_tail
4475		    {
4476			$$ = new_args($1, Qnone, $3, Qnone, $4);
4477		    }
4478		| f_arg ',' f_rest_arg ',' f_arg opt_args_tail
4479		    {
4480			$$ = new_args($1, Qnone, $3, $5, $6);
4481		    }
4482		| f_arg opt_args_tail
4483		    {
4484			$$ = new_args($1, Qnone, Qnone, Qnone, $2);
4485		    }
4486		| f_optarg ',' f_rest_arg opt_args_tail
4487		    {
4488			$$ = new_args(Qnone, $1, $3, Qnone, $4);
4489		    }
4490		| f_optarg ',' f_rest_arg ',' f_arg opt_args_tail
4491		    {
4492			$$ = new_args(Qnone, $1, $3, $5, $6);
4493		    }
4494		| f_optarg opt_args_tail
4495		    {
4496			$$ = new_args(Qnone, $1, Qnone, Qnone, $2);
4497		    }
4498		| f_optarg ',' f_arg opt_args_tail
4499		    {
4500			$$ = new_args(Qnone, $1, Qnone, $3, $4);
4501		    }
4502		| f_rest_arg opt_args_tail
4503		    {
4504			$$ = new_args(Qnone, Qnone, $1, Qnone, $2);
4505		    }
4506		| f_rest_arg ',' f_arg opt_args_tail
4507		    {
4508			$$ = new_args(Qnone, Qnone, $1, $3, $4);
4509		    }
4510		| args_tail
4511		    {
4512			$$ = new_args(Qnone, Qnone, Qnone, Qnone, $1);
4513		    }
4514		| /* none */
4515		    {
4516			$$ = new_args_tail(Qnone, Qnone, Qnone);
4517			$$ = new_args(Qnone, Qnone, Qnone, Qnone, $$);
4518		    }
4519		;
4520
4521f_bad_arg	: tCONSTANT
4522		    {
4523		    /*%%%*/
4524			yyerror("formal argument cannot be a constant");
4525			$$ = 0;
4526		    /*%
4527			$$ = dispatch1(param_error, $1);
4528		    %*/
4529		    }
4530		| tIVAR
4531		    {
4532		    /*%%%*/
4533			yyerror("formal argument cannot be an instance variable");
4534			$$ = 0;
4535		    /*%
4536			$$ = dispatch1(param_error, $1);
4537		    %*/
4538		    }
4539		| tGVAR
4540		    {
4541		    /*%%%*/
4542			yyerror("formal argument cannot be a global variable");
4543			$$ = 0;
4544		    /*%
4545			$$ = dispatch1(param_error, $1);
4546		    %*/
4547		    }
4548		| tCVAR
4549		    {
4550		    /*%%%*/
4551			yyerror("formal argument cannot be a class variable");
4552			$$ = 0;
4553		    /*%
4554			$$ = dispatch1(param_error, $1);
4555		    %*/
4556		    }
4557		;
4558
4559f_norm_arg	: f_bad_arg
4560		| tIDENTIFIER
4561		    {
4562			formal_argument(get_id($1));
4563			$$ = $1;
4564		    }
4565		;
4566
4567f_arg_item	: f_norm_arg
4568		    {
4569			arg_var(get_id($1));
4570		    /*%%%*/
4571			$$ = NEW_ARGS_AUX($1, 1);
4572		    /*%
4573			$$ = get_value($1);
4574		    %*/
4575		    }
4576		| tLPAREN f_margs rparen
4577		    {
4578			ID tid = internal_id();
4579			arg_var(tid);
4580		    /*%%%*/
4581			if (dyna_in_block()) {
4582			    $2->nd_value = NEW_DVAR(tid);
4583			}
4584			else {
4585			    $2->nd_value = NEW_LVAR(tid);
4586			}
4587			$$ = NEW_ARGS_AUX(tid, 1);
4588			$$->nd_next = $2;
4589		    /*%
4590			$$ = dispatch1(mlhs_paren, $2);
4591		    %*/
4592		    }
4593		;
4594
4595f_arg		: f_arg_item
4596		    /*%c%*/
4597		    /*%c
4598		    {
4599			$$ = rb_ary_new3(1, $1);
4600		    }
4601		    c%*/
4602		| f_arg ',' f_arg_item
4603		    {
4604		    /*%%%*/
4605			$$ = $1;
4606			$$->nd_plen++;
4607			$$->nd_next = block_append($$->nd_next, $3->nd_next);
4608			rb_gc_force_recycle((VALUE)$3);
4609		    /*%
4610			$$ = rb_ary_push($1, $3);
4611		    %*/
4612		    }
4613		;
4614
4615f_kw		: tLABEL arg_value
4616		    {
4617			arg_var(formal_argument(get_id($1)));
4618			$$ = assignable($1, $2);
4619		    /*%%%*/
4620			$$ = NEW_KW_ARG(0, $$);
4621		    /*%
4622			$$ = rb_assoc_new($$, $2);
4623		    %*/
4624		    }
4625		;
4626
4627f_block_kw	: tLABEL primary_value
4628		    {
4629			arg_var(formal_argument(get_id($1)));
4630			$$ = assignable($1, $2);
4631		    /*%%%*/
4632			$$ = NEW_KW_ARG(0, $$);
4633		    /*%
4634			$$ = rb_assoc_new($$, $2);
4635		    %*/
4636		    }
4637		;
4638
4639f_block_kwarg	: f_block_kw
4640		    {
4641		    /*%%%*/
4642			$$ = $1;
4643		    /*%
4644			$$ = rb_ary_new3(1, $1);
4645		    %*/
4646		    }
4647		| f_block_kwarg ',' f_block_kw
4648		    {
4649		    /*%%%*/
4650			NODE *kws = $1;
4651
4652			while (kws->nd_next) {
4653			    kws = kws->nd_next;
4654			}
4655			kws->nd_next = $3;
4656			$$ = $1;
4657		    /*%
4658			$$ = rb_ary_push($1, $3);
4659		    %*/
4660		    }
4661		;
4662
4663
4664f_kwarg		: f_kw
4665		    {
4666		    /*%%%*/
4667			$$ = $1;
4668		    /*%
4669			$$ = rb_ary_new3(1, $1);
4670		    %*/
4671		    }
4672		| f_kwarg ',' f_kw
4673		    {
4674		    /*%%%*/
4675			NODE *kws = $1;
4676
4677			while (kws->nd_next) {
4678			    kws = kws->nd_next;
4679			}
4680			kws->nd_next = $3;
4681			$$ = $1;
4682		    /*%
4683			$$ = rb_ary_push($1, $3);
4684		    %*/
4685		    }
4686		;
4687
4688kwrest_mark	: tPOW
4689		| tDSTAR
4690		;
4691
4692f_kwrest	: kwrest_mark tIDENTIFIER
4693		    {
4694			shadowing_lvar(get_id($2));
4695			$$ = $2;
4696		    }
4697		| kwrest_mark
4698		    {
4699			$$ = internal_id();
4700		    }
4701		;
4702
4703f_opt		: tIDENTIFIER '=' arg_value
4704		    {
4705			arg_var(formal_argument(get_id($1)));
4706			$$ = assignable($1, $3);
4707		    /*%%%*/
4708			$$ = NEW_OPT_ARG(0, $$);
4709		    /*%
4710			$$ = rb_assoc_new($$, $3);
4711		    %*/
4712		    }
4713		;
4714
4715f_block_opt	: tIDENTIFIER '=' primary_value
4716		    {
4717			arg_var(formal_argument(get_id($1)));
4718			$$ = assignable($1, $3);
4719		    /*%%%*/
4720			$$ = NEW_OPT_ARG(0, $$);
4721		    /*%
4722			$$ = rb_assoc_new($$, $3);
4723		    %*/
4724		    }
4725		;
4726
4727f_block_optarg	: f_block_opt
4728		    {
4729		    /*%%%*/
4730			$$ = $1;
4731		    /*%
4732			$$ = rb_ary_new3(1, $1);
4733		    %*/
4734		    }
4735		| f_block_optarg ',' f_block_opt
4736		    {
4737		    /*%%%*/
4738			NODE *opts = $1;
4739
4740			while (opts->nd_next) {
4741			    opts = opts->nd_next;
4742			}
4743			opts->nd_next = $3;
4744			$$ = $1;
4745		    /*%
4746			$$ = rb_ary_push($1, $3);
4747		    %*/
4748		    }
4749		;
4750
4751f_optarg	: f_opt
4752		    {
4753		    /*%%%*/
4754			$$ = $1;
4755		    /*%
4756			$$ = rb_ary_new3(1, $1);
4757		    %*/
4758		    }
4759		| f_optarg ',' f_opt
4760		    {
4761		    /*%%%*/
4762			NODE *opts = $1;
4763
4764			while (opts->nd_next) {
4765			    opts = opts->nd_next;
4766			}
4767			opts->nd_next = $3;
4768			$$ = $1;
4769		    /*%
4770			$$ = rb_ary_push($1, $3);
4771		    %*/
4772		    }
4773		;
4774
4775restarg_mark	: '*'
4776		| tSTAR
4777		;
4778
4779f_rest_arg	: restarg_mark tIDENTIFIER
4780		    {
4781		    /*%%%*/
4782			if (!is_local_id($2))
4783			    yyerror("rest argument must be local variable");
4784		    /*% %*/
4785			arg_var(shadowing_lvar(get_id($2)));
4786		    /*%%%*/
4787			$$ = $2;
4788		    /*%
4789			$$ = dispatch1(rest_param, $2);
4790		    %*/
4791		    }
4792		| restarg_mark
4793		    {
4794		    /*%%%*/
4795			$$ = internal_id();
4796			arg_var($$);
4797		    /*%
4798			$$ = dispatch1(rest_param, Qnil);
4799		    %*/
4800		    }
4801		;
4802
4803blkarg_mark	: '&'
4804		| tAMPER
4805		;
4806
4807f_block_arg	: blkarg_mark tIDENTIFIER
4808		    {
4809		    /*%%%*/
4810			if (!is_local_id($2))
4811			    yyerror("block argument must be local variable");
4812			else if (!dyna_in_block() && local_id($2))
4813			    yyerror("duplicated block argument name");
4814		    /*% %*/
4815			arg_var(shadowing_lvar(get_id($2)));
4816		    /*%%%*/
4817			$$ = $2;
4818		    /*%
4819			$$ = dispatch1(blockarg, $2);
4820		    %*/
4821		    }
4822		;
4823
4824opt_f_block_arg	: ',' f_block_arg
4825		    {
4826			$$ = $2;
4827		    }
4828		| none
4829		    {
4830		    /*%%%*/
4831			$$ = 0;
4832		    /*%
4833			$$ = Qundef;
4834		    %*/
4835		    }
4836		;
4837
4838singleton	: var_ref
4839		    {
4840		    /*%%%*/
4841			value_expr($1);
4842			$$ = $1;
4843		        if (!$$) $$ = NEW_NIL();
4844		    /*%
4845			$$ = $1;
4846		    %*/
4847		    }
4848		| '(' {lex_state = EXPR_BEG;} expr rparen
4849		    {
4850		    /*%%%*/
4851			if ($3 == 0) {
4852			    yyerror("can't define singleton method for ().");
4853			}
4854			else {
4855			    switch (nd_type($3)) {
4856			      case NODE_STR:
4857			      case NODE_DSTR:
4858			      case NODE_XSTR:
4859			      case NODE_DXSTR:
4860			      case NODE_DREGX:
4861			      case NODE_LIT:
4862			      case NODE_ARRAY:
4863			      case NODE_ZARRAY:
4864				yyerror("can't define singleton method for literals");
4865			      default:
4866				value_expr($3);
4867				break;
4868			    }
4869			}
4870			$$ = $3;
4871		    /*%
4872			$$ = dispatch1(paren, $3);
4873		    %*/
4874		    }
4875		;
4876
4877assoc_list	: none
4878		| assocs trailer
4879		    {
4880		    /*%%%*/
4881			$$ = $1;
4882		    /*%
4883			$$ = dispatch1(assoclist_from_args, $1);
4884		    %*/
4885		    }
4886		;
4887
4888assocs		: assoc
4889		    /*%c%*/
4890		    /*%c
4891		    {
4892			$$ = rb_ary_new3(1, $1);
4893		    }
4894		    %*/
4895		| assocs ',' assoc
4896		    {
4897		    /*%%%*/
4898			$$ = list_concat($1, $3);
4899		    /*%
4900			$$ = rb_ary_push($1, $3);
4901		    %*/
4902		    }
4903		;
4904
4905assoc		: arg_value tASSOC arg_value
4906		    {
4907		    /*%%%*/
4908			$$ = list_append(NEW_LIST($1), $3);
4909		    /*%
4910			$$ = dispatch2(assoc_new, $1, $3);
4911		    %*/
4912		    }
4913		| tLABEL arg_value
4914		    {
4915		    /*%%%*/
4916			$$ = list_append(NEW_LIST(NEW_LIT(ID2SYM($1))), $2);
4917		    /*%
4918			$$ = dispatch2(assoc_new, $1, $2);
4919		    %*/
4920		    }
4921		| tDSTAR arg_value
4922		    {
4923		    /*%%%*/
4924			$$ = list_append(NEW_LIST(0), $2);
4925		    /*%
4926			$$ = dispatch1(assoc_splat, $2);
4927		    %*/
4928		    }
4929		;
4930
4931		;
4932
4933operation	: tIDENTIFIER
4934		| tCONSTANT
4935		| tFID
4936		;
4937
4938operation2	: tIDENTIFIER
4939		| tCONSTANT
4940		| tFID
4941		| op
4942		;
4943
4944operation3	: tIDENTIFIER
4945		| tFID
4946		| op
4947		;
4948
4949dot_or_colon	: '.'
4950		    /*%c%*/
4951		    /*%c
4952		    { $$ = $<val>1; }
4953		    %*/
4954		| tCOLON2
4955		    /*%c%*/
4956		    /*%c
4957		    { $$ = $<val>1; }
4958		    %*/
4959		;
4960
4961opt_terms	: /* none */
4962		| terms
4963		;
4964
4965opt_nl		: /* none */
4966		| '\n'
4967		;
4968
4969rparen		: opt_nl ')'
4970		;
4971
4972rbracket	: opt_nl ']'
4973		;
4974
4975trailer		: /* none */
4976		| '\n'
4977		| ','
4978		;
4979
4980term		: ';' {yyerrok;}
4981		| '\n'
4982		;
4983
4984terms		: term
4985		| terms ';' {yyerrok;}
4986		;
4987
4988none		: /* none */
4989		    {
4990		    /*%%%*/
4991			$$ = 0;
4992		    /*%
4993			$$ = Qundef;
4994		    %*/
4995		    }
4996		;
4997%%
4998# undef parser
4999# undef yylex
5000# undef yylval
5001# define yylval  (*((YYSTYPE*)(parser->parser_yylval)))
5002
5003static int parser_regx_options(struct parser_params*);
5004static int parser_tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**);
5005static void parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc);
5006static int parser_parse_string(struct parser_params*,NODE*);
5007static int parser_here_document(struct parser_params*,NODE*);
5008
5009
5010# define nextc()                   parser_nextc(parser)
5011# define pushback(c)               parser_pushback(parser, (c))
5012# define newtok()                  parser_newtok(parser)
5013# define tokspace(n)               parser_tokspace(parser, (n))
5014# define tokadd(c)                 parser_tokadd(parser, (c))
5015# define tok_hex(numlen)           parser_tok_hex(parser, (numlen))
5016# define read_escape(flags,e)      parser_read_escape(parser, (flags), (e))
5017# define tokadd_escape(e)          parser_tokadd_escape(parser, (e))
5018# define regx_options()            parser_regx_options(parser)
5019# define tokadd_string(f,t,p,n,e)  parser_tokadd_string(parser,(f),(t),(p),(n),(e))
5020# define parse_string(n)           parser_parse_string(parser,(n))
5021# define tokaddmbc(c, enc)         parser_tokaddmbc(parser, (c), (enc))
5022# define here_document(n)          parser_here_document(parser,(n))
5023# define heredoc_identifier()      parser_heredoc_identifier(parser)
5024# define heredoc_restore(n)        parser_heredoc_restore(parser,(n))
5025# define whole_match_p(e,l,i)      parser_whole_match_p(parser,(e),(l),(i))
5026
5027#ifndef RIPPER
5028# define set_yylval_str(x) (yylval.node = NEW_STR(x))
5029# define set_yylval_num(x) (yylval.num = (x))
5030# define set_yylval_id(x)  (yylval.id = (x))
5031# define set_yylval_name(x)  (yylval.id = (x))
5032# define set_yylval_literal(x) (yylval.node = NEW_LIT(x))
5033# define set_yylval_node(x) (yylval.node = (x))
5034# define yylval_id() (yylval.id)
5035#else
5036static inline VALUE
5037ripper_yylval_id(ID x)
5038{
5039    return (VALUE)NEW_LASGN(x, ID2SYM(x));
5040}
5041# define set_yylval_str(x) (void)(x)
5042# define set_yylval_num(x) (void)(x)
5043# define set_yylval_id(x)  (void)(x)
5044# define set_yylval_name(x) (void)(yylval.val = ripper_yylval_id(x))
5045# define set_yylval_literal(x) (void)(x)
5046# define set_yylval_node(x) (void)(x)
5047# define yylval_id() yylval.id
5048#endif
5049
5050#ifndef RIPPER
5051#define ripper_flush(p) (void)(p)
5052#else
5053#define ripper_flush(p) ((p)->tokp = (p)->parser_lex_p)
5054
5055#define yylval_rval (*(RB_TYPE_P(yylval.val, T_NODE) ? &yylval.node->nd_rval : &yylval.val))
5056
5057static int
5058ripper_has_scan_event(struct parser_params *parser)
5059{
5060
5061    if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp");
5062    return lex_p > parser->tokp;
5063}
5064
5065static VALUE
5066ripper_scan_event_val(struct parser_params *parser, int t)
5067{
5068    VALUE str = STR_NEW(parser->tokp, lex_p - parser->tokp);
5069    VALUE rval = ripper_dispatch1(parser, ripper_token2eventid(t), str);
5070    ripper_flush(parser);
5071    return rval;
5072}
5073
5074static void
5075ripper_dispatch_scan_event(struct parser_params *parser, int t)
5076{
5077    if (!ripper_has_scan_event(parser)) return;
5078    yylval_rval = ripper_scan_event_val(parser, t);
5079}
5080
5081static void
5082ripper_dispatch_ignored_scan_event(struct parser_params *parser, int t)
5083{
5084    if (!ripper_has_scan_event(parser)) return;
5085    (void)ripper_scan_event_val(parser, t);
5086}
5087
5088static void
5089ripper_dispatch_delayed_token(struct parser_params *parser, int t)
5090{
5091    int saved_line = ruby_sourceline;
5092    const char *saved_tokp = parser->tokp;
5093
5094    ruby_sourceline = parser->delayed_line;
5095    parser->tokp = lex_pbeg + parser->delayed_col;
5096    yylval_rval = ripper_dispatch1(parser, ripper_token2eventid(t), parser->delayed);
5097    parser->delayed = Qnil;
5098    ruby_sourceline = saved_line;
5099    parser->tokp = saved_tokp;
5100}
5101#endif /* RIPPER */
5102
5103#include "ruby/regex.h"
5104#include "ruby/util.h"
5105
5106/* We remove any previous definition of `SIGN_EXTEND_CHAR',
5107   since ours (we hope) works properly with all combinations of
5108   machines, compilers, `char' and `unsigned char' argument types.
5109   (Per Bothner suggested the basic approach.)  */
5110#undef SIGN_EXTEND_CHAR
5111#if __STDC__
5112# define SIGN_EXTEND_CHAR(c) ((signed char)(c))
5113#else  /* not __STDC__ */
5114/* As in Harbison and Steele.  */
5115# define SIGN_EXTEND_CHAR(c) ((((unsigned char)(c)) ^ 128) - 128)
5116#endif
5117
5118#define parser_encoding_name()  (current_enc->name)
5119#define parser_mbclen()  mbclen((lex_p-1),lex_pend,current_enc)
5120#define parser_precise_mbclen()  rb_enc_precise_mbclen((lex_p-1),lex_pend,current_enc)
5121#define is_identchar(p,e,enc) (rb_enc_isalnum(*(p),(enc)) || (*(p)) == '_' || !ISASCII(*(p)))
5122#define parser_is_identchar() (!parser->eofp && is_identchar((lex_p-1),lex_pend,current_enc))
5123
5124#define parser_isascii() ISASCII(*(lex_p-1))
5125
5126#ifndef RIPPER
5127static int
5128token_info_get_column(struct parser_params *parser, const char *token)
5129{
5130    int column = 1;
5131    const char *p, *pend = lex_p - strlen(token);
5132    for (p = lex_pbeg; p < pend; p++) {
5133	if (*p == '\t') {
5134	    column = (((column - 1) / 8) + 1) * 8;
5135	}
5136	column++;
5137    }
5138    return column;
5139}
5140
5141static int
5142token_info_has_nonspaces(struct parser_params *parser, const char *token)
5143{
5144    const char *p, *pend = lex_p - strlen(token);
5145    for (p = lex_pbeg; p < pend; p++) {
5146	if (*p != ' ' && *p != '\t') {
5147	    return 1;
5148	}
5149    }
5150    return 0;
5151}
5152
5153#undef token_info_push
5154static void
5155token_info_push(struct parser_params *parser, const char *token)
5156{
5157    token_info *ptinfo;
5158
5159    if (!parser->parser_token_info_enabled) return;
5160    ptinfo = ALLOC(token_info);
5161    ptinfo->token = token;
5162    ptinfo->linenum = ruby_sourceline;
5163    ptinfo->column = token_info_get_column(parser, token);
5164    ptinfo->nonspc = token_info_has_nonspaces(parser, token);
5165    ptinfo->next = parser->parser_token_info;
5166
5167    parser->parser_token_info = ptinfo;
5168}
5169
5170#undef token_info_pop
5171static void
5172token_info_pop(struct parser_params *parser, const char *token)
5173{
5174    int linenum;
5175    token_info *ptinfo = parser->parser_token_info;
5176
5177    if (!ptinfo) return;
5178    parser->parser_token_info = ptinfo->next;
5179    if (token_info_get_column(parser, token) == ptinfo->column) { /* OK */
5180	goto finish;
5181    }
5182    linenum = ruby_sourceline;
5183    if (linenum == ptinfo->linenum) { /* SKIP */
5184	goto finish;
5185    }
5186    if (token_info_has_nonspaces(parser, token) || ptinfo->nonspc) { /* SKIP */
5187	goto finish;
5188    }
5189    if (parser->parser_token_info_enabled) {
5190	rb_compile_warn(ruby_sourcefile, linenum,
5191			"mismatched indentations at '%s' with '%s' at %d",
5192			token, ptinfo->token, ptinfo->linenum);
5193    }
5194
5195  finish:
5196    xfree(ptinfo);
5197}
5198#endif	/* RIPPER */
5199
5200static int
5201parser_yyerror(struct parser_params *parser, const char *msg)
5202{
5203#ifndef RIPPER
5204    const int max_line_margin = 30;
5205    const char *p, *pe;
5206    char *buf;
5207    long len;
5208    int i;
5209
5210    compile_error(PARSER_ARG "%s", msg);
5211    p = lex_p;
5212    while (lex_pbeg <= p) {
5213	if (*p == '\n') break;
5214	p--;
5215    }
5216    p++;
5217
5218    pe = lex_p;
5219    while (pe < lex_pend) {
5220	if (*pe == '\n') break;
5221	pe++;
5222    }
5223
5224    len = pe - p;
5225    if (len > 4) {
5226	char *p2;
5227	const char *pre = "", *post = "";
5228
5229	if (len > max_line_margin * 2 + 10) {
5230	    if (lex_p - p > max_line_margin) {
5231		p = rb_enc_prev_char(p, lex_p - max_line_margin, pe, rb_enc_get(lex_lastline));
5232		pre = "...";
5233	    }
5234	    if (pe - lex_p > max_line_margin) {
5235		pe = rb_enc_prev_char(lex_p, lex_p + max_line_margin, pe, rb_enc_get(lex_lastline));
5236		post = "...";
5237	    }
5238	    len = pe - p;
5239	}
5240	buf = ALLOCA_N(char, len+2);
5241	MEMCPY(buf, p, char, len);
5242	buf[len] = '\0';
5243	rb_compile_error_append("%s%s%s", pre, buf, post);
5244
5245	i = (int)(lex_p - p);
5246	p2 = buf; pe = buf + len;
5247
5248	while (p2 < pe) {
5249	    if (*p2 != '\t') *p2 = ' ';
5250	    p2++;
5251	}
5252	buf[i] = '^';
5253	buf[i+1] = '\0';
5254	rb_compile_error_append("%s%s", pre, buf);
5255    }
5256#else
5257    dispatch1(parse_error, STR_NEW2(msg));
5258#endif /* !RIPPER */
5259    return 0;
5260}
5261
5262static void parser_prepare(struct parser_params *parser);
5263
5264#ifndef RIPPER
5265static VALUE
5266debug_lines(VALUE fname)
5267{
5268    ID script_lines;
5269    CONST_ID(script_lines, "SCRIPT_LINES__");
5270    if (rb_const_defined_at(rb_cObject, script_lines)) {
5271	VALUE hash = rb_const_get_at(rb_cObject, script_lines);
5272	if (RB_TYPE_P(hash, T_HASH)) {
5273	    VALUE lines = rb_ary_new();
5274	    rb_hash_aset(hash, fname, lines);
5275	    return lines;
5276	}
5277    }
5278    return 0;
5279}
5280
5281static VALUE
5282coverage(VALUE fname, int n)
5283{
5284    VALUE coverages = rb_get_coverages();
5285    if (RTEST(coverages) && RBASIC(coverages)->klass == 0) {
5286	VALUE lines = rb_ary_new2(n);
5287	int i;
5288	RBASIC(lines)->klass = 0;
5289	for (i = 0; i < n; i++) RARRAY_PTR(lines)[i] = Qnil;
5290	RARRAY(lines)->as.heap.len = n;
5291	rb_hash_aset(coverages, fname, lines);
5292	return lines;
5293    }
5294    return 0;
5295}
5296
5297static int
5298e_option_supplied(struct parser_params *parser)
5299{
5300    return strcmp(ruby_sourcefile, "-e") == 0;
5301}
5302
5303static VALUE
5304yycompile0(VALUE arg)
5305{
5306    int n;
5307    NODE *tree;
5308    struct parser_params *parser = (struct parser_params *)arg;
5309
5310    if (!compile_for_eval && rb_safe_level() == 0) {
5311	ruby_debug_lines = debug_lines(ruby_sourcefile_string);
5312	if (ruby_debug_lines && ruby_sourceline > 0) {
5313	    VALUE str = STR_NEW0();
5314	    n = ruby_sourceline;
5315	    do {
5316		rb_ary_push(ruby_debug_lines, str);
5317	    } while (--n);
5318	}
5319
5320	if (!e_option_supplied(parser)) {
5321	    ruby_coverage = coverage(ruby_sourcefile_string, ruby_sourceline);
5322	}
5323    }
5324
5325    parser_prepare(parser);
5326    deferred_nodes = 0;
5327#ifndef RIPPER
5328    parser->parser_token_info_enabled = !compile_for_eval && RTEST(ruby_verbose);
5329#endif
5330#ifndef RIPPER
5331    if (RUBY_DTRACE_PARSE_BEGIN_ENABLED()) {
5332	RUBY_DTRACE_PARSE_BEGIN(parser->parser_ruby_sourcefile,
5333				parser->parser_ruby_sourceline);
5334    }
5335#endif
5336    n = yyparse((void*)parser);
5337#ifndef RIPPER
5338    if (RUBY_DTRACE_PARSE_END_ENABLED()) {
5339	RUBY_DTRACE_PARSE_END(parser->parser_ruby_sourcefile,
5340			      parser->parser_ruby_sourceline);
5341    }
5342#endif
5343    ruby_debug_lines = 0;
5344    ruby_coverage = 0;
5345    compile_for_eval = 0;
5346
5347    lex_strterm = 0;
5348    lex_p = lex_pbeg = lex_pend = 0;
5349    lex_lastline = lex_nextline = 0;
5350    if (parser->nerr) {
5351	return 0;
5352    }
5353    tree = ruby_eval_tree;
5354    if (!tree) {
5355	tree = NEW_NIL();
5356    }
5357    else if (ruby_eval_tree_begin) {
5358	tree->nd_body = NEW_PRELUDE(ruby_eval_tree_begin, tree->nd_body);
5359    }
5360    return (VALUE)tree;
5361}
5362
5363static NODE*
5364yycompile(struct parser_params *parser, VALUE fname, int line)
5365{
5366    ruby_sourcefile_string = rb_str_new_frozen(fname);
5367    ruby_sourcefile = RSTRING_PTR(fname);
5368    ruby_sourceline = line - 1;
5369    return (NODE *)rb_suppress_tracing(yycompile0, (VALUE)parser);
5370}
5371#endif /* !RIPPER */
5372
5373static rb_encoding *
5374must_be_ascii_compatible(VALUE s)
5375{
5376    rb_encoding *enc = rb_enc_get(s);
5377    if (!rb_enc_asciicompat(enc)) {
5378	rb_raise(rb_eArgError, "invalid source encoding");
5379    }
5380    return enc;
5381}
5382
5383static VALUE
5384lex_get_str(struct parser_params *parser, VALUE s)
5385{
5386    char *beg, *end, *pend;
5387    rb_encoding *enc = must_be_ascii_compatible(s);
5388
5389    beg = RSTRING_PTR(s);
5390    if (lex_gets_ptr) {
5391	if (RSTRING_LEN(s) == lex_gets_ptr) return Qnil;
5392	beg += lex_gets_ptr;
5393    }
5394    pend = RSTRING_PTR(s) + RSTRING_LEN(s);
5395    end = beg;
5396    while (end < pend) {
5397	if (*end++ == '\n') break;
5398    }
5399    lex_gets_ptr = end - RSTRING_PTR(s);
5400    return rb_enc_str_new(beg, end - beg, enc);
5401}
5402
5403static VALUE
5404lex_getline(struct parser_params *parser)
5405{
5406    VALUE line = (*parser->parser_lex_gets)(parser, parser->parser_lex_input);
5407    if (NIL_P(line)) return line;
5408    must_be_ascii_compatible(line);
5409#ifndef RIPPER
5410    if (ruby_debug_lines) {
5411	rb_enc_associate(line, current_enc);
5412	rb_ary_push(ruby_debug_lines, line);
5413    }
5414    if (ruby_coverage) {
5415	rb_ary_push(ruby_coverage, Qnil);
5416    }
5417#endif
5418    return line;
5419}
5420
5421#ifdef RIPPER
5422static rb_data_type_t parser_data_type;
5423#else
5424static const rb_data_type_t parser_data_type;
5425
5426static NODE*
5427parser_compile_string(volatile VALUE vparser, VALUE fname, VALUE s, int line)
5428{
5429    struct parser_params *parser;
5430    NODE *node;
5431
5432    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
5433    lex_gets = lex_get_str;
5434    lex_gets_ptr = 0;
5435    lex_input = s;
5436    lex_pbeg = lex_p = lex_pend = 0;
5437    compile_for_eval = rb_parse_in_eval();
5438
5439    node = yycompile(parser, fname, line);
5440    RB_GC_GUARD(vparser); /* prohibit tail call optimization */
5441
5442    return node;
5443}
5444
5445NODE*
5446rb_compile_string(const char *f, VALUE s, int line)
5447{
5448    must_be_ascii_compatible(s);
5449    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), s, line);
5450}
5451
5452NODE*
5453rb_parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
5454{
5455    return rb_parser_compile_string_path(vparser, rb_filesystem_str_new_cstr(f), s, line);
5456}
5457
5458NODE*
5459rb_parser_compile_string_path(volatile VALUE vparser, VALUE f, VALUE s, int line)
5460{
5461    must_be_ascii_compatible(s);
5462    return parser_compile_string(vparser, f, s, line);
5463}
5464
5465NODE*
5466rb_compile_cstr(const char *f, const char *s, int len, int line)
5467{
5468    VALUE str = rb_str_new(s, len);
5469    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), str, line);
5470}
5471
5472NODE*
5473rb_parser_compile_cstr(volatile VALUE vparser, const char *f, const char *s, int len, int line)
5474{
5475    VALUE str = rb_str_new(s, len);
5476    return parser_compile_string(vparser, rb_filesystem_str_new_cstr(f), str, line);
5477}
5478
5479static VALUE
5480lex_io_gets(struct parser_params *parser, VALUE io)
5481{
5482    return rb_io_gets(io);
5483}
5484
5485NODE*
5486rb_compile_file(const char *f, VALUE file, int start)
5487{
5488    VALUE volatile vparser = rb_parser_new();
5489
5490    return rb_parser_compile_file(vparser, f, file, start);
5491}
5492
5493NODE*
5494rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int start)
5495{
5496    return rb_parser_compile_file_path(vparser, rb_filesystem_str_new_cstr(f), file, start);
5497}
5498
5499NODE*
5500rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE file, int start)
5501{
5502    struct parser_params *parser;
5503    NODE *node;
5504
5505    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
5506    lex_gets = lex_io_gets;
5507    lex_input = file;
5508    lex_pbeg = lex_p = lex_pend = 0;
5509    compile_for_eval = rb_parse_in_eval();
5510
5511    node = yycompile(parser, fname, start);
5512    RB_GC_GUARD(vparser); /* prohibit tail call optimization */
5513
5514    return node;
5515}
5516#endif  /* !RIPPER */
5517
5518#define STR_FUNC_ESCAPE 0x01
5519#define STR_FUNC_EXPAND 0x02
5520#define STR_FUNC_REGEXP 0x04
5521#define STR_FUNC_QWORDS 0x08
5522#define STR_FUNC_SYMBOL 0x10
5523#define STR_FUNC_INDENT 0x20
5524
5525enum string_type {
5526    str_squote = (0),
5527    str_dquote = (STR_FUNC_EXPAND),
5528    str_xquote = (STR_FUNC_EXPAND),
5529    str_regexp = (STR_FUNC_REGEXP|STR_FUNC_ESCAPE|STR_FUNC_EXPAND),
5530    str_sword  = (STR_FUNC_QWORDS),
5531    str_dword  = (STR_FUNC_QWORDS|STR_FUNC_EXPAND),
5532    str_ssym   = (STR_FUNC_SYMBOL),
5533    str_dsym   = (STR_FUNC_SYMBOL|STR_FUNC_EXPAND)
5534};
5535
5536static VALUE
5537parser_str_new(const char *p, long n, rb_encoding *enc, int func, rb_encoding *enc0)
5538{
5539    VALUE str;
5540
5541    str = rb_enc_str_new(p, n, enc);
5542    if (!(func & STR_FUNC_REGEXP) && rb_enc_asciicompat(enc)) {
5543	if (rb_enc_str_coderange(str) == ENC_CODERANGE_7BIT) {
5544	}
5545	else if (enc0 == rb_usascii_encoding() && enc != rb_utf8_encoding()) {
5546	    rb_enc_associate(str, rb_ascii8bit_encoding());
5547	}
5548    }
5549
5550    return str;
5551}
5552
5553#define lex_goto_eol(parser) ((parser)->parser_lex_p = (parser)->parser_lex_pend)
5554#define lex_eol_p() (lex_p >= lex_pend)
5555#define peek(c) peek_n((c), 0)
5556#define peek_n(c,n) (lex_p+(n) < lex_pend && (c) == (unsigned char)lex_p[n])
5557
5558static inline int
5559parser_nextc(struct parser_params *parser)
5560{
5561    int c;
5562
5563    if (lex_p == lex_pend) {
5564	VALUE v = lex_nextline;
5565	lex_nextline = 0;
5566	if (!v) {
5567	    if (parser->eofp)
5568		return -1;
5569
5570	    if (!lex_input || NIL_P(v = lex_getline(parser))) {
5571		parser->eofp = Qtrue;
5572		lex_goto_eol(parser);
5573		return -1;
5574	    }
5575	}
5576	{
5577#ifdef RIPPER
5578	    if (parser->tokp < lex_pend) {
5579		if (NIL_P(parser->delayed)) {
5580		    parser->delayed = rb_str_buf_new(1024);
5581		    rb_enc_associate(parser->delayed, current_enc);
5582		    rb_str_buf_cat(parser->delayed,
5583				   parser->tokp, lex_pend - parser->tokp);
5584		    parser->delayed_line = ruby_sourceline;
5585		    parser->delayed_col = (int)(parser->tokp - lex_pbeg);
5586		}
5587		else {
5588		    rb_str_buf_cat(parser->delayed,
5589				   parser->tokp, lex_pend - parser->tokp);
5590		}
5591	    }
5592#endif
5593	    if (heredoc_end > 0) {
5594		ruby_sourceline = heredoc_end;
5595		heredoc_end = 0;
5596	    }
5597	    ruby_sourceline++;
5598	    parser->line_count++;
5599	    lex_pbeg = lex_p = RSTRING_PTR(v);
5600	    lex_pend = lex_p + RSTRING_LEN(v);
5601	    ripper_flush(parser);
5602	    lex_lastline = v;
5603	}
5604    }
5605    c = (unsigned char)*lex_p++;
5606    if (c == '\r' && peek('\n')) {
5607	lex_p++;
5608	c = '\n';
5609    }
5610
5611    return c;
5612}
5613
5614static void
5615parser_pushback(struct parser_params *parser, int c)
5616{
5617    if (c == -1) return;
5618    lex_p--;
5619    if (lex_p > lex_pbeg && lex_p[0] == '\n' && lex_p[-1] == '\r') {
5620	lex_p--;
5621    }
5622}
5623
5624#define was_bol() (lex_p == lex_pbeg + 1)
5625
5626#define tokfix() (tokenbuf[tokidx]='\0')
5627#define tok() tokenbuf
5628#define toklen() tokidx
5629#define toklast() (tokidx>0?tokenbuf[tokidx-1]:0)
5630
5631static char*
5632parser_newtok(struct parser_params *parser)
5633{
5634    tokidx = 0;
5635    tokline = ruby_sourceline;
5636    if (!tokenbuf) {
5637	toksiz = 60;
5638	tokenbuf = ALLOC_N(char, 60);
5639    }
5640    if (toksiz > 4096) {
5641	toksiz = 60;
5642	REALLOC_N(tokenbuf, char, 60);
5643    }
5644    return tokenbuf;
5645}
5646
5647static char *
5648parser_tokspace(struct parser_params *parser, int n)
5649{
5650    tokidx += n;
5651
5652    if (tokidx >= toksiz) {
5653	do {toksiz *= 2;} while (toksiz < tokidx);
5654	REALLOC_N(tokenbuf, char, toksiz);
5655    }
5656    return &tokenbuf[tokidx-n];
5657}
5658
5659static void
5660parser_tokadd(struct parser_params *parser, int c)
5661{
5662    tokenbuf[tokidx++] = (char)c;
5663    if (tokidx >= toksiz) {
5664	toksiz *= 2;
5665	REALLOC_N(tokenbuf, char, toksiz);
5666    }
5667}
5668
5669static int
5670parser_tok_hex(struct parser_params *parser, size_t *numlen)
5671{
5672    int c;
5673
5674    c = scan_hex(lex_p, 2, numlen);
5675    if (!*numlen) {
5676	yyerror("invalid hex escape");
5677	return 0;
5678    }
5679    lex_p += *numlen;
5680    return c;
5681}
5682
5683#define tokcopy(n) memcpy(tokspace(n), lex_p - (n), (n))
5684
5685/* return value is for ?\u3042 */
5686static int
5687parser_tokadd_utf8(struct parser_params *parser, rb_encoding **encp,
5688                   int string_literal, int symbol_literal, int regexp_literal)
5689{
5690    /*
5691     * If string_literal is true, then we allow multiple codepoints
5692     * in \u{}, and add the codepoints to the current token.
5693     * Otherwise we're parsing a character literal and return a single
5694     * codepoint without adding it
5695     */
5696
5697    int codepoint;
5698    size_t numlen;
5699
5700    if (regexp_literal) { tokadd('\\'); tokadd('u'); }
5701
5702    if (peek('{')) {  /* handle \u{...} form */
5703	do {
5704            if (regexp_literal) { tokadd(*lex_p); }
5705	    nextc();
5706	    codepoint = scan_hex(lex_p, 6, &numlen);
5707	    if (numlen == 0)  {
5708		yyerror("invalid Unicode escape");
5709		return 0;
5710	    }
5711	    if (codepoint > 0x10ffff) {
5712		yyerror("invalid Unicode codepoint (too large)");
5713		return 0;
5714	    }
5715	    lex_p += numlen;
5716            if (regexp_literal) {
5717                tokcopy((int)numlen);
5718            }
5719            else if (codepoint >= 0x80) {
5720		*encp = rb_utf8_encoding();
5721		if (string_literal) tokaddmbc(codepoint, *encp);
5722	    }
5723	    else if (string_literal) {
5724		tokadd(codepoint);
5725	    }
5726	} while (string_literal && (peek(' ') || peek('\t')));
5727
5728	if (!peek('}')) {
5729	    yyerror("unterminated Unicode escape");
5730	    return 0;
5731	}
5732
5733        if (regexp_literal) { tokadd('}'); }
5734	nextc();
5735    }
5736    else {			/* handle \uxxxx form */
5737	codepoint = scan_hex(lex_p, 4, &numlen);
5738	if (numlen < 4) {
5739	    yyerror("invalid Unicode escape");
5740	    return 0;
5741	}
5742	lex_p += 4;
5743        if (regexp_literal) {
5744            tokcopy(4);
5745        }
5746	else if (codepoint >= 0x80) {
5747	    *encp = rb_utf8_encoding();
5748	    if (string_literal) tokaddmbc(codepoint, *encp);
5749	}
5750	else if (string_literal) {
5751	    tokadd(codepoint);
5752	}
5753    }
5754
5755    return codepoint;
5756}
5757
5758#define ESCAPE_CONTROL 1
5759#define ESCAPE_META    2
5760
5761static int
5762parser_read_escape(struct parser_params *parser, int flags,
5763		   rb_encoding **encp)
5764{
5765    int c;
5766    size_t numlen;
5767
5768    switch (c = nextc()) {
5769      case '\\':	/* Backslash */
5770	return c;
5771
5772      case 'n':	/* newline */
5773	return '\n';
5774
5775      case 't':	/* horizontal tab */
5776	return '\t';
5777
5778      case 'r':	/* carriage-return */
5779	return '\r';
5780
5781      case 'f':	/* form-feed */
5782	return '\f';
5783
5784      case 'v':	/* vertical tab */
5785	return '\13';
5786
5787      case 'a':	/* alarm(bell) */
5788	return '\007';
5789
5790      case 'e':	/* escape */
5791	return 033;
5792
5793      case '0': case '1': case '2': case '3': /* octal constant */
5794      case '4': case '5': case '6': case '7':
5795	pushback(c);
5796	c = scan_oct(lex_p, 3, &numlen);
5797	lex_p += numlen;
5798	return c;
5799
5800      case 'x':	/* hex constant */
5801	c = tok_hex(&numlen);
5802	if (numlen == 0) return 0;
5803	return c;
5804
5805      case 'b':	/* backspace */
5806	return '\010';
5807
5808      case 's':	/* space */
5809	return ' ';
5810
5811      case 'M':
5812	if (flags & ESCAPE_META) goto eof;
5813	if ((c = nextc()) != '-') {
5814	    pushback(c);
5815	    goto eof;
5816	}
5817	if ((c = nextc()) == '\\') {
5818	    if (peek('u')) goto eof;
5819	    return read_escape(flags|ESCAPE_META, encp) | 0x80;
5820	}
5821	else if (c == -1 || !ISASCII(c)) goto eof;
5822	else {
5823	    return ((c & 0xff) | 0x80);
5824	}
5825
5826      case 'C':
5827	if ((c = nextc()) != '-') {
5828	    pushback(c);
5829	    goto eof;
5830	}
5831      case 'c':
5832	if (flags & ESCAPE_CONTROL) goto eof;
5833	if ((c = nextc())== '\\') {
5834	    if (peek('u')) goto eof;
5835	    c = read_escape(flags|ESCAPE_CONTROL, encp);
5836	}
5837	else if (c == '?')
5838	    return 0177;
5839	else if (c == -1 || !ISASCII(c)) goto eof;
5840	return c & 0x9f;
5841
5842      eof:
5843      case -1:
5844        yyerror("Invalid escape character syntax");
5845	return '\0';
5846
5847      default:
5848	return c;
5849    }
5850}
5851
5852static void
5853parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc)
5854{
5855    int len = rb_enc_codelen(c, enc);
5856    rb_enc_mbcput(c, tokspace(len), enc);
5857}
5858
5859static int
5860parser_tokadd_escape(struct parser_params *parser, rb_encoding **encp)
5861{
5862    int c;
5863    int flags = 0;
5864    size_t numlen;
5865
5866  first:
5867    switch (c = nextc()) {
5868      case '\n':
5869	return 0;		/* just ignore */
5870
5871      case '0': case '1': case '2': case '3': /* octal constant */
5872      case '4': case '5': case '6': case '7':
5873	{
5874	    ruby_scan_oct(--lex_p, 3, &numlen);
5875	    if (numlen == 0) goto eof;
5876	    lex_p += numlen;
5877	    tokcopy((int)numlen + 1);
5878	}
5879	return 0;
5880
5881      case 'x':	/* hex constant */
5882	{
5883	    tok_hex(&numlen);
5884	    if (numlen == 0) return -1;
5885	    tokcopy((int)numlen + 2);
5886	}
5887	return 0;
5888
5889      case 'M':
5890	if (flags & ESCAPE_META) goto eof;
5891	if ((c = nextc()) != '-') {
5892	    pushback(c);
5893	    goto eof;
5894	}
5895	tokcopy(3);
5896	flags |= ESCAPE_META;
5897	goto escaped;
5898
5899      case 'C':
5900	if (flags & ESCAPE_CONTROL) goto eof;
5901	if ((c = nextc()) != '-') {
5902	    pushback(c);
5903	    goto eof;
5904	}
5905	tokcopy(3);
5906	goto escaped;
5907
5908      case 'c':
5909	if (flags & ESCAPE_CONTROL) goto eof;
5910	tokcopy(2);
5911	flags |= ESCAPE_CONTROL;
5912      escaped:
5913	if ((c = nextc()) == '\\') {
5914	    goto first;
5915	}
5916	else if (c == -1) goto eof;
5917	tokadd(c);
5918	return 0;
5919
5920      eof:
5921      case -1:
5922        yyerror("Invalid escape character syntax");
5923	return -1;
5924
5925      default:
5926        tokadd('\\');
5927	tokadd(c);
5928    }
5929    return 0;
5930}
5931
5932static int
5933parser_regx_options(struct parser_params *parser)
5934{
5935    int kcode = 0;
5936    int kopt = 0;
5937    int options = 0;
5938    int c, opt, kc;
5939
5940    newtok();
5941    while (c = nextc(), ISALPHA(c)) {
5942        if (c == 'o') {
5943            options |= RE_OPTION_ONCE;
5944        }
5945        else if (rb_char_to_option_kcode(c, &opt, &kc)) {
5946	    if (kc >= 0) {
5947		if (kc != rb_ascii8bit_encindex()) kcode = c;
5948		kopt = opt;
5949	    }
5950	    else {
5951		options |= opt;
5952	    }
5953        }
5954        else {
5955	    tokadd(c);
5956        }
5957    }
5958    options |= kopt;
5959    pushback(c);
5960    if (toklen()) {
5961	tokfix();
5962	compile_error(PARSER_ARG "unknown regexp option%s - %s",
5963		      toklen() > 1 ? "s" : "", tok());
5964    }
5965    return options | RE_OPTION_ENCODING(kcode);
5966}
5967
5968static void
5969dispose_string(VALUE str)
5970{
5971    rb_str_free(str);
5972    rb_gc_force_recycle(str);
5973}
5974
5975static int
5976parser_tokadd_mbchar(struct parser_params *parser, int c)
5977{
5978    int len = parser_precise_mbclen();
5979    if (!MBCLEN_CHARFOUND_P(len)) {
5980	compile_error(PARSER_ARG "invalid multibyte char (%s)", parser_encoding_name());
5981	return -1;
5982    }
5983    tokadd(c);
5984    lex_p += --len;
5985    if (len > 0) tokcopy(len);
5986    return c;
5987}
5988
5989#define tokadd_mbchar(c) parser_tokadd_mbchar(parser, (c))
5990
5991static inline int
5992simple_re_meta(int c)
5993{
5994    switch (c) {
5995      case '$': case '*': case '+': case '.':
5996      case '?': case '^': case '|':
5997      case ')': case ']': case '}': case '>':
5998	return TRUE;
5999      default:
6000	return FALSE;
6001    }
6002}
6003
6004static int
6005parser_tokadd_string(struct parser_params *parser,
6006		     int func, int term, int paren, long *nest,
6007		     rb_encoding **encp)
6008{
6009    int c;
6010    int has_nonascii = 0;
6011    rb_encoding *enc = *encp;
6012    char *errbuf = 0;
6013    static const char mixed_msg[] = "%s mixed within %s source";
6014
6015#define mixed_error(enc1, enc2) if (!errbuf) {	\
6016	size_t len = sizeof(mixed_msg) - 4;	\
6017	len += strlen(rb_enc_name(enc1));	\
6018	len += strlen(rb_enc_name(enc2));	\
6019	errbuf = ALLOCA_N(char, len);		\
6020	snprintf(errbuf, len, mixed_msg,	\
6021		 rb_enc_name(enc1),		\
6022		 rb_enc_name(enc2));		\
6023	yyerror(errbuf);			\
6024    }
6025#define mixed_escape(beg, enc1, enc2) do {	\
6026	const char *pos = lex_p;		\
6027	lex_p = (beg);				\
6028	mixed_error((enc1), (enc2));		\
6029	lex_p = pos;				\
6030    } while (0)
6031
6032    while ((c = nextc()) != -1) {
6033	if (paren && c == paren) {
6034	    ++*nest;
6035	}
6036	else if (c == term) {
6037	    if (!nest || !*nest) {
6038		pushback(c);
6039		break;
6040	    }
6041	    --*nest;
6042	}
6043	else if ((func & STR_FUNC_EXPAND) && c == '#' && lex_p < lex_pend) {
6044	    int c2 = *lex_p;
6045	    if (c2 == '$' || c2 == '@' || c2 == '{') {
6046		pushback(c);
6047		break;
6048	    }
6049	}
6050	else if (c == '\\') {
6051	    const char *beg = lex_p - 1;
6052	    c = nextc();
6053	    switch (c) {
6054	      case '\n':
6055		if (func & STR_FUNC_QWORDS) break;
6056		if (func & STR_FUNC_EXPAND) continue;
6057		tokadd('\\');
6058		break;
6059
6060	      case '\\':
6061		if (func & STR_FUNC_ESCAPE) tokadd(c);
6062		break;
6063
6064	      case 'u':
6065		if ((func & STR_FUNC_EXPAND) == 0) {
6066		    tokadd('\\');
6067		    break;
6068		}
6069		parser_tokadd_utf8(parser, &enc, 1,
6070				   func & STR_FUNC_SYMBOL,
6071                                   func & STR_FUNC_REGEXP);
6072		if (has_nonascii && enc != *encp) {
6073		    mixed_escape(beg, enc, *encp);
6074		}
6075		continue;
6076
6077	      default:
6078		if (c == -1) return -1;
6079		if (!ISASCII(c)) {
6080		    if ((func & STR_FUNC_EXPAND) == 0) tokadd('\\');
6081		    goto non_ascii;
6082		}
6083		if (func & STR_FUNC_REGEXP) {
6084		    if (c == term && !simple_re_meta(c)) {
6085			tokadd(c);
6086			continue;
6087		    }
6088		    pushback(c);
6089		    if ((c = tokadd_escape(&enc)) < 0)
6090			return -1;
6091		    if (has_nonascii && enc != *encp) {
6092			mixed_escape(beg, enc, *encp);
6093		    }
6094		    continue;
6095		}
6096		else if (func & STR_FUNC_EXPAND) {
6097		    pushback(c);
6098		    if (func & STR_FUNC_ESCAPE) tokadd('\\');
6099		    c = read_escape(0, &enc);
6100		}
6101		else if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
6102		    /* ignore backslashed spaces in %w */
6103		}
6104		else if (c != term && !(paren && c == paren)) {
6105		    tokadd('\\');
6106		    pushback(c);
6107		    continue;
6108		}
6109	    }
6110	}
6111	else if (!parser_isascii()) {
6112	  non_ascii:
6113	    has_nonascii = 1;
6114	    if (enc != *encp) {
6115		mixed_error(enc, *encp);
6116		continue;
6117	    }
6118	    if (tokadd_mbchar(c) == -1) return -1;
6119	    continue;
6120	}
6121	else if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
6122	    pushback(c);
6123	    break;
6124	}
6125        if (c & 0x80) {
6126	    has_nonascii = 1;
6127	    if (enc != *encp) {
6128		mixed_error(enc, *encp);
6129		continue;
6130	    }
6131        }
6132	tokadd(c);
6133    }
6134    *encp = enc;
6135    return c;
6136}
6137
6138#define NEW_STRTERM(func, term, paren) \
6139	rb_node_newnode(NODE_STRTERM, (func), (term) | ((paren) << (CHAR_BIT * 2)), 0)
6140
6141#ifdef RIPPER
6142static void
6143ripper_flush_string_content(struct parser_params *parser, rb_encoding *enc)
6144{
6145    if (!NIL_P(parser->delayed)) {
6146	ptrdiff_t len = lex_p - parser->tokp;
6147	if (len > 0) {
6148	    rb_enc_str_buf_cat(parser->delayed, parser->tokp, len, enc);
6149	}
6150	ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
6151	parser->tokp = lex_p;
6152    }
6153}
6154
6155#define flush_string_content(enc) ripper_flush_string_content(parser, (enc))
6156#else
6157#define flush_string_content(enc) ((void)(enc))
6158#endif
6159
6160RUBY_FUNC_EXPORTED const unsigned int ruby_global_name_punct_bits[(0x7e - 0x20 + 31) / 32];
6161/* this can be shared with ripper, since it's independent from struct
6162 * parser_params. */
6163#ifndef RIPPER
6164#define BIT(c, idx) (((c) / 32 - 1 == idx) ? (1U << ((c) % 32)) : 0)
6165#define SPECIAL_PUNCT(idx) ( \
6166	BIT('~', idx) | BIT('*', idx) | BIT('$', idx) | BIT('?', idx) | \
6167	BIT('!', idx) | BIT('@', idx) | BIT('/', idx) | BIT('\\', idx) | \
6168	BIT(';', idx) | BIT(',', idx) | BIT('.', idx) | BIT('=', idx) | \
6169	BIT(':', idx) | BIT('<', idx) | BIT('>', idx) | BIT('\"', idx) | \
6170	BIT('&', idx) | BIT('`', idx) | BIT('\'', idx) | BIT('+', idx) | \
6171	BIT('0', idx))
6172const unsigned int ruby_global_name_punct_bits[] = {
6173    SPECIAL_PUNCT(0),
6174    SPECIAL_PUNCT(1),
6175    SPECIAL_PUNCT(2),
6176};
6177#undef BIT
6178#undef SPECIAL_PUNCT
6179#endif
6180
6181static inline int
6182is_global_name_punct(const char c)
6183{
6184    if (c <= 0x20 || 0x7e < c) return 0;
6185    return (ruby_global_name_punct_bits[(c - 0x20) / 32] >> (c % 32)) & 1;
6186}
6187
6188static int
6189parser_peek_variable_name(struct parser_params *parser)
6190{
6191    int c;
6192    const char *p = lex_p;
6193
6194    if (p + 1 >= lex_pend) return 0;
6195    c = *p++;
6196    switch (c) {
6197      case '$':
6198	if ((c = *p) == '-') {
6199	    if (++p >= lex_pend) return 0;
6200	    c = *p;
6201	}
6202	else if (is_global_name_punct(c) || ISDIGIT(c)) {
6203	    return tSTRING_DVAR;
6204	}
6205	break;
6206      case '@':
6207	if ((c = *p) == '@') {
6208	    if (++p >= lex_pend) return 0;
6209	    c = *p;
6210	}
6211	break;
6212      case '{':
6213	lex_p = p;
6214	command_start = TRUE;
6215	return tSTRING_DBEG;
6216      default:
6217	return 0;
6218    }
6219    if (!ISASCII(c) || c == '_' || ISALPHA(c))
6220	return tSTRING_DVAR;
6221    return 0;
6222}
6223
6224static int
6225parser_parse_string(struct parser_params *parser, NODE *quote)
6226{
6227    int func = (int)quote->nd_func;
6228    int term = nd_term(quote);
6229    int paren = nd_paren(quote);
6230    int c, space = 0;
6231    rb_encoding *enc = current_enc;
6232
6233    if (func == -1) return tSTRING_END;
6234    c = nextc();
6235    if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
6236	do {c = nextc();} while (ISSPACE(c));
6237	space = 1;
6238    }
6239    if (c == term && !quote->nd_nest) {
6240	if (func & STR_FUNC_QWORDS) {
6241	    quote->nd_func = -1;
6242	    return ' ';
6243	}
6244	if (!(func & STR_FUNC_REGEXP)) return tSTRING_END;
6245        set_yylval_num(regx_options());
6246	return tREGEXP_END;
6247    }
6248    if (space) {
6249	pushback(c);
6250	return ' ';
6251    }
6252    newtok();
6253    if ((func & STR_FUNC_EXPAND) && c == '#') {
6254	int t = parser_peek_variable_name(parser);
6255	if (t) return t;
6256	tokadd('#');
6257	c = nextc();
6258    }
6259    pushback(c);
6260    if (tokadd_string(func, term, paren, &quote->nd_nest,
6261		      &enc) == -1) {
6262	ruby_sourceline = nd_line(quote);
6263	if (func & STR_FUNC_REGEXP) {
6264	    if (parser->eofp)
6265		compile_error(PARSER_ARG "unterminated regexp meets end of file");
6266	    return tREGEXP_END;
6267	}
6268	else {
6269	    if (parser->eofp)
6270		compile_error(PARSER_ARG "unterminated string meets end of file");
6271	    return tSTRING_END;
6272	}
6273    }
6274
6275    tokfix();
6276    set_yylval_str(STR_NEW3(tok(), toklen(), enc, func));
6277    flush_string_content(enc);
6278
6279    return tSTRING_CONTENT;
6280}
6281
6282static int
6283parser_heredoc_identifier(struct parser_params *parser)
6284{
6285    int c = nextc(), term, func = 0;
6286    long len;
6287
6288    if (c == '-') {
6289	c = nextc();
6290	func = STR_FUNC_INDENT;
6291    }
6292    switch (c) {
6293      case '\'':
6294	func |= str_squote; goto quoted;
6295      case '"':
6296	func |= str_dquote; goto quoted;
6297      case '`':
6298	func |= str_xquote;
6299      quoted:
6300	newtok();
6301	tokadd(func);
6302	term = c;
6303	while ((c = nextc()) != -1 && c != term) {
6304	    if (tokadd_mbchar(c) == -1) return 0;
6305	}
6306	if (c == -1) {
6307	    compile_error(PARSER_ARG "unterminated here document identifier");
6308	    return 0;
6309	}
6310	break;
6311
6312      default:
6313	if (!parser_is_identchar()) {
6314	    pushback(c);
6315	    if (func & STR_FUNC_INDENT) {
6316		pushback('-');
6317	    }
6318	    return 0;
6319	}
6320	newtok();
6321	term = '"';
6322	tokadd(func |= str_dquote);
6323	do {
6324	    if (tokadd_mbchar(c) == -1) return 0;
6325	} while ((c = nextc()) != -1 && parser_is_identchar());
6326	pushback(c);
6327	break;
6328    }
6329
6330    tokfix();
6331#ifdef RIPPER
6332    ripper_dispatch_scan_event(parser, tHEREDOC_BEG);
6333#endif
6334    len = lex_p - lex_pbeg;
6335    lex_goto_eol(parser);
6336    lex_strterm = rb_node_newnode(NODE_HEREDOC,
6337				  STR_NEW(tok(), toklen()),	/* nd_lit */
6338				  len,				/* nd_nth */
6339				  lex_lastline);		/* nd_orig */
6340    nd_set_line(lex_strterm, ruby_sourceline);
6341    ripper_flush(parser);
6342    return term == '`' ? tXSTRING_BEG : tSTRING_BEG;
6343}
6344
6345static void
6346parser_heredoc_restore(struct parser_params *parser, NODE *here)
6347{
6348    VALUE line;
6349
6350    line = here->nd_orig;
6351    lex_lastline = line;
6352    lex_pbeg = RSTRING_PTR(line);
6353    lex_pend = lex_pbeg + RSTRING_LEN(line);
6354    lex_p = lex_pbeg + here->nd_nth;
6355    heredoc_end = ruby_sourceline;
6356    ruby_sourceline = nd_line(here);
6357    dispose_string(here->nd_lit);
6358    rb_gc_force_recycle((VALUE)here);
6359    ripper_flush(parser);
6360}
6361
6362static int
6363parser_whole_match_p(struct parser_params *parser,
6364    const char *eos, long len, int indent)
6365{
6366    const char *p = lex_pbeg;
6367    long n;
6368
6369    if (indent) {
6370	while (*p && ISSPACE(*p)) p++;
6371    }
6372    n = lex_pend - (p + len);
6373    if (n < 0 || (n > 0 && p[len] != '\n' && p[len] != '\r')) return FALSE;
6374    return strncmp(eos, p, len) == 0;
6375}
6376
6377#ifdef RIPPER
6378static void
6379ripper_dispatch_heredoc_end(struct parser_params *parser)
6380{
6381    if (!NIL_P(parser->delayed))
6382	ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
6383    lex_goto_eol(parser);
6384    ripper_dispatch_ignored_scan_event(parser, tHEREDOC_END);
6385}
6386
6387#define dispatch_heredoc_end() ripper_dispatch_heredoc_end(parser)
6388#else
6389#define dispatch_heredoc_end() ((void)0)
6390#endif
6391
6392static int
6393parser_here_document(struct parser_params *parser, NODE *here)
6394{
6395    int c, func, indent = 0;
6396    const char *eos, *p, *pend;
6397    long len;
6398    VALUE str = 0;
6399    rb_encoding *enc = current_enc;
6400
6401    eos = RSTRING_PTR(here->nd_lit);
6402    len = RSTRING_LEN(here->nd_lit) - 1;
6403    indent = (func = *eos++) & STR_FUNC_INDENT;
6404
6405    if ((c = nextc()) == -1) {
6406      error:
6407	compile_error(PARSER_ARG "can't find string \"%s\" anywhere before EOF", eos);
6408#ifdef RIPPER
6409	if (NIL_P(parser->delayed)) {
6410	    ripper_dispatch_scan_event(parser, tSTRING_CONTENT);
6411	}
6412	else {
6413	    if (str ||
6414		((len = lex_p - parser->tokp) > 0 &&
6415		 (str = STR_NEW3(parser->tokp, len, enc, func), 1))) {
6416		rb_str_append(parser->delayed, str);
6417	    }
6418	    ripper_dispatch_delayed_token(parser, tSTRING_CONTENT);
6419	}
6420	lex_goto_eol(parser);
6421#endif
6422      restore:
6423	heredoc_restore(lex_strterm);
6424	lex_strterm = 0;
6425	return 0;
6426    }
6427    if (was_bol() && whole_match_p(eos, len, indent)) {
6428	dispatch_heredoc_end();
6429	heredoc_restore(lex_strterm);
6430	return tSTRING_END;
6431    }
6432
6433    if (!(func & STR_FUNC_EXPAND)) {
6434	do {
6435	    p = RSTRING_PTR(lex_lastline);
6436	    pend = lex_pend;
6437	    if (pend > p) {
6438		switch (pend[-1]) {
6439		  case '\n':
6440		    if (--pend == p || pend[-1] != '\r') {
6441			pend++;
6442			break;
6443		    }
6444		  case '\r':
6445		    --pend;
6446		}
6447	    }
6448	    if (str)
6449		rb_str_cat(str, p, pend - p);
6450	    else
6451		str = STR_NEW(p, pend - p);
6452	    if (pend < lex_pend) rb_str_cat(str, "\n", 1);
6453	    lex_goto_eol(parser);
6454	    if (nextc() == -1) {
6455		if (str) dispose_string(str);
6456		goto error;
6457	    }
6458	} while (!whole_match_p(eos, len, indent));
6459    }
6460    else {
6461	/*	int mb = ENC_CODERANGE_7BIT, *mbp = &mb;*/
6462	newtok();
6463	if (c == '#') {
6464	    int t = parser_peek_variable_name(parser);
6465	    if (t) return t;
6466	    tokadd('#');
6467	    c = nextc();
6468	}
6469	do {
6470	    pushback(c);
6471	    if ((c = tokadd_string(func, '\n', 0, NULL, &enc)) == -1) {
6472		if (parser->eofp) goto error;
6473		goto restore;
6474	    }
6475	    if (c != '\n') {
6476		set_yylval_str(STR_NEW3(tok(), toklen(), enc, func));
6477		flush_string_content(enc);
6478		return tSTRING_CONTENT;
6479	    }
6480	    tokadd(nextc());
6481	    /*	    if (mbp && mb == ENC_CODERANGE_UNKNOWN) mbp = 0;*/
6482	    if ((c = nextc()) == -1) goto error;
6483	} while (!whole_match_p(eos, len, indent));
6484	str = STR_NEW3(tok(), toklen(), enc, func);
6485    }
6486    dispatch_heredoc_end();
6487    heredoc_restore(lex_strterm);
6488    lex_strterm = NEW_STRTERM(-1, 0, 0);
6489    set_yylval_str(str);
6490    return tSTRING_CONTENT;
6491}
6492
6493#include "lex.c"
6494
6495static void
6496arg_ambiguous_gen(struct parser_params *parser)
6497{
6498#ifndef RIPPER
6499    rb_warning0("ambiguous first argument; put parentheses or even spaces");
6500#else
6501    dispatch0(arg_ambiguous);
6502#endif
6503}
6504#define arg_ambiguous() (arg_ambiguous_gen(parser), 1)
6505
6506static ID
6507formal_argument_gen(struct parser_params *parser, ID lhs)
6508{
6509#ifndef RIPPER
6510    if (!is_local_id(lhs))
6511	yyerror("formal argument must be local variable");
6512#endif
6513    shadowing_lvar(lhs);
6514    return lhs;
6515}
6516
6517static int
6518lvar_defined_gen(struct parser_params *parser, ID id)
6519{
6520    return (dyna_in_block() && dvar_defined_get(id)) || local_id(id);
6521}
6522
6523/* emacsen -*- hack */
6524static long
6525parser_encode_length(struct parser_params *parser, const char *name, long len)
6526{
6527    long nlen;
6528
6529    if (len > 5 && name[nlen = len - 5] == '-') {
6530	if (rb_memcicmp(name + nlen + 1, "unix", 4) == 0)
6531	    return nlen;
6532    }
6533    if (len > 4 && name[nlen = len - 4] == '-') {
6534	if (rb_memcicmp(name + nlen + 1, "dos", 3) == 0)
6535	    return nlen;
6536	if (rb_memcicmp(name + nlen + 1, "mac", 3) == 0 &&
6537	    !(len == 8 && rb_memcicmp(name, "utf8-mac", len) == 0))
6538	    /* exclude UTF8-MAC because the encoding named "UTF8" doesn't exist in Ruby */
6539	    return nlen;
6540    }
6541    return len;
6542}
6543
6544static void
6545parser_set_encode(struct parser_params *parser, const char *name)
6546{
6547    int idx = rb_enc_find_index(name);
6548    rb_encoding *enc;
6549    VALUE excargs[3];
6550
6551    if (idx < 0) {
6552	excargs[1] = rb_sprintf("unknown encoding name: %s", name);
6553      error:
6554	excargs[0] = rb_eArgError;
6555	excargs[2] = rb_make_backtrace();
6556	rb_ary_unshift(excargs[2], rb_sprintf("%s:%d", ruby_sourcefile, ruby_sourceline));
6557	rb_exc_raise(rb_make_exception(3, excargs));
6558    }
6559    enc = rb_enc_from_index(idx);
6560    if (!rb_enc_asciicompat(enc)) {
6561	excargs[1] = rb_sprintf("%s is not ASCII compatible", rb_enc_name(enc));
6562	goto error;
6563    }
6564    parser->enc = enc;
6565#ifndef RIPPER
6566    if (ruby_debug_lines) {
6567	long i, n = RARRAY_LEN(ruby_debug_lines);
6568	const VALUE *p = RARRAY_PTR(ruby_debug_lines);
6569	for (i = 0; i < n; ++i) {
6570	    rb_enc_associate_index(*p, idx);
6571	}
6572    }
6573#endif
6574}
6575
6576static int
6577comment_at_top(struct parser_params *parser)
6578{
6579    const char *p = lex_pbeg, *pend = lex_p - 1;
6580    if (parser->line_count != (parser->has_shebang ? 2 : 1)) return 0;
6581    while (p < pend) {
6582	if (!ISSPACE(*p)) return 0;
6583	p++;
6584    }
6585    return 1;
6586}
6587
6588#ifndef RIPPER
6589typedef long (*rb_magic_comment_length_t)(struct parser_params *parser, const char *name, long len);
6590typedef void (*rb_magic_comment_setter_t)(struct parser_params *parser, const char *name, const char *val);
6591
6592static void
6593magic_comment_encoding(struct parser_params *parser, const char *name, const char *val)
6594{
6595    if (!comment_at_top(parser)) {
6596	return;
6597    }
6598    parser_set_encode(parser, val);
6599}
6600
6601static void
6602parser_set_token_info(struct parser_params *parser, const char *name, const char *val)
6603{
6604    int *p = &parser->parser_token_info_enabled;
6605
6606    switch (*val) {
6607      case 't': case 'T':
6608	if (strcasecmp(val, "true") == 0) {
6609	    *p = TRUE;
6610	    return;
6611	}
6612	break;
6613      case 'f': case 'F':
6614	if (strcasecmp(val, "false") == 0) {
6615	    *p = FALSE;
6616	    return;
6617	}
6618	break;
6619    }
6620    rb_compile_warning(ruby_sourcefile, ruby_sourceline, "invalid value for %s: %s", name, val);
6621}
6622
6623struct magic_comment {
6624    const char *name;
6625    rb_magic_comment_setter_t func;
6626    rb_magic_comment_length_t length;
6627};
6628
6629static const struct magic_comment magic_comments[] = {
6630    {"coding", magic_comment_encoding, parser_encode_length},
6631    {"encoding", magic_comment_encoding, parser_encode_length},
6632    {"warn_indent", parser_set_token_info},
6633};
6634#endif
6635
6636static const char *
6637magic_comment_marker(const char *str, long len)
6638{
6639    long i = 2;
6640
6641    while (i < len) {
6642	switch (str[i]) {
6643	  case '-':
6644	    if (str[i-1] == '*' && str[i-2] == '-') {
6645		return str + i + 1;
6646	    }
6647	    i += 2;
6648	    break;
6649	  case '*':
6650	    if (i + 1 >= len) return 0;
6651	    if (str[i+1] != '-') {
6652		i += 4;
6653	    }
6654	    else if (str[i-1] != '-') {
6655		i += 2;
6656	    }
6657	    else {
6658		return str + i + 2;
6659	    }
6660	    break;
6661	  default:
6662	    i += 3;
6663	    break;
6664	}
6665    }
6666    return 0;
6667}
6668
6669static int
6670parser_magic_comment(struct parser_params *parser, const char *str, long len)
6671{
6672    VALUE name = 0, val = 0;
6673    const char *beg, *end, *vbeg, *vend;
6674#define str_copy(_s, _p, _n) ((_s) \
6675	? (void)(rb_str_resize((_s), (_n)), \
6676	   MEMCPY(RSTRING_PTR(_s), (_p), char, (_n)), (_s)) \
6677	: (void)((_s) = STR_NEW((_p), (_n))))
6678
6679    if (len <= 7) return FALSE;
6680    if (!(beg = magic_comment_marker(str, len))) return FALSE;
6681    if (!(end = magic_comment_marker(beg, str + len - beg))) return FALSE;
6682    str = beg;
6683    len = end - beg - 3;
6684
6685    /* %r"([^\\s\'\":;]+)\\s*:\\s*(\"(?:\\\\.|[^\"])*\"|[^\"\\s;]+)[\\s;]*" */
6686    while (len > 0) {
6687#ifndef RIPPER
6688	const struct magic_comment *p = magic_comments;
6689#endif
6690	char *s;
6691	int i;
6692	long n = 0;
6693
6694	for (; len > 0 && *str; str++, --len) {
6695	    switch (*str) {
6696	      case '\'': case '"': case ':': case ';':
6697		continue;
6698	    }
6699	    if (!ISSPACE(*str)) break;
6700	}
6701	for (beg = str; len > 0; str++, --len) {
6702	    switch (*str) {
6703	      case '\'': case '"': case ':': case ';':
6704		break;
6705	      default:
6706		if (ISSPACE(*str)) break;
6707		continue;
6708	    }
6709	    break;
6710	}
6711	for (end = str; len > 0 && ISSPACE(*str); str++, --len);
6712	if (!len) break;
6713	if (*str != ':') continue;
6714
6715	do str++; while (--len > 0 && ISSPACE(*str));
6716	if (!len) break;
6717	if (*str == '"') {
6718	    for (vbeg = ++str; --len > 0 && *str != '"'; str++) {
6719		if (*str == '\\') {
6720		    --len;
6721		    ++str;
6722		}
6723	    }
6724	    vend = str;
6725	    if (len) {
6726		--len;
6727		++str;
6728	    }
6729	}
6730	else {
6731	    for (vbeg = str; len > 0 && *str != '"' && *str != ';' && !ISSPACE(*str); --len, str++);
6732	    vend = str;
6733	}
6734	while (len > 0 && (*str == ';' || ISSPACE(*str))) --len, str++;
6735
6736	n = end - beg;
6737	str_copy(name, beg, n);
6738	s = RSTRING_PTR(name);
6739	for (i = 0; i < n; ++i) {
6740	    if (s[i] == '-') s[i] = '_';
6741	}
6742#ifndef RIPPER
6743	do {
6744	    if (STRNCASECMP(p->name, s, n) == 0) {
6745		n = vend - vbeg;
6746		if (p->length) {
6747		    n = (*p->length)(parser, vbeg, n);
6748		}
6749		str_copy(val, vbeg, n);
6750		(*p->func)(parser, s, RSTRING_PTR(val));
6751		break;
6752	    }
6753	} while (++p < magic_comments + numberof(magic_comments));
6754#else
6755	str_copy(val, vbeg, vend - vbeg);
6756	dispatch2(magic_comment, name, val);
6757#endif
6758    }
6759
6760    return TRUE;
6761}
6762
6763static void
6764set_file_encoding(struct parser_params *parser, const char *str, const char *send)
6765{
6766    int sep = 0;
6767    const char *beg = str;
6768    VALUE s;
6769
6770    for (;;) {
6771	if (send - str <= 6) return;
6772	switch (str[6]) {
6773	  case 'C': case 'c': str += 6; continue;
6774	  case 'O': case 'o': str += 5; continue;
6775	  case 'D': case 'd': str += 4; continue;
6776	  case 'I': case 'i': str += 3; continue;
6777	  case 'N': case 'n': str += 2; continue;
6778	  case 'G': case 'g': str += 1; continue;
6779	  case '=': case ':':
6780	    sep = 1;
6781	    str += 6;
6782	    break;
6783	  default:
6784	    str += 6;
6785	    if (ISSPACE(*str)) break;
6786	    continue;
6787	}
6788	if (STRNCASECMP(str-6, "coding", 6) == 0) break;
6789    }
6790    for (;;) {
6791	do {
6792	    if (++str >= send) return;
6793	} while (ISSPACE(*str));
6794	if (sep) break;
6795	if (*str != '=' && *str != ':') return;
6796	sep = 1;
6797	str++;
6798    }
6799    beg = str;
6800    while ((*str == '-' || *str == '_' || ISALNUM(*str)) && ++str < send);
6801    s = rb_str_new(beg, parser_encode_length(parser, beg, str - beg));
6802    parser_set_encode(parser, RSTRING_PTR(s));
6803    rb_str_resize(s, 0);
6804}
6805
6806static void
6807parser_prepare(struct parser_params *parser)
6808{
6809    int c = nextc();
6810    switch (c) {
6811      case '#':
6812	if (peek('!')) parser->has_shebang = 1;
6813	break;
6814      case 0xef:		/* UTF-8 BOM marker */
6815	if (lex_pend - lex_p >= 2 &&
6816	    (unsigned char)lex_p[0] == 0xbb &&
6817	    (unsigned char)lex_p[1] == 0xbf) {
6818	    parser->enc = rb_utf8_encoding();
6819	    lex_p += 2;
6820	    lex_pbeg = lex_p;
6821	    return;
6822	}
6823	break;
6824      case EOF:
6825	return;
6826    }
6827    pushback(c);
6828    parser->enc = rb_enc_get(lex_lastline);
6829}
6830
6831#define IS_ARG() IS_lex_state(EXPR_ARG_ANY)
6832#define IS_END() IS_lex_state(EXPR_END_ANY)
6833#define IS_BEG() IS_lex_state(EXPR_BEG_ANY)
6834#define IS_SPCARG(c) (IS_ARG() && space_seen && !ISSPACE(c))
6835#define IS_LABEL_POSSIBLE() ((IS_lex_state(EXPR_BEG | EXPR_ENDFN) && !cmd_state) || IS_ARG())
6836#define IS_LABEL_SUFFIX(n) (peek_n(':',(n)) && !peek_n(':', (n)+1))
6837#define IS_AFTER_OPERATOR() IS_lex_state(EXPR_FNAME | EXPR_DOT)
6838
6839#ifndef RIPPER
6840#define ambiguous_operator(op, syn) ( \
6841    rb_warning0("`"op"' after local variable is interpreted as binary operator"), \
6842    rb_warning0("even though it seems like "syn""))
6843#else
6844#define ambiguous_operator(op, syn) dispatch2(operator_ambiguous, ripper_intern(op), rb_str_new_cstr(syn))
6845#endif
6846#define warn_balanced(op, syn) ((void) \
6847    (!IS_lex_state_for(last_state, EXPR_CLASS|EXPR_DOT|EXPR_FNAME|EXPR_ENDFN|EXPR_ENDARG) && \
6848     space_seen && !ISSPACE(c) && \
6849     (ambiguous_operator(op, syn), 0)))
6850
6851static int
6852parser_yylex(struct parser_params *parser)
6853{
6854    register int c;
6855    int space_seen = 0;
6856    int cmd_state;
6857    enum lex_state_e last_state;
6858    rb_encoding *enc;
6859    int mb;
6860#ifdef RIPPER
6861    int fallthru = FALSE;
6862#endif
6863
6864    if (lex_strterm) {
6865	int token;
6866	if (nd_type(lex_strterm) == NODE_HEREDOC) {
6867	    token = here_document(lex_strterm);
6868	    if (token == tSTRING_END) {
6869		lex_strterm = 0;
6870		lex_state = EXPR_END;
6871	    }
6872	}
6873	else {
6874	    token = parse_string(lex_strterm);
6875	    if (token == tSTRING_END || token == tREGEXP_END) {
6876		rb_gc_force_recycle((VALUE)lex_strterm);
6877		lex_strterm = 0;
6878		lex_state = EXPR_END;
6879	    }
6880	}
6881	return token;
6882    }
6883    cmd_state = command_start;
6884    command_start = FALSE;
6885  retry:
6886    last_state = lex_state;
6887    switch (c = nextc()) {
6888      case '\0':		/* NUL */
6889      case '\004':		/* ^D */
6890      case '\032':		/* ^Z */
6891      case -1:			/* end of script. */
6892	return 0;
6893
6894	/* white spaces */
6895      case ' ': case '\t': case '\f': case '\r':
6896      case '\13': /* '\v' */
6897	space_seen = 1;
6898#ifdef RIPPER
6899	while ((c = nextc())) {
6900	    switch (c) {
6901	      case ' ': case '\t': case '\f': case '\r':
6902	      case '\13': /* '\v' */
6903		break;
6904	      default:
6905		goto outofloop;
6906	    }
6907	}
6908      outofloop:
6909	pushback(c);
6910	ripper_dispatch_scan_event(parser, tSP);
6911#endif
6912	goto retry;
6913
6914      case '#':		/* it's a comment */
6915	/* no magic_comment in shebang line */
6916	if (!parser_magic_comment(parser, lex_p, lex_pend - lex_p)) {
6917	    if (comment_at_top(parser)) {
6918		set_file_encoding(parser, lex_p, lex_pend);
6919	    }
6920	}
6921	lex_p = lex_pend;
6922#ifdef RIPPER
6923        ripper_dispatch_scan_event(parser, tCOMMENT);
6924        fallthru = TRUE;
6925#endif
6926	/* fall through */
6927      case '\n':
6928	if (IS_lex_state(EXPR_BEG | EXPR_VALUE | EXPR_CLASS | EXPR_FNAME | EXPR_DOT)) {
6929#ifdef RIPPER
6930            if (!fallthru) {
6931                ripper_dispatch_scan_event(parser, tIGNORED_NL);
6932            }
6933            fallthru = FALSE;
6934#endif
6935	    goto retry;
6936	}
6937	while ((c = nextc())) {
6938	    switch (c) {
6939	      case ' ': case '\t': case '\f': case '\r':
6940	      case '\13': /* '\v' */
6941		space_seen = 1;
6942		break;
6943	      case '.': {
6944		  if ((c = nextc()) != '.') {
6945		      pushback(c);
6946		      pushback('.');
6947		      goto retry;
6948		  }
6949	      }
6950	      default:
6951		--ruby_sourceline;
6952		lex_nextline = lex_lastline;
6953	      case -1:		/* EOF no decrement*/
6954		lex_goto_eol(parser);
6955#ifdef RIPPER
6956		if (c != -1) {
6957		    parser->tokp = lex_p;
6958		}
6959#endif
6960		goto normal_newline;
6961	    }
6962	}
6963      normal_newline:
6964	command_start = TRUE;
6965	lex_state = EXPR_BEG;
6966	return '\n';
6967
6968      case '*':
6969	if ((c = nextc()) == '*') {
6970	    if ((c = nextc()) == '=') {
6971                set_yylval_id(tPOW);
6972		lex_state = EXPR_BEG;
6973		return tOP_ASGN;
6974	    }
6975	    pushback(c);
6976	    if (IS_SPCARG(c)) {
6977		rb_warning0("`**' interpreted as argument prefix");
6978		c = tDSTAR;
6979	    }
6980	    else if (IS_BEG()) {
6981		c = tDSTAR;
6982	    }
6983	    else {
6984		warn_balanced("**", "argument prefix");
6985		c = tPOW;
6986	    }
6987	}
6988	else {
6989	    if (c == '=') {
6990                set_yylval_id('*');
6991		lex_state = EXPR_BEG;
6992		return tOP_ASGN;
6993	    }
6994	    pushback(c);
6995	    if (IS_SPCARG(c)) {
6996		rb_warning0("`*' interpreted as argument prefix");
6997		c = tSTAR;
6998	    }
6999	    else if (IS_BEG()) {
7000		c = tSTAR;
7001	    }
7002	    else {
7003		warn_balanced("*", "argument prefix");
7004		c = '*';
7005	    }
7006	}
7007	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7008	return c;
7009
7010      case '!':
7011	c = nextc();
7012	if (IS_AFTER_OPERATOR()) {
7013	    lex_state = EXPR_ARG;
7014	    if (c == '@') {
7015		return '!';
7016	    }
7017	}
7018	else {
7019	    lex_state = EXPR_BEG;
7020	}
7021	if (c == '=') {
7022	    return tNEQ;
7023	}
7024	if (c == '~') {
7025	    return tNMATCH;
7026	}
7027	pushback(c);
7028	return '!';
7029
7030      case '=':
7031	if (was_bol()) {
7032	    /* skip embedded rd document */
7033	    if (strncmp(lex_p, "begin", 5) == 0 && ISSPACE(lex_p[5])) {
7034#ifdef RIPPER
7035                int first_p = TRUE;
7036
7037                lex_goto_eol(parser);
7038                ripper_dispatch_scan_event(parser, tEMBDOC_BEG);
7039#endif
7040		for (;;) {
7041		    lex_goto_eol(parser);
7042#ifdef RIPPER
7043                    if (!first_p) {
7044                        ripper_dispatch_scan_event(parser, tEMBDOC);
7045                    }
7046                    first_p = FALSE;
7047#endif
7048		    c = nextc();
7049		    if (c == -1) {
7050			compile_error(PARSER_ARG "embedded document meets end of file");
7051			return 0;
7052		    }
7053		    if (c != '=') continue;
7054		    if (strncmp(lex_p, "end", 3) == 0 &&
7055			(lex_p + 3 == lex_pend || ISSPACE(lex_p[3]))) {
7056			break;
7057		    }
7058		}
7059		lex_goto_eol(parser);
7060#ifdef RIPPER
7061                ripper_dispatch_scan_event(parser, tEMBDOC_END);
7062#endif
7063		goto retry;
7064	    }
7065	}
7066
7067	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7068	if ((c = nextc()) == '=') {
7069	    if ((c = nextc()) == '=') {
7070		return tEQQ;
7071	    }
7072	    pushback(c);
7073	    return tEQ;
7074	}
7075	if (c == '~') {
7076	    return tMATCH;
7077	}
7078	else if (c == '>') {
7079	    return tASSOC;
7080	}
7081	pushback(c);
7082	return '=';
7083
7084      case '<':
7085	last_state = lex_state;
7086	c = nextc();
7087	if (c == '<' &&
7088	    !IS_lex_state(EXPR_DOT | EXPR_CLASS) &&
7089	    !IS_END() &&
7090	    (!IS_ARG() || space_seen)) {
7091	    int token = heredoc_identifier();
7092	    if (token) return token;
7093	}
7094	if (IS_AFTER_OPERATOR()) {
7095	    lex_state = EXPR_ARG;
7096	}
7097	else {
7098	    if (IS_lex_state(EXPR_CLASS))
7099		command_start = TRUE;
7100	    lex_state = EXPR_BEG;
7101	}
7102	if (c == '=') {
7103	    if ((c = nextc()) == '>') {
7104		return tCMP;
7105	    }
7106	    pushback(c);
7107	    return tLEQ;
7108	}
7109	if (c == '<') {
7110	    if ((c = nextc()) == '=') {
7111                set_yylval_id(tLSHFT);
7112		lex_state = EXPR_BEG;
7113		return tOP_ASGN;
7114	    }
7115	    pushback(c);
7116	    warn_balanced("<<", "here document");
7117	    return tLSHFT;
7118	}
7119	pushback(c);
7120	return '<';
7121
7122      case '>':
7123	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7124	if ((c = nextc()) == '=') {
7125	    return tGEQ;
7126	}
7127	if (c == '>') {
7128	    if ((c = nextc()) == '=') {
7129                set_yylval_id(tRSHFT);
7130		lex_state = EXPR_BEG;
7131		return tOP_ASGN;
7132	    }
7133	    pushback(c);
7134	    return tRSHFT;
7135	}
7136	pushback(c);
7137	return '>';
7138
7139      case '"':
7140	lex_strterm = NEW_STRTERM(str_dquote, '"', 0);
7141	return tSTRING_BEG;
7142
7143      case '`':
7144	if (IS_lex_state(EXPR_FNAME)) {
7145	    lex_state = EXPR_ENDFN;
7146	    return c;
7147	}
7148	if (IS_lex_state(EXPR_DOT)) {
7149	    if (cmd_state)
7150		lex_state = EXPR_CMDARG;
7151	    else
7152		lex_state = EXPR_ARG;
7153	    return c;
7154	}
7155	lex_strterm = NEW_STRTERM(str_xquote, '`', 0);
7156	return tXSTRING_BEG;
7157
7158      case '\'':
7159	lex_strterm = NEW_STRTERM(str_squote, '\'', 0);
7160	return tSTRING_BEG;
7161
7162      case '?':
7163	if (IS_END()) {
7164	    lex_state = EXPR_VALUE;
7165	    return '?';
7166	}
7167	c = nextc();
7168	if (c == -1) {
7169	    compile_error(PARSER_ARG "incomplete character syntax");
7170	    return 0;
7171	}
7172	if (rb_enc_isspace(c, current_enc)) {
7173	    if (!IS_ARG()) {
7174		int c2 = 0;
7175		switch (c) {
7176		  case ' ':
7177		    c2 = 's';
7178		    break;
7179		  case '\n':
7180		    c2 = 'n';
7181		    break;
7182		  case '\t':
7183		    c2 = 't';
7184		    break;
7185		  case '\v':
7186		    c2 = 'v';
7187		    break;
7188		  case '\r':
7189		    c2 = 'r';
7190		    break;
7191		  case '\f':
7192		    c2 = 'f';
7193		    break;
7194		}
7195		if (c2) {
7196		    rb_warnI("invalid character syntax; use ?\\%c", c2);
7197		}
7198	    }
7199	  ternary:
7200	    pushback(c);
7201	    lex_state = EXPR_VALUE;
7202	    return '?';
7203	}
7204	newtok();
7205	enc = current_enc;
7206	if (!parser_isascii()) {
7207	    if (tokadd_mbchar(c) == -1) return 0;
7208	}
7209	else if ((rb_enc_isalnum(c, current_enc) || c == '_') &&
7210		 lex_p < lex_pend && is_identchar(lex_p, lex_pend, current_enc)) {
7211	    goto ternary;
7212	}
7213        else if (c == '\\') {
7214            if (peek('u')) {
7215                nextc();
7216                c = parser_tokadd_utf8(parser, &enc, 0, 0, 0);
7217                if (0x80 <= c) {
7218                    tokaddmbc(c, enc);
7219                }
7220                else {
7221                    tokadd(c);
7222                }
7223            }
7224            else if (!lex_eol_p() && !(c = *lex_p, ISASCII(c))) {
7225                nextc();
7226                if (tokadd_mbchar(c) == -1) return 0;
7227            }
7228            else {
7229                c = read_escape(0, &enc);
7230                tokadd(c);
7231            }
7232        }
7233        else {
7234	    tokadd(c);
7235        }
7236	tokfix();
7237	set_yylval_str(STR_NEW3(tok(), toklen(), enc, 0));
7238	lex_state = EXPR_END;
7239	return tCHAR;
7240
7241      case '&':
7242	if ((c = nextc()) == '&') {
7243	    lex_state = EXPR_BEG;
7244	    if ((c = nextc()) == '=') {
7245                set_yylval_id(tANDOP);
7246		lex_state = EXPR_BEG;
7247		return tOP_ASGN;
7248	    }
7249	    pushback(c);
7250	    return tANDOP;
7251	}
7252	else if (c == '=') {
7253            set_yylval_id('&');
7254	    lex_state = EXPR_BEG;
7255	    return tOP_ASGN;
7256	}
7257	pushback(c);
7258	if (IS_SPCARG(c)) {
7259	    rb_warning0("`&' interpreted as argument prefix");
7260	    c = tAMPER;
7261	}
7262	else if (IS_BEG()) {
7263	    c = tAMPER;
7264	}
7265	else {
7266	    warn_balanced("&", "argument prefix");
7267	    c = '&';
7268	}
7269	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7270	return c;
7271
7272      case '|':
7273	if ((c = nextc()) == '|') {
7274	    lex_state = EXPR_BEG;
7275	    if ((c = nextc()) == '=') {
7276                set_yylval_id(tOROP);
7277		lex_state = EXPR_BEG;
7278		return tOP_ASGN;
7279	    }
7280	    pushback(c);
7281	    return tOROP;
7282	}
7283	if (c == '=') {
7284            set_yylval_id('|');
7285	    lex_state = EXPR_BEG;
7286	    return tOP_ASGN;
7287	}
7288	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7289	pushback(c);
7290	return '|';
7291
7292      case '+':
7293	c = nextc();
7294	if (IS_AFTER_OPERATOR()) {
7295	    lex_state = EXPR_ARG;
7296	    if (c == '@') {
7297		return tUPLUS;
7298	    }
7299	    pushback(c);
7300	    return '+';
7301	}
7302	if (c == '=') {
7303            set_yylval_id('+');
7304	    lex_state = EXPR_BEG;
7305	    return tOP_ASGN;
7306	}
7307	if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous())) {
7308	    lex_state = EXPR_BEG;
7309	    pushback(c);
7310	    if (c != -1 && ISDIGIT(c)) {
7311		c = '+';
7312		goto start_num;
7313	    }
7314	    return tUPLUS;
7315	}
7316	lex_state = EXPR_BEG;
7317	pushback(c);
7318	warn_balanced("+", "unary operator");
7319	return '+';
7320
7321      case '-':
7322	c = nextc();
7323	if (IS_AFTER_OPERATOR()) {
7324	    lex_state = EXPR_ARG;
7325	    if (c == '@') {
7326		return tUMINUS;
7327	    }
7328	    pushback(c);
7329	    return '-';
7330	}
7331	if (c == '=') {
7332            set_yylval_id('-');
7333	    lex_state = EXPR_BEG;
7334	    return tOP_ASGN;
7335	}
7336	if (c == '>') {
7337	    lex_state = EXPR_ENDFN;
7338	    return tLAMBDA;
7339	}
7340	if (IS_BEG() || (IS_SPCARG(c) && arg_ambiguous())) {
7341	    lex_state = EXPR_BEG;
7342	    pushback(c);
7343	    if (c != -1 && ISDIGIT(c)) {
7344		return tUMINUS_NUM;
7345	    }
7346	    return tUMINUS;
7347	}
7348	lex_state = EXPR_BEG;
7349	pushback(c);
7350	warn_balanced("-", "unary operator");
7351	return '-';
7352
7353      case '.':
7354	lex_state = EXPR_BEG;
7355	if ((c = nextc()) == '.') {
7356	    if ((c = nextc()) == '.') {
7357		return tDOT3;
7358	    }
7359	    pushback(c);
7360	    return tDOT2;
7361	}
7362	pushback(c);
7363	if (c != -1 && ISDIGIT(c)) {
7364	    yyerror("no .<digit> floating literal anymore; put 0 before dot");
7365	}
7366	lex_state = EXPR_DOT;
7367	return '.';
7368
7369      start_num:
7370      case '0': case '1': case '2': case '3': case '4':
7371      case '5': case '6': case '7': case '8': case '9':
7372	{
7373	    int is_float, seen_point, seen_e, nondigit;
7374
7375	    is_float = seen_point = seen_e = nondigit = 0;
7376	    lex_state = EXPR_END;
7377	    newtok();
7378	    if (c == '-' || c == '+') {
7379		tokadd(c);
7380		c = nextc();
7381	    }
7382	    if (c == '0') {
7383#define no_digits() do {yyerror("numeric literal without digits"); return 0;} while (0)
7384		int start = toklen();
7385		c = nextc();
7386		if (c == 'x' || c == 'X') {
7387		    /* hexadecimal */
7388		    c = nextc();
7389		    if (c != -1 && ISXDIGIT(c)) {
7390			do {
7391			    if (c == '_') {
7392				if (nondigit) break;
7393				nondigit = c;
7394				continue;
7395			    }
7396			    if (!ISXDIGIT(c)) break;
7397			    nondigit = 0;
7398			    tokadd(c);
7399			} while ((c = nextc()) != -1);
7400		    }
7401		    pushback(c);
7402		    tokfix();
7403		    if (toklen() == start) {
7404			no_digits();
7405		    }
7406		    else if (nondigit) goto trailing_uc;
7407		    set_yylval_literal(rb_cstr_to_inum(tok(), 16, FALSE));
7408		    return tINTEGER;
7409		}
7410		if (c == 'b' || c == 'B') {
7411		    /* binary */
7412		    c = nextc();
7413		    if (c == '0' || c == '1') {
7414			do {
7415			    if (c == '_') {
7416				if (nondigit) break;
7417				nondigit = c;
7418				continue;
7419			    }
7420			    if (c != '0' && c != '1') break;
7421			    nondigit = 0;
7422			    tokadd(c);
7423			} while ((c = nextc()) != -1);
7424		    }
7425		    pushback(c);
7426		    tokfix();
7427		    if (toklen() == start) {
7428			no_digits();
7429		    }
7430		    else if (nondigit) goto trailing_uc;
7431		    set_yylval_literal(rb_cstr_to_inum(tok(), 2, FALSE));
7432		    return tINTEGER;
7433		}
7434		if (c == 'd' || c == 'D') {
7435		    /* decimal */
7436		    c = nextc();
7437		    if (c != -1 && ISDIGIT(c)) {
7438			do {
7439			    if (c == '_') {
7440				if (nondigit) break;
7441				nondigit = c;
7442				continue;
7443			    }
7444			    if (!ISDIGIT(c)) break;
7445			    nondigit = 0;
7446			    tokadd(c);
7447			} while ((c = nextc()) != -1);
7448		    }
7449		    pushback(c);
7450		    tokfix();
7451		    if (toklen() == start) {
7452			no_digits();
7453		    }
7454		    else if (nondigit) goto trailing_uc;
7455		    set_yylval_literal(rb_cstr_to_inum(tok(), 10, FALSE));
7456		    return tINTEGER;
7457		}
7458		if (c == '_') {
7459		    /* 0_0 */
7460		    goto octal_number;
7461		}
7462		if (c == 'o' || c == 'O') {
7463		    /* prefixed octal */
7464		    c = nextc();
7465		    if (c == -1 || c == '_' || !ISDIGIT(c)) {
7466			no_digits();
7467		    }
7468		}
7469		if (c >= '0' && c <= '7') {
7470		    /* octal */
7471		  octal_number:
7472	            do {
7473			if (c == '_') {
7474			    if (nondigit) break;
7475			    nondigit = c;
7476			    continue;
7477			}
7478			if (c < '0' || c > '9') break;
7479			if (c > '7') goto invalid_octal;
7480			nondigit = 0;
7481			tokadd(c);
7482		    } while ((c = nextc()) != -1);
7483		    if (toklen() > start) {
7484			pushback(c);
7485			tokfix();
7486			if (nondigit) goto trailing_uc;
7487			set_yylval_literal(rb_cstr_to_inum(tok(), 8, FALSE));
7488			return tINTEGER;
7489		    }
7490		    if (nondigit) {
7491			pushback(c);
7492			goto trailing_uc;
7493		    }
7494		}
7495		if (c > '7' && c <= '9') {
7496		  invalid_octal:
7497		    yyerror("Invalid octal digit");
7498		}
7499		else if (c == '.' || c == 'e' || c == 'E') {
7500		    tokadd('0');
7501		}
7502		else {
7503		    pushback(c);
7504                    set_yylval_literal(INT2FIX(0));
7505		    return tINTEGER;
7506		}
7507	    }
7508
7509	    for (;;) {
7510		switch (c) {
7511		  case '0': case '1': case '2': case '3': case '4':
7512		  case '5': case '6': case '7': case '8': case '9':
7513		    nondigit = 0;
7514		    tokadd(c);
7515		    break;
7516
7517		  case '.':
7518		    if (nondigit) goto trailing_uc;
7519		    if (seen_point || seen_e) {
7520			goto decode_num;
7521		    }
7522		    else {
7523			int c0 = nextc();
7524			if (c0 == -1 || !ISDIGIT(c0)) {
7525			    pushback(c0);
7526			    goto decode_num;
7527			}
7528			c = c0;
7529		    }
7530		    tokadd('.');
7531		    tokadd(c);
7532		    is_float++;
7533		    seen_point++;
7534		    nondigit = 0;
7535		    break;
7536
7537		  case 'e':
7538		  case 'E':
7539		    if (nondigit) {
7540			pushback(c);
7541			c = nondigit;
7542			goto decode_num;
7543		    }
7544		    if (seen_e) {
7545			goto decode_num;
7546		    }
7547		    tokadd(c);
7548		    seen_e++;
7549		    is_float++;
7550		    nondigit = c;
7551		    c = nextc();
7552		    if (c != '-' && c != '+') continue;
7553		    tokadd(c);
7554		    nondigit = c;
7555		    break;
7556
7557		  case '_':	/* `_' in number just ignored */
7558		    if (nondigit) goto decode_num;
7559		    nondigit = c;
7560		    break;
7561
7562		  default:
7563		    goto decode_num;
7564		}
7565		c = nextc();
7566	    }
7567
7568	  decode_num:
7569	    pushback(c);
7570	    if (nondigit) {
7571		char tmp[30];
7572	      trailing_uc:
7573		snprintf(tmp, sizeof(tmp), "trailing `%c' in number", nondigit);
7574		yyerror(tmp);
7575	    }
7576	    tokfix();
7577	    if (is_float) {
7578		double d = strtod(tok(), 0);
7579		if (errno == ERANGE) {
7580		    rb_warningS("Float %s out of range", tok());
7581		    errno = 0;
7582		}
7583                set_yylval_literal(DBL2NUM(d));
7584		return tFLOAT;
7585	    }
7586	    set_yylval_literal(rb_cstr_to_inum(tok(), 10, FALSE));
7587	    return tINTEGER;
7588	}
7589
7590      case ')':
7591      case ']':
7592	paren_nest--;
7593      case '}':
7594	COND_LEXPOP();
7595	CMDARG_LEXPOP();
7596	if (c == ')')
7597	    lex_state = EXPR_ENDFN;
7598	else
7599	    lex_state = EXPR_ENDARG;
7600	if (c == '}') {
7601	    if (!brace_nest--) c = tSTRING_DEND;
7602	}
7603	return c;
7604
7605      case ':':
7606	c = nextc();
7607	if (c == ':') {
7608	    if (IS_BEG() || IS_lex_state(EXPR_CLASS) || IS_SPCARG(-1)) {
7609		lex_state = EXPR_BEG;
7610		return tCOLON3;
7611	    }
7612	    lex_state = EXPR_DOT;
7613	    return tCOLON2;
7614	}
7615	if (IS_END() || ISSPACE(c)) {
7616	    pushback(c);
7617	    warn_balanced(":", "symbol literal");
7618	    lex_state = EXPR_BEG;
7619	    return ':';
7620	}
7621	switch (c) {
7622	  case '\'':
7623	    lex_strterm = NEW_STRTERM(str_ssym, c, 0);
7624	    break;
7625	  case '"':
7626	    lex_strterm = NEW_STRTERM(str_dsym, c, 0);
7627	    break;
7628	  default:
7629	    pushback(c);
7630	    break;
7631	}
7632	lex_state = EXPR_FNAME;
7633	return tSYMBEG;
7634
7635      case '/':
7636	if (IS_lex_state(EXPR_BEG_ANY)) {
7637	    lex_strterm = NEW_STRTERM(str_regexp, '/', 0);
7638	    return tREGEXP_BEG;
7639	}
7640	if ((c = nextc()) == '=') {
7641            set_yylval_id('/');
7642	    lex_state = EXPR_BEG;
7643	    return tOP_ASGN;
7644	}
7645	pushback(c);
7646	if (IS_SPCARG(c)) {
7647	    (void)arg_ambiguous();
7648	    lex_strterm = NEW_STRTERM(str_regexp, '/', 0);
7649	    return tREGEXP_BEG;
7650	}
7651	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7652	warn_balanced("/", "regexp literal");
7653	return '/';
7654
7655      case '^':
7656	if ((c = nextc()) == '=') {
7657            set_yylval_id('^');
7658	    lex_state = EXPR_BEG;
7659	    return tOP_ASGN;
7660	}
7661	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7662	pushback(c);
7663	return '^';
7664
7665      case ';':
7666	lex_state = EXPR_BEG;
7667	command_start = TRUE;
7668	return ';';
7669
7670      case ',':
7671	lex_state = EXPR_BEG;
7672	return ',';
7673
7674      case '~':
7675	if (IS_AFTER_OPERATOR()) {
7676	    if ((c = nextc()) != '@') {
7677		pushback(c);
7678	    }
7679	    lex_state = EXPR_ARG;
7680	}
7681	else {
7682	    lex_state = EXPR_BEG;
7683	}
7684	return '~';
7685
7686      case '(':
7687	if (IS_BEG()) {
7688	    c = tLPAREN;
7689	}
7690	else if (IS_SPCARG(-1)) {
7691	    c = tLPAREN_ARG;
7692	}
7693	paren_nest++;
7694	COND_PUSH(0);
7695	CMDARG_PUSH(0);
7696	lex_state = EXPR_BEG;
7697	return c;
7698
7699      case '[':
7700	paren_nest++;
7701	if (IS_AFTER_OPERATOR()) {
7702	    lex_state = EXPR_ARG;
7703	    if ((c = nextc()) == ']') {
7704		if ((c = nextc()) == '=') {
7705		    return tASET;
7706		}
7707		pushback(c);
7708		return tAREF;
7709	    }
7710	    pushback(c);
7711	    return '[';
7712	}
7713	else if (IS_BEG()) {
7714	    c = tLBRACK;
7715	}
7716	else if (IS_ARG() && space_seen) {
7717	    c = tLBRACK;
7718	}
7719	lex_state = EXPR_BEG;
7720	COND_PUSH(0);
7721	CMDARG_PUSH(0);
7722	return c;
7723
7724      case '{':
7725	++brace_nest;
7726	if (lpar_beg && lpar_beg == paren_nest) {
7727	    lex_state = EXPR_BEG;
7728	    lpar_beg = 0;
7729	    --paren_nest;
7730	    COND_PUSH(0);
7731	    CMDARG_PUSH(0);
7732	    return tLAMBEG;
7733	}
7734	if (IS_ARG() || IS_lex_state(EXPR_END | EXPR_ENDFN))
7735	    c = '{';          /* block (primary) */
7736	else if (IS_lex_state(EXPR_ENDARG))
7737	    c = tLBRACE_ARG;  /* block (expr) */
7738	else
7739	    c = tLBRACE;      /* hash */
7740	COND_PUSH(0);
7741	CMDARG_PUSH(0);
7742	lex_state = EXPR_BEG;
7743	if (c != tLBRACE) command_start = TRUE;
7744	return c;
7745
7746      case '\\':
7747	c = nextc();
7748	if (c == '\n') {
7749	    space_seen = 1;
7750#ifdef RIPPER
7751	    ripper_dispatch_scan_event(parser, tSP);
7752#endif
7753	    goto retry; /* skip \\n */
7754	}
7755	pushback(c);
7756	return '\\';
7757
7758      case '%':
7759	if (IS_lex_state(EXPR_BEG_ANY)) {
7760	    int term;
7761	    int paren;
7762
7763	    c = nextc();
7764	  quotation:
7765	    if (c == -1 || !ISALNUM(c)) {
7766		term = c;
7767		c = 'Q';
7768	    }
7769	    else {
7770		term = nextc();
7771		if (rb_enc_isalnum(term, current_enc) || !parser_isascii()) {
7772		    yyerror("unknown type of %string");
7773		    return 0;
7774		}
7775	    }
7776	    if (c == -1 || term == -1) {
7777		compile_error(PARSER_ARG "unterminated quoted string meets end of file");
7778		return 0;
7779	    }
7780	    paren = term;
7781	    if (term == '(') term = ')';
7782	    else if (term == '[') term = ']';
7783	    else if (term == '{') term = '}';
7784	    else if (term == '<') term = '>';
7785	    else paren = 0;
7786
7787	    switch (c) {
7788	      case 'Q':
7789		lex_strterm = NEW_STRTERM(str_dquote, term, paren);
7790		return tSTRING_BEG;
7791
7792	      case 'q':
7793		lex_strterm = NEW_STRTERM(str_squote, term, paren);
7794		return tSTRING_BEG;
7795
7796	      case 'W':
7797		lex_strterm = NEW_STRTERM(str_dword, term, paren);
7798		do {c = nextc();} while (ISSPACE(c));
7799		pushback(c);
7800		return tWORDS_BEG;
7801
7802	      case 'w':
7803		lex_strterm = NEW_STRTERM(str_sword, term, paren);
7804		do {c = nextc();} while (ISSPACE(c));
7805		pushback(c);
7806		return tQWORDS_BEG;
7807
7808	      case 'I':
7809		lex_strterm = NEW_STRTERM(str_dword, term, paren);
7810		do {c = nextc();} while (ISSPACE(c));
7811		pushback(c);
7812		return tSYMBOLS_BEG;
7813
7814	      case 'i':
7815		lex_strterm = NEW_STRTERM(str_sword, term, paren);
7816		do {c = nextc();} while (ISSPACE(c));
7817		pushback(c);
7818		return tQSYMBOLS_BEG;
7819
7820	      case 'x':
7821		lex_strterm = NEW_STRTERM(str_xquote, term, paren);
7822		return tXSTRING_BEG;
7823
7824	      case 'r':
7825		lex_strterm = NEW_STRTERM(str_regexp, term, paren);
7826		return tREGEXP_BEG;
7827
7828	      case 's':
7829		lex_strterm = NEW_STRTERM(str_ssym, term, paren);
7830		lex_state = EXPR_FNAME;
7831		return tSYMBEG;
7832
7833	      default:
7834		yyerror("unknown type of %string");
7835		return 0;
7836	    }
7837	}
7838	if ((c = nextc()) == '=') {
7839            set_yylval_id('%');
7840	    lex_state = EXPR_BEG;
7841	    return tOP_ASGN;
7842	}
7843	if (IS_SPCARG(c)) {
7844	    goto quotation;
7845	}
7846	lex_state = IS_AFTER_OPERATOR() ? EXPR_ARG : EXPR_BEG;
7847	pushback(c);
7848	warn_balanced("%%", "string literal");
7849	return '%';
7850
7851      case '$':
7852	lex_state = EXPR_END;
7853	newtok();
7854	c = nextc();
7855	switch (c) {
7856	  case '_':		/* $_: last read line string */
7857	    c = nextc();
7858	    if (parser_is_identchar()) {
7859		tokadd('$');
7860		tokadd('_');
7861		break;
7862	    }
7863	    pushback(c);
7864	    c = '_';
7865	    /* fall through */
7866	  case '~':		/* $~: match-data */
7867	  case '*':		/* $*: argv */
7868	  case '$':		/* $$: pid */
7869	  case '?':		/* $?: last status */
7870	  case '!':		/* $!: error string */
7871	  case '@':		/* $@: error position */
7872	  case '/':		/* $/: input record separator */
7873	  case '\\':		/* $\: output record separator */
7874	  case ';':		/* $;: field separator */
7875	  case ',':		/* $,: output field separator */
7876	  case '.':		/* $.: last read line number */
7877	  case '=':		/* $=: ignorecase */
7878	  case ':':		/* $:: load path */
7879	  case '<':		/* $<: reading filename */
7880	  case '>':		/* $>: default output handle */
7881	  case '\"':		/* $": already loaded files */
7882	    tokadd('$');
7883	    tokadd(c);
7884	    tokfix();
7885	    set_yylval_name(rb_intern(tok()));
7886	    return tGVAR;
7887
7888	  case '-':
7889	    tokadd('$');
7890	    tokadd(c);
7891	    c = nextc();
7892	    if (parser_is_identchar()) {
7893		if (tokadd_mbchar(c) == -1) return 0;
7894	    }
7895	    else {
7896		pushback(c);
7897	    }
7898	  gvar:
7899	    tokfix();
7900	    set_yylval_name(rb_intern(tok()));
7901	    return tGVAR;
7902
7903	  case '&':		/* $&: last match */
7904	  case '`':		/* $`: string before last match */
7905	  case '\'':		/* $': string after last match */
7906	  case '+':		/* $+: string matches last paren. */
7907	    if (IS_lex_state_for(last_state, EXPR_FNAME)) {
7908		tokadd('$');
7909		tokadd(c);
7910		goto gvar;
7911	    }
7912	    set_yylval_node(NEW_BACK_REF(c));
7913	    return tBACK_REF;
7914
7915	  case '1': case '2': case '3':
7916	  case '4': case '5': case '6':
7917	  case '7': case '8': case '9':
7918	    tokadd('$');
7919	    do {
7920		tokadd(c);
7921		c = nextc();
7922	    } while (c != -1 && ISDIGIT(c));
7923	    pushback(c);
7924	    if (IS_lex_state_for(last_state, EXPR_FNAME)) goto gvar;
7925	    tokfix();
7926	    set_yylval_node(NEW_NTH_REF(atoi(tok()+1)));
7927	    return tNTH_REF;
7928
7929	  default:
7930	    if (!parser_is_identchar()) {
7931		pushback(c);
7932		compile_error(PARSER_ARG "`$%c' is not allowed as a global variable name", c);
7933		return 0;
7934	    }
7935	  case '0':
7936	    tokadd('$');
7937	}
7938	break;
7939
7940      case '@':
7941	c = nextc();
7942	newtok();
7943	tokadd('@');
7944	if (c == '@') {
7945	    tokadd('@');
7946	    c = nextc();
7947	}
7948	if (c != -1 && (ISDIGIT(c) || !parser_is_identchar())) {
7949	    pushback(c);
7950	    if (tokidx == 1) {
7951		compile_error(PARSER_ARG "`@%c' is not allowed as an instance variable name", c);
7952	    }
7953	    else {
7954		compile_error(PARSER_ARG "`@@%c' is not allowed as a class variable name", c);
7955	    }
7956	    return 0;
7957	}
7958	break;
7959
7960      case '_':
7961	if (was_bol() && whole_match_p("__END__", 7, 0)) {
7962	    ruby__end__seen = 1;
7963	    parser->eofp = Qtrue;
7964#ifndef RIPPER
7965	    return -1;
7966#else
7967            lex_goto_eol(parser);
7968            ripper_dispatch_scan_event(parser, k__END__);
7969            return 0;
7970#endif
7971	}
7972	newtok();
7973	break;
7974
7975      default:
7976	if (!parser_is_identchar()) {
7977	    rb_compile_error(PARSER_ARG  "Invalid char `\\x%02X' in expression", c);
7978	    goto retry;
7979	}
7980
7981	newtok();
7982	break;
7983    }
7984
7985    mb = ENC_CODERANGE_7BIT;
7986    do {
7987	if (!ISASCII(c)) mb = ENC_CODERANGE_UNKNOWN;
7988	if (tokadd_mbchar(c) == -1) return 0;
7989	c = nextc();
7990    } while (parser_is_identchar());
7991    switch (tok()[0]) {
7992      case '@': case '$':
7993	pushback(c);
7994	break;
7995      default:
7996	if ((c == '!' || c == '?') && !peek('=')) {
7997	    tokadd(c);
7998	}
7999	else {
8000	    pushback(c);
8001	}
8002    }
8003    tokfix();
8004
8005    {
8006	int result = 0;
8007
8008	last_state = lex_state;
8009	switch (tok()[0]) {
8010	  case '$':
8011	    lex_state = EXPR_END;
8012	    result = tGVAR;
8013	    break;
8014	  case '@':
8015	    lex_state = EXPR_END;
8016	    if (tok()[1] == '@')
8017		result = tCVAR;
8018	    else
8019		result = tIVAR;
8020	    break;
8021
8022	  default:
8023	    if (toklast() == '!' || toklast() == '?') {
8024		result = tFID;
8025	    }
8026	    else {
8027		if (IS_lex_state(EXPR_FNAME)) {
8028		    if ((c = nextc()) == '=' && !peek('~') && !peek('>') &&
8029			(!peek('=') || (peek_n('>', 1)))) {
8030			result = tIDENTIFIER;
8031			tokadd(c);
8032			tokfix();
8033		    }
8034		    else {
8035			pushback(c);
8036		    }
8037		}
8038		if (result == 0 && ISUPPER(tok()[0])) {
8039		    result = tCONSTANT;
8040		}
8041		else {
8042		    result = tIDENTIFIER;
8043		}
8044	    }
8045
8046	    if (IS_LABEL_POSSIBLE()) {
8047		if (IS_LABEL_SUFFIX(0)) {
8048		    lex_state = EXPR_BEG;
8049		    nextc();
8050		    set_yylval_name(TOK_INTERN(!ENC_SINGLE(mb)));
8051		    return tLABEL;
8052		}
8053	    }
8054	    if (mb == ENC_CODERANGE_7BIT && !IS_lex_state(EXPR_DOT)) {
8055		const struct kwtable *kw;
8056
8057		/* See if it is a reserved word.  */
8058		kw = rb_reserved_word(tok(), toklen());
8059		if (kw) {
8060		    enum lex_state_e state = lex_state;
8061		    lex_state = kw->state;
8062		    if (state == EXPR_FNAME) {
8063			set_yylval_name(rb_intern(kw->name));
8064			return kw->id[0];
8065		    }
8066		    if (lex_state == EXPR_BEG) {
8067			command_start = TRUE;
8068		    }
8069		    if (kw->id[0] == keyword_do) {
8070			if (lpar_beg && lpar_beg == paren_nest) {
8071			    lpar_beg = 0;
8072			    --paren_nest;
8073			    return keyword_do_LAMBDA;
8074			}
8075			if (COND_P()) return keyword_do_cond;
8076			if (CMDARG_P() && state != EXPR_CMDARG)
8077			    return keyword_do_block;
8078			if (state & (EXPR_BEG | EXPR_ENDARG))
8079			    return keyword_do_block;
8080			return keyword_do;
8081		    }
8082		    if (state & (EXPR_BEG | EXPR_VALUE))
8083			return kw->id[0];
8084		    else {
8085			if (kw->id[0] != kw->id[1])
8086			    lex_state = EXPR_BEG;
8087			return kw->id[1];
8088		    }
8089		}
8090	    }
8091
8092	    if (IS_lex_state(EXPR_BEG_ANY | EXPR_ARG_ANY | EXPR_DOT)) {
8093		if (cmd_state) {
8094		    lex_state = EXPR_CMDARG;
8095		}
8096		else {
8097		    lex_state = EXPR_ARG;
8098		}
8099	    }
8100	    else if (lex_state == EXPR_FNAME) {
8101		lex_state = EXPR_ENDFN;
8102	    }
8103	    else {
8104		lex_state = EXPR_END;
8105	    }
8106	}
8107        {
8108            ID ident = TOK_INTERN(!ENC_SINGLE(mb));
8109
8110            set_yylval_name(ident);
8111            if (!IS_lex_state_for(last_state, EXPR_DOT|EXPR_FNAME) &&
8112		is_local_id(ident) && lvar_defined(ident)) {
8113                lex_state = EXPR_END;
8114            }
8115        }
8116	return result;
8117    }
8118}
8119
8120#if YYPURE
8121static int
8122yylex(void *lval, void *p)
8123#else
8124yylex(void *p)
8125#endif
8126{
8127    struct parser_params *parser = (struct parser_params*)p;
8128    int t;
8129
8130#if YYPURE
8131    parser->parser_yylval = lval;
8132    parser->parser_yylval->val = Qundef;
8133#endif
8134    t = parser_yylex(parser);
8135#ifdef RIPPER
8136    if (!NIL_P(parser->delayed)) {
8137	ripper_dispatch_delayed_token(parser, t);
8138	return t;
8139    }
8140    if (t != 0)
8141	ripper_dispatch_scan_event(parser, t);
8142#endif
8143
8144    return t;
8145}
8146
8147#ifndef RIPPER
8148static NODE*
8149node_newnode(struct parser_params *parser, enum node_type type, VALUE a0, VALUE a1, VALUE a2)
8150{
8151    NODE *n = (rb_node_newnode)(type, a0, a1, a2);
8152    nd_set_line(n, ruby_sourceline);
8153    return n;
8154}
8155
8156static enum node_type
8157nodetype(NODE *node)			/* for debug */
8158{
8159    return (enum node_type)nd_type(node);
8160}
8161
8162static int
8163nodeline(NODE *node)
8164{
8165    return nd_line(node);
8166}
8167
8168static NODE*
8169newline_node(NODE *node)
8170{
8171    if (node) {
8172	node = remove_begin(node);
8173	node->flags |= NODE_FL_NEWLINE;
8174    }
8175    return node;
8176}
8177
8178static void
8179fixpos(NODE *node, NODE *orig)
8180{
8181    if (!node) return;
8182    if (!orig) return;
8183    if (orig == (NODE*)1) return;
8184    nd_set_line(node, nd_line(orig));
8185}
8186
8187static void
8188parser_warning(struct parser_params *parser, NODE *node, const char *mesg)
8189{
8190    rb_compile_warning(ruby_sourcefile, nd_line(node), "%s", mesg);
8191}
8192#define parser_warning(node, mesg) parser_warning(parser, (node), (mesg))
8193
8194static void
8195parser_warn(struct parser_params *parser, NODE *node, const char *mesg)
8196{
8197    rb_compile_warn(ruby_sourcefile, nd_line(node), "%s", mesg);
8198}
8199#define parser_warn(node, mesg) parser_warn(parser, (node), (mesg))
8200
8201static NODE*
8202block_append_gen(struct parser_params *parser, NODE *head, NODE *tail)
8203{
8204    NODE *end, *h = head, *nd;
8205
8206    if (tail == 0) return head;
8207
8208    if (h == 0) return tail;
8209    switch (nd_type(h)) {
8210      case NODE_LIT:
8211      case NODE_STR:
8212      case NODE_SELF:
8213      case NODE_TRUE:
8214      case NODE_FALSE:
8215      case NODE_NIL:
8216	parser_warning(h, "unused literal ignored");
8217	return tail;
8218      default:
8219	h = end = NEW_BLOCK(head);
8220	end->nd_end = end;
8221	fixpos(end, head);
8222	head = end;
8223	break;
8224      case NODE_BLOCK:
8225	end = h->nd_end;
8226	break;
8227    }
8228
8229    nd = end->nd_head;
8230    switch (nd_type(nd)) {
8231      case NODE_RETURN:
8232      case NODE_BREAK:
8233      case NODE_NEXT:
8234      case NODE_REDO:
8235      case NODE_RETRY:
8236	if (RTEST(ruby_verbose)) {
8237	    parser_warning(tail, "statement not reached");
8238	}
8239	break;
8240
8241      default:
8242	break;
8243    }
8244
8245    if (nd_type(tail) != NODE_BLOCK) {
8246	tail = NEW_BLOCK(tail);
8247	tail->nd_end = tail;
8248    }
8249    end->nd_next = tail;
8250    h->nd_end = tail->nd_end;
8251    return head;
8252}
8253
8254/* append item to the list */
8255static NODE*
8256list_append_gen(struct parser_params *parser, NODE *list, NODE *item)
8257{
8258    NODE *last;
8259
8260    if (list == 0) return NEW_LIST(item);
8261    if (list->nd_next) {
8262	last = list->nd_next->nd_end;
8263    }
8264    else {
8265	last = list;
8266    }
8267
8268    list->nd_alen += 1;
8269    last->nd_next = NEW_LIST(item);
8270    list->nd_next->nd_end = last->nd_next;
8271    return list;
8272}
8273
8274/* concat two lists */
8275static NODE*
8276list_concat_gen(struct parser_params *parser, NODE *head, NODE *tail)
8277{
8278    NODE *last;
8279
8280    if (head->nd_next) {
8281	last = head->nd_next->nd_end;
8282    }
8283    else {
8284	last = head;
8285    }
8286
8287    head->nd_alen += tail->nd_alen;
8288    last->nd_next = tail;
8289    if (tail->nd_next) {
8290	head->nd_next->nd_end = tail->nd_next->nd_end;
8291    }
8292    else {
8293	head->nd_next->nd_end = tail;
8294    }
8295
8296    return head;
8297}
8298
8299static int
8300literal_concat0(struct parser_params *parser, VALUE head, VALUE tail)
8301{
8302    if (NIL_P(tail)) return 1;
8303    if (!rb_enc_compatible(head, tail)) {
8304	compile_error(PARSER_ARG "string literal encodings differ (%s / %s)",
8305		      rb_enc_name(rb_enc_get(head)),
8306		      rb_enc_name(rb_enc_get(tail)));
8307	rb_str_resize(head, 0);
8308	rb_str_resize(tail, 0);
8309	return 0;
8310    }
8311    rb_str_buf_append(head, tail);
8312    return 1;
8313}
8314
8315/* concat two string literals */
8316static NODE *
8317literal_concat_gen(struct parser_params *parser, NODE *head, NODE *tail)
8318{
8319    enum node_type htype;
8320    NODE *headlast;
8321    VALUE lit;
8322
8323    if (!head) return tail;
8324    if (!tail) return head;
8325
8326    htype = nd_type(head);
8327    if (htype == NODE_EVSTR) {
8328	NODE *node = NEW_DSTR(Qnil);
8329	head = list_append(node, head);
8330	htype = NODE_DSTR;
8331    }
8332    switch (nd_type(tail)) {
8333      case NODE_STR:
8334	if (htype == NODE_DSTR && (headlast = head->nd_next->nd_end->nd_head) &&
8335	    nd_type(headlast) == NODE_STR) {
8336	    htype = NODE_STR;
8337	    lit = headlast->nd_lit;
8338	}
8339	else {
8340	    lit = head->nd_lit;
8341	}
8342	if (htype == NODE_STR) {
8343	    if (!literal_concat0(parser, lit, tail->nd_lit)) {
8344	      error:
8345		rb_gc_force_recycle((VALUE)head);
8346		rb_gc_force_recycle((VALUE)tail);
8347		return 0;
8348	    }
8349	    rb_gc_force_recycle((VALUE)tail);
8350	}
8351	else {
8352	    list_append(head, tail);
8353	}
8354	break;
8355
8356      case NODE_DSTR:
8357	if (htype == NODE_STR) {
8358	    if (!literal_concat0(parser, head->nd_lit, tail->nd_lit))
8359		goto error;
8360	    tail->nd_lit = head->nd_lit;
8361	    rb_gc_force_recycle((VALUE)head);
8362	    head = tail;
8363	}
8364	else if (NIL_P(tail->nd_lit)) {
8365	  append:
8366	    head->nd_alen += tail->nd_alen - 1;
8367	    head->nd_next->nd_end->nd_next = tail->nd_next;
8368	    head->nd_next->nd_end = tail->nd_next->nd_end;
8369	    rb_gc_force_recycle((VALUE)tail);
8370	}
8371	else if (htype == NODE_DSTR && (headlast = head->nd_next->nd_end->nd_head) &&
8372		 nd_type(headlast) == NODE_STR) {
8373	    lit = headlast->nd_lit;
8374	    if (!literal_concat0(parser, lit, tail->nd_lit))
8375		goto error;
8376	    tail->nd_lit = Qnil;
8377	    goto append;
8378	}
8379	else {
8380	    nd_set_type(tail, NODE_ARRAY);
8381	    tail->nd_head = NEW_STR(tail->nd_lit);
8382	    list_concat(head, tail);
8383	}
8384	break;
8385
8386      case NODE_EVSTR:
8387	if (htype == NODE_STR) {
8388	    nd_set_type(head, NODE_DSTR);
8389	    head->nd_alen = 1;
8390	}
8391	list_append(head, tail);
8392	break;
8393    }
8394    return head;
8395}
8396
8397static NODE *
8398evstr2dstr_gen(struct parser_params *parser, NODE *node)
8399{
8400    if (nd_type(node) == NODE_EVSTR) {
8401	node = list_append(NEW_DSTR(Qnil), node);
8402    }
8403    return node;
8404}
8405
8406static NODE *
8407new_evstr_gen(struct parser_params *parser, NODE *node)
8408{
8409    NODE *head = node;
8410
8411    if (node) {
8412	switch (nd_type(node)) {
8413	  case NODE_STR: case NODE_DSTR: case NODE_EVSTR:
8414	    return node;
8415	}
8416    }
8417    return NEW_EVSTR(head);
8418}
8419
8420static NODE *
8421call_bin_op_gen(struct parser_params *parser, NODE *recv, ID id, NODE *arg1)
8422{
8423    value_expr(recv);
8424    value_expr(arg1);
8425    return NEW_CALL(recv, id, NEW_LIST(arg1));
8426}
8427
8428static NODE *
8429call_uni_op_gen(struct parser_params *parser, NODE *recv, ID id)
8430{
8431    value_expr(recv);
8432    return NEW_CALL(recv, id, 0);
8433}
8434
8435static NODE*
8436match_op_gen(struct parser_params *parser, NODE *node1, NODE *node2)
8437{
8438    value_expr(node1);
8439    value_expr(node2);
8440    if (node1) {
8441	switch (nd_type(node1)) {
8442	  case NODE_DREGX:
8443	  case NODE_DREGX_ONCE:
8444	    return NEW_MATCH2(node1, node2);
8445
8446	  case NODE_LIT:
8447	    if (RB_TYPE_P(node1->nd_lit, T_REGEXP)) {
8448		return NEW_MATCH2(node1, node2);
8449	    }
8450	}
8451    }
8452
8453    if (node2) {
8454	switch (nd_type(node2)) {
8455	  case NODE_DREGX:
8456	  case NODE_DREGX_ONCE:
8457	    return NEW_MATCH3(node2, node1);
8458
8459	  case NODE_LIT:
8460	    if (RB_TYPE_P(node2->nd_lit, T_REGEXP)) {
8461		return NEW_MATCH3(node2, node1);
8462	    }
8463	}
8464    }
8465
8466    return NEW_CALL(node1, tMATCH, NEW_LIST(node2));
8467}
8468
8469static NODE*
8470gettable_gen(struct parser_params *parser, ID id)
8471{
8472    switch (id) {
8473      case keyword_self:
8474	return NEW_SELF();
8475      case keyword_nil:
8476	return NEW_NIL();
8477      case keyword_true:
8478	return NEW_TRUE();
8479      case keyword_false:
8480	return NEW_FALSE();
8481      case keyword__FILE__:
8482	return NEW_STR(rb_str_dup(ruby_sourcefile_string));
8483      case keyword__LINE__:
8484	return NEW_LIT(INT2FIX(tokline));
8485      case keyword__ENCODING__:
8486	return NEW_LIT(rb_enc_from_encoding(current_enc));
8487    }
8488    switch (id_type(id)) {
8489      case ID_LOCAL:
8490	if (dyna_in_block() && dvar_defined(id)) return NEW_DVAR(id);
8491	if (local_id(id)) return NEW_LVAR(id);
8492	/* method call without arguments */
8493	return NEW_VCALL(id);
8494      case ID_GLOBAL:
8495	return NEW_GVAR(id);
8496      case ID_INSTANCE:
8497	return NEW_IVAR(id);
8498      case ID_CONST:
8499	return NEW_CONST(id);
8500      case ID_CLASS:
8501	return NEW_CVAR(id);
8502    }
8503    compile_error(PARSER_ARG "identifier %s is not valid to get", rb_id2name(id));
8504    return 0;
8505}
8506#else  /* !RIPPER */
8507static int
8508id_is_var_gen(struct parser_params *parser, ID id)
8509{
8510    if (is_notop_id(id)) {
8511	switch (id & ID_SCOPE_MASK) {
8512	  case ID_GLOBAL: case ID_INSTANCE: case ID_CONST: case ID_CLASS:
8513	    return 1;
8514	  case ID_LOCAL:
8515	    if (dyna_in_block() && dvar_defined(id)) return 1;
8516	    if (local_id(id)) return 1;
8517	    /* method call without arguments */
8518	    return 0;
8519	}
8520    }
8521    compile_error(PARSER_ARG "identifier %s is not valid to get", rb_id2name(id));
8522    return 0;
8523}
8524#endif /* !RIPPER */
8525
8526#if PARSER_DEBUG
8527static const char *
8528lex_state_name(enum lex_state_e state)
8529{
8530    static const char names[][12] = {
8531	"EXPR_BEG",    "EXPR_END",    "EXPR_ENDARG", "EXPR_ENDFN",  "EXPR_ARG",
8532	"EXPR_CMDARG", "EXPR_MID",    "EXPR_FNAME",  "EXPR_DOT",    "EXPR_CLASS",
8533	"EXPR_VALUE",
8534    };
8535
8536    if ((unsigned)state & ~(~0u << EXPR_MAX_STATE))
8537	return names[ffs(state)];
8538    return NULL;
8539}
8540#endif
8541
8542#ifdef RIPPER
8543static VALUE
8544assignable_gen(struct parser_params *parser, VALUE lhs)
8545#else
8546static NODE*
8547assignable_gen(struct parser_params *parser, ID id, NODE *val)
8548#endif
8549{
8550#ifdef RIPPER
8551    ID id = get_id(lhs);
8552# define assignable_result(x) get_value(lhs)
8553# define parser_yyerror(parser, x) dispatch1(assign_error, lhs)
8554#else
8555# define assignable_result(x) (x)
8556#endif
8557    if (!id) return assignable_result(0);
8558    switch (id) {
8559      case keyword_self:
8560	yyerror("Can't change the value of self");
8561	goto error;
8562      case keyword_nil:
8563	yyerror("Can't assign to nil");
8564	goto error;
8565      case keyword_true:
8566	yyerror("Can't assign to true");
8567	goto error;
8568      case keyword_false:
8569	yyerror("Can't assign to false");
8570	goto error;
8571      case keyword__FILE__:
8572	yyerror("Can't assign to __FILE__");
8573	goto error;
8574      case keyword__LINE__:
8575	yyerror("Can't assign to __LINE__");
8576	goto error;
8577      case keyword__ENCODING__:
8578	yyerror("Can't assign to __ENCODING__");
8579	goto error;
8580    }
8581    switch (id_type(id)) {
8582      case ID_LOCAL:
8583	if (dyna_in_block()) {
8584	    if (dvar_curr(id)) {
8585		return assignable_result(NEW_DASGN_CURR(id, val));
8586	    }
8587	    else if (dvar_defined(id)) {
8588		return assignable_result(NEW_DASGN(id, val));
8589	    }
8590	    else if (local_id(id)) {
8591		return assignable_result(NEW_LASGN(id, val));
8592	    }
8593	    else {
8594		dyna_var(id);
8595		return assignable_result(NEW_DASGN_CURR(id, val));
8596	    }
8597	}
8598	else {
8599	    if (!local_id(id)) {
8600		local_var(id);
8601	    }
8602	    return assignable_result(NEW_LASGN(id, val));
8603	}
8604	break;
8605      case ID_GLOBAL:
8606	return assignable_result(NEW_GASGN(id, val));
8607      case ID_INSTANCE:
8608	return assignable_result(NEW_IASGN(id, val));
8609      case ID_CONST:
8610	if (!in_def && !in_single)
8611	    return assignable_result(NEW_CDECL(id, val, 0));
8612	yyerror("dynamic constant assignment");
8613	break;
8614      case ID_CLASS:
8615	return assignable_result(NEW_CVASGN(id, val));
8616      default:
8617	compile_error(PARSER_ARG "identifier %s is not valid to set", rb_id2name(id));
8618    }
8619  error:
8620    return assignable_result(0);
8621#undef assignable_result
8622#undef parser_yyerror
8623}
8624
8625static int
8626is_private_local_id(ID name)
8627{
8628    VALUE s;
8629    if (name == idUScore) return 1;
8630    if (!is_local_id(name)) return 0;
8631    s = rb_id2str(name);
8632    if (!s) return 0;
8633    return RSTRING_PTR(s)[0] == '_';
8634}
8635
8636#define LVAR_USED ((ID)1 << (sizeof(ID) * CHAR_BIT - 1))
8637
8638static ID
8639shadowing_lvar_gen(struct parser_params *parser, ID name)
8640{
8641    if (is_private_local_id(name)) return name;
8642    if (dyna_in_block()) {
8643	if (dvar_curr(name)) {
8644	    yyerror("duplicated argument name");
8645	}
8646	else if (dvar_defined_get(name) || local_id(name)) {
8647	    rb_warningS("shadowing outer local variable - %s", rb_id2name(name));
8648	    vtable_add(lvtbl->vars, name);
8649	    if (lvtbl->used) {
8650		vtable_add(lvtbl->used, (ID)ruby_sourceline | LVAR_USED);
8651	    }
8652	}
8653    }
8654    else {
8655	if (local_id(name)) {
8656	    yyerror("duplicated argument name");
8657	}
8658    }
8659    return name;
8660}
8661
8662static void
8663new_bv_gen(struct parser_params *parser, ID name)
8664{
8665    if (!name) return;
8666    if (!is_local_id(name)) {
8667	compile_error(PARSER_ARG "invalid local variable - %s",
8668		      rb_id2name(name));
8669	return;
8670    }
8671    shadowing_lvar(name);
8672    dyna_var(name);
8673}
8674
8675#ifndef RIPPER
8676static NODE *
8677aryset_gen(struct parser_params *parser, NODE *recv, NODE *idx)
8678{
8679    if (recv && nd_type(recv) == NODE_SELF)
8680	recv = (NODE *)1;
8681    return NEW_ATTRASGN(recv, tASET, idx);
8682}
8683
8684static void
8685block_dup_check_gen(struct parser_params *parser, NODE *node1, NODE *node2)
8686{
8687    if (node2 && node1 && nd_type(node1) == NODE_BLOCK_PASS) {
8688	compile_error(PARSER_ARG "both block arg and actual block given");
8689    }
8690}
8691
8692static const char id_type_names[][9] = {
8693    "LOCAL",
8694    "INSTANCE",
8695    "",				/* INSTANCE2 */
8696    "GLOBAL",
8697    "ATTRSET",
8698    "CONST",
8699    "CLASS",
8700    "JUNK",
8701};
8702
8703ID
8704rb_id_attrset(ID id)
8705{
8706    if (!is_notop_id(id)) {
8707	switch (id) {
8708	  case tAREF: case tASET:
8709	    return tASET;	/* only exception */
8710	}
8711	rb_name_error(id, "cannot make operator ID :%s attrset", rb_id2name(id));
8712    }
8713    else {
8714	int scope = (int)(id & ID_SCOPE_MASK);
8715	switch (scope) {
8716	  case ID_LOCAL: case ID_INSTANCE: case ID_GLOBAL:
8717	  case ID_CONST: case ID_CLASS: case ID_JUNK:
8718	    break;
8719	  case ID_ATTRSET:
8720	    return id;
8721	  default:
8722	    rb_name_error(id, "cannot make %s ID %+"PRIsVALUE" attrset",
8723			  id_type_names[scope], ID2SYM(id));
8724
8725	}
8726    }
8727    id &= ~ID_SCOPE_MASK;
8728    id |= ID_ATTRSET;
8729    return id;
8730}
8731
8732static NODE *
8733attrset_gen(struct parser_params *parser, NODE *recv, ID id)
8734{
8735    if (recv && nd_type(recv) == NODE_SELF)
8736	recv = (NODE *)1;
8737    return NEW_ATTRASGN(recv, rb_id_attrset(id), 0);
8738}
8739
8740static void
8741rb_backref_error_gen(struct parser_params *parser, NODE *node)
8742{
8743    switch (nd_type(node)) {
8744      case NODE_NTH_REF:
8745	compile_error(PARSER_ARG "Can't set variable $%ld", node->nd_nth);
8746	break;
8747      case NODE_BACK_REF:
8748	compile_error(PARSER_ARG "Can't set variable $%c", (int)node->nd_nth);
8749	break;
8750    }
8751}
8752
8753static NODE *
8754arg_concat_gen(struct parser_params *parser, NODE *node1, NODE *node2)
8755{
8756    if (!node2) return node1;
8757    switch (nd_type(node1)) {
8758      case NODE_BLOCK_PASS:
8759	if (node1->nd_head)
8760	    node1->nd_head = arg_concat(node1->nd_head, node2);
8761	else
8762	    node1->nd_head = NEW_LIST(node2);
8763	return node1;
8764      case NODE_ARGSPUSH:
8765	if (nd_type(node2) != NODE_ARRAY) break;
8766	node1->nd_body = list_concat(NEW_LIST(node1->nd_body), node2);
8767	nd_set_type(node1, NODE_ARGSCAT);
8768	return node1;
8769      case NODE_ARGSCAT:
8770	if (nd_type(node2) != NODE_ARRAY ||
8771	    nd_type(node1->nd_body) != NODE_ARRAY) break;
8772	node1->nd_body = list_concat(node1->nd_body, node2);
8773	return node1;
8774    }
8775    return NEW_ARGSCAT(node1, node2);
8776}
8777
8778static NODE *
8779arg_append_gen(struct parser_params *parser, NODE *node1, NODE *node2)
8780{
8781    if (!node1) return NEW_LIST(node2);
8782    switch (nd_type(node1))  {
8783      case NODE_ARRAY:
8784	return list_append(node1, node2);
8785      case NODE_BLOCK_PASS:
8786	node1->nd_head = arg_append(node1->nd_head, node2);
8787	return node1;
8788      case NODE_ARGSPUSH:
8789	node1->nd_body = list_append(NEW_LIST(node1->nd_body), node2);
8790	nd_set_type(node1, NODE_ARGSCAT);
8791	return node1;
8792    }
8793    return NEW_ARGSPUSH(node1, node2);
8794}
8795
8796static NODE *
8797splat_array(NODE* node)
8798{
8799    if (nd_type(node) == NODE_SPLAT) node = node->nd_head;
8800    if (nd_type(node) == NODE_ARRAY) return node;
8801    return 0;
8802}
8803
8804static NODE *
8805node_assign_gen(struct parser_params *parser, NODE *lhs, NODE *rhs)
8806{
8807    if (!lhs) return 0;
8808
8809    switch (nd_type(lhs)) {
8810      case NODE_GASGN:
8811      case NODE_IASGN:
8812      case NODE_IASGN2:
8813      case NODE_LASGN:
8814      case NODE_DASGN:
8815      case NODE_DASGN_CURR:
8816      case NODE_MASGN:
8817      case NODE_CDECL:
8818      case NODE_CVASGN:
8819	lhs->nd_value = rhs;
8820	break;
8821
8822      case NODE_ATTRASGN:
8823      case NODE_CALL:
8824	lhs->nd_args = arg_append(lhs->nd_args, rhs);
8825	break;
8826
8827      default:
8828	/* should not happen */
8829	break;
8830    }
8831
8832    return lhs;
8833}
8834
8835static int
8836value_expr_gen(struct parser_params *parser, NODE *node)
8837{
8838    int cond = 0;
8839
8840    if (!node) {
8841	rb_warning0("empty expression");
8842    }
8843    while (node) {
8844	switch (nd_type(node)) {
8845	  case NODE_DEFN:
8846	  case NODE_DEFS:
8847	    parser_warning(node, "void value expression");
8848	    return FALSE;
8849
8850	  case NODE_RETURN:
8851	  case NODE_BREAK:
8852	  case NODE_NEXT:
8853	  case NODE_REDO:
8854	  case NODE_RETRY:
8855	    if (!cond) yyerror("void value expression");
8856	    /* or "control never reach"? */
8857	    return FALSE;
8858
8859	  case NODE_BLOCK:
8860	    while (node->nd_next) {
8861		node = node->nd_next;
8862	    }
8863	    node = node->nd_head;
8864	    break;
8865
8866	  case NODE_BEGIN:
8867	    node = node->nd_body;
8868	    break;
8869
8870	  case NODE_IF:
8871	    if (!node->nd_body) {
8872		node = node->nd_else;
8873		break;
8874	    }
8875	    else if (!node->nd_else) {
8876		node = node->nd_body;
8877		break;
8878	    }
8879	    if (!value_expr(node->nd_body)) return FALSE;
8880	    node = node->nd_else;
8881	    break;
8882
8883	  case NODE_AND:
8884	  case NODE_OR:
8885	    cond = 1;
8886	    node = node->nd_2nd;
8887	    break;
8888
8889	  default:
8890	    return TRUE;
8891	}
8892    }
8893
8894    return TRUE;
8895}
8896
8897static void
8898void_expr_gen(struct parser_params *parser, NODE *node)
8899{
8900    const char *useless = 0;
8901
8902    if (!RTEST(ruby_verbose)) return;
8903
8904    if (!node) return;
8905    switch (nd_type(node)) {
8906      case NODE_CALL:
8907	switch (node->nd_mid) {
8908	  case '+':
8909	  case '-':
8910	  case '*':
8911	  case '/':
8912	  case '%':
8913	  case tPOW:
8914	  case tUPLUS:
8915	  case tUMINUS:
8916	  case '|':
8917	  case '^':
8918	  case '&':
8919	  case tCMP:
8920	  case '>':
8921	  case tGEQ:
8922	  case '<':
8923	  case tLEQ:
8924	  case tEQ:
8925	  case tNEQ:
8926	    useless = rb_id2name(node->nd_mid);
8927	    break;
8928	}
8929	break;
8930
8931      case NODE_LVAR:
8932      case NODE_DVAR:
8933      case NODE_GVAR:
8934      case NODE_IVAR:
8935      case NODE_CVAR:
8936      case NODE_NTH_REF:
8937      case NODE_BACK_REF:
8938	useless = "a variable";
8939	break;
8940      case NODE_CONST:
8941	useless = "a constant";
8942	break;
8943      case NODE_LIT:
8944      case NODE_STR:
8945      case NODE_DSTR:
8946      case NODE_DREGX:
8947      case NODE_DREGX_ONCE:
8948	useless = "a literal";
8949	break;
8950      case NODE_COLON2:
8951      case NODE_COLON3:
8952	useless = "::";
8953	break;
8954      case NODE_DOT2:
8955	useless = "..";
8956	break;
8957      case NODE_DOT3:
8958	useless = "...";
8959	break;
8960      case NODE_SELF:
8961	useless = "self";
8962	break;
8963      case NODE_NIL:
8964	useless = "nil";
8965	break;
8966      case NODE_TRUE:
8967	useless = "true";
8968	break;
8969      case NODE_FALSE:
8970	useless = "false";
8971	break;
8972      case NODE_DEFINED:
8973	useless = "defined?";
8974	break;
8975    }
8976
8977    if (useless) {
8978	int line = ruby_sourceline;
8979
8980	ruby_sourceline = nd_line(node);
8981	rb_warnS("possibly useless use of %s in void context", useless);
8982	ruby_sourceline = line;
8983    }
8984}
8985
8986static void
8987void_stmts_gen(struct parser_params *parser, NODE *node)
8988{
8989    if (!RTEST(ruby_verbose)) return;
8990    if (!node) return;
8991    if (nd_type(node) != NODE_BLOCK) return;
8992
8993    for (;;) {
8994	if (!node->nd_next) return;
8995	void_expr0(node->nd_head);
8996	node = node->nd_next;
8997    }
8998}
8999
9000static NODE *
9001remove_begin(NODE *node)
9002{
9003    NODE **n = &node, *n1 = node;
9004    while (n1 && nd_type(n1) == NODE_BEGIN && n1->nd_body) {
9005	*n = n1 = n1->nd_body;
9006    }
9007    return node;
9008}
9009
9010static void
9011reduce_nodes_gen(struct parser_params *parser, NODE **body)
9012{
9013    NODE *node = *body;
9014
9015    if (!node) {
9016	*body = NEW_NIL();
9017	return;
9018    }
9019#define subnodes(n1, n2) \
9020    ((!node->n1) ? (node->n2 ? (body = &node->n2, 1) : 0) : \
9021     (!node->n2) ? (body = &node->n1, 1) : \
9022     (reduce_nodes(&node->n1), body = &node->n2, 1))
9023
9024    while (node) {
9025	int newline = (int)(node->flags & NODE_FL_NEWLINE);
9026	switch (nd_type(node)) {
9027	  end:
9028	  case NODE_NIL:
9029	    *body = 0;
9030	    return;
9031	  case NODE_RETURN:
9032	    *body = node = node->nd_stts;
9033	    if (newline && node) node->flags |= NODE_FL_NEWLINE;
9034	    continue;
9035	  case NODE_BEGIN:
9036	    *body = node = node->nd_body;
9037	    if (newline && node) node->flags |= NODE_FL_NEWLINE;
9038	    continue;
9039	  case NODE_BLOCK:
9040	    body = &node->nd_end->nd_head;
9041	    break;
9042	  case NODE_IF:
9043	    if (subnodes(nd_body, nd_else)) break;
9044	    return;
9045	  case NODE_CASE:
9046	    body = &node->nd_body;
9047	    break;
9048	  case NODE_WHEN:
9049	    if (!subnodes(nd_body, nd_next)) goto end;
9050	    break;
9051	  case NODE_ENSURE:
9052	    if (!subnodes(nd_head, nd_resq)) goto end;
9053	    break;
9054	  case NODE_RESCUE:
9055	    if (node->nd_else) {
9056		body = &node->nd_resq;
9057		break;
9058	    }
9059	    if (!subnodes(nd_head, nd_resq)) goto end;
9060	    break;
9061	  default:
9062	    return;
9063	}
9064	node = *body;
9065	if (newline && node) node->flags |= NODE_FL_NEWLINE;
9066    }
9067
9068#undef subnodes
9069}
9070
9071static int
9072is_static_content(NODE *node)
9073{
9074    if (!node) return 1;
9075    switch (nd_type(node)) {
9076      case NODE_HASH:
9077	if (!(node = node->nd_head)) break;
9078      case NODE_ARRAY:
9079	do {
9080	    if (!is_static_content(node->nd_head)) return 0;
9081	} while ((node = node->nd_next) != 0);
9082      case NODE_LIT:
9083      case NODE_STR:
9084      case NODE_NIL:
9085      case NODE_TRUE:
9086      case NODE_FALSE:
9087      case NODE_ZARRAY:
9088	break;
9089      default:
9090	return 0;
9091    }
9092    return 1;
9093}
9094
9095static int
9096assign_in_cond(struct parser_params *parser, NODE *node)
9097{
9098    switch (nd_type(node)) {
9099      case NODE_MASGN:
9100	yyerror("multiple assignment in conditional");
9101	return 1;
9102
9103      case NODE_LASGN:
9104      case NODE_DASGN:
9105      case NODE_DASGN_CURR:
9106      case NODE_GASGN:
9107      case NODE_IASGN:
9108	break;
9109
9110      default:
9111	return 0;
9112    }
9113
9114    if (!node->nd_value) return 1;
9115    if (is_static_content(node->nd_value)) {
9116	/* reports always */
9117	parser_warn(node->nd_value, "found = in conditional, should be ==");
9118    }
9119    return 1;
9120}
9121
9122static void
9123warn_unless_e_option(struct parser_params *parser, NODE *node, const char *str)
9124{
9125    if (!e_option_supplied(parser)) parser_warn(node, str);
9126}
9127
9128static void
9129warning_unless_e_option(struct parser_params *parser, NODE *node, const char *str)
9130{
9131    if (!e_option_supplied(parser)) parser_warning(node, str);
9132}
9133
9134static void
9135fixup_nodes(NODE **rootnode)
9136{
9137    NODE *node, *next, *head;
9138
9139    for (node = *rootnode; node; node = next) {
9140	enum node_type type;
9141	VALUE val;
9142
9143	next = node->nd_next;
9144	head = node->nd_head;
9145	rb_gc_force_recycle((VALUE)node);
9146	*rootnode = next;
9147	switch (type = nd_type(head)) {
9148	  case NODE_DOT2:
9149	  case NODE_DOT3:
9150	    val = rb_range_new(head->nd_beg->nd_lit, head->nd_end->nd_lit,
9151			       type == NODE_DOT3);
9152	    rb_gc_force_recycle((VALUE)head->nd_beg);
9153	    rb_gc_force_recycle((VALUE)head->nd_end);
9154	    nd_set_type(head, NODE_LIT);
9155	    head->nd_lit = val;
9156	    break;
9157	  default:
9158	    break;
9159	}
9160    }
9161}
9162
9163static NODE *cond0(struct parser_params*,NODE*);
9164
9165static NODE*
9166range_op(struct parser_params *parser, NODE *node)
9167{
9168    enum node_type type;
9169
9170    if (node == 0) return 0;
9171
9172    type = nd_type(node);
9173    value_expr(node);
9174    if (type == NODE_LIT && FIXNUM_P(node->nd_lit)) {
9175	warn_unless_e_option(parser, node, "integer literal in conditional range");
9176	return NEW_CALL(node, tEQ, NEW_LIST(NEW_GVAR(rb_intern("$."))));
9177    }
9178    return cond0(parser, node);
9179}
9180
9181static int
9182literal_node(NODE *node)
9183{
9184    if (!node) return 1;	/* same as NODE_NIL */
9185    switch (nd_type(node)) {
9186      case NODE_LIT:
9187      case NODE_STR:
9188      case NODE_DSTR:
9189      case NODE_EVSTR:
9190      case NODE_DREGX:
9191      case NODE_DREGX_ONCE:
9192      case NODE_DSYM:
9193	return 2;
9194      case NODE_TRUE:
9195      case NODE_FALSE:
9196      case NODE_NIL:
9197	return 1;
9198    }
9199    return 0;
9200}
9201
9202static NODE*
9203cond0(struct parser_params *parser, NODE *node)
9204{
9205    if (node == 0) return 0;
9206    assign_in_cond(parser, node);
9207
9208    switch (nd_type(node)) {
9209      case NODE_DSTR:
9210      case NODE_EVSTR:
9211      case NODE_STR:
9212	rb_warn0("string literal in condition");
9213	break;
9214
9215      case NODE_DREGX:
9216      case NODE_DREGX_ONCE:
9217	warning_unless_e_option(parser, node, "regex literal in condition");
9218	return NEW_MATCH2(node, NEW_GVAR(rb_intern("$_")));
9219
9220      case NODE_AND:
9221      case NODE_OR:
9222	node->nd_1st = cond0(parser, node->nd_1st);
9223	node->nd_2nd = cond0(parser, node->nd_2nd);
9224	break;
9225
9226      case NODE_DOT2:
9227      case NODE_DOT3:
9228	node->nd_beg = range_op(parser, node->nd_beg);
9229	node->nd_end = range_op(parser, node->nd_end);
9230	if (nd_type(node) == NODE_DOT2) nd_set_type(node,NODE_FLIP2);
9231	else if (nd_type(node) == NODE_DOT3) nd_set_type(node, NODE_FLIP3);
9232	if (!e_option_supplied(parser)) {
9233	    int b = literal_node(node->nd_beg);
9234	    int e = literal_node(node->nd_end);
9235	    if ((b == 1 && e == 1) || (b + e >= 2 && RTEST(ruby_verbose))) {
9236		parser_warn(node, "range literal in condition");
9237	    }
9238	}
9239	break;
9240
9241      case NODE_DSYM:
9242	parser_warning(node, "literal in condition");
9243	break;
9244
9245      case NODE_LIT:
9246	if (RB_TYPE_P(node->nd_lit, T_REGEXP)) {
9247	    warn_unless_e_option(parser, node, "regex literal in condition");
9248	    nd_set_type(node, NODE_MATCH);
9249	}
9250	else {
9251	    parser_warning(node, "literal in condition");
9252	}
9253      default:
9254	break;
9255    }
9256    return node;
9257}
9258
9259static NODE*
9260cond_gen(struct parser_params *parser, NODE *node)
9261{
9262    if (node == 0) return 0;
9263    return cond0(parser, node);
9264}
9265
9266static NODE*
9267logop_gen(struct parser_params *parser, enum node_type type, NODE *left, NODE *right)
9268{
9269    value_expr(left);
9270    if (left && (enum node_type)nd_type(left) == type) {
9271	NODE *node = left, *second;
9272	while ((second = node->nd_2nd) != 0 && (enum node_type)nd_type(second) == type) {
9273	    node = second;
9274	}
9275	node->nd_2nd = NEW_NODE(type, second, right, 0);
9276	return left;
9277    }
9278    return NEW_NODE(type, left, right, 0);
9279}
9280
9281static void
9282no_blockarg(struct parser_params *parser, NODE *node)
9283{
9284    if (node && nd_type(node) == NODE_BLOCK_PASS) {
9285	compile_error(PARSER_ARG "block argument should not be given");
9286    }
9287}
9288
9289static NODE *
9290ret_args_gen(struct parser_params *parser, NODE *node)
9291{
9292    if (node) {
9293	no_blockarg(parser, node);
9294	if (nd_type(node) == NODE_ARRAY) {
9295	    if (node->nd_next == 0) {
9296		node = node->nd_head;
9297	    }
9298	    else {
9299		nd_set_type(node, NODE_VALUES);
9300	    }
9301	}
9302    }
9303    return node;
9304}
9305
9306static NODE *
9307new_yield_gen(struct parser_params *parser, NODE *node)
9308{
9309    if (node) no_blockarg(parser, node);
9310
9311    return NEW_YIELD(node);
9312}
9313
9314static NODE*
9315negate_lit(NODE *node)
9316{
9317    switch (TYPE(node->nd_lit)) {
9318      case T_FIXNUM:
9319	node->nd_lit = LONG2FIX(-FIX2LONG(node->nd_lit));
9320	break;
9321      case T_BIGNUM:
9322	node->nd_lit = rb_funcall(node->nd_lit,tUMINUS,0,0);
9323	break;
9324      case T_FLOAT:
9325#if USE_FLONUM
9326	if (FLONUM_P(node->nd_lit)) {
9327	    node->nd_lit = DBL2NUM(-RFLOAT_VALUE(node->nd_lit));
9328	}
9329	else {
9330	    RFLOAT(node->nd_lit)->float_value = -RFLOAT_VALUE(node->nd_lit);
9331	}
9332#else
9333	RFLOAT(node->nd_lit)->float_value = -RFLOAT_VALUE(node->nd_lit);
9334#endif
9335	break;
9336      default:
9337	break;
9338    }
9339    return node;
9340}
9341
9342static NODE *
9343arg_blk_pass(NODE *node1, NODE *node2)
9344{
9345    if (node2) {
9346	node2->nd_head = node1;
9347	return node2;
9348    }
9349    return node1;
9350}
9351
9352
9353static NODE*
9354new_args_gen(struct parser_params *parser, NODE *m, NODE *o, ID r, NODE *p, NODE *tail)
9355{
9356    int saved_line = ruby_sourceline;
9357    struct rb_args_info *args = tail->nd_ainfo;
9358
9359    args->pre_args_num   = m ? rb_long2int(m->nd_plen) : 0;
9360    args->pre_init       = m ? m->nd_next : 0;
9361
9362    args->post_args_num  = p ? rb_long2int(p->nd_plen) : 0;
9363    args->post_init      = p ? p->nd_next : 0;
9364    args->first_post_arg = p ? p->nd_pid : 0;
9365
9366    args->rest_arg       = r;
9367
9368    args->opt_args       = o;
9369
9370    ruby_sourceline = saved_line;
9371
9372    return tail;
9373}
9374
9375static NODE*
9376new_args_tail_gen(struct parser_params *parser, NODE *k, ID kr, ID b)
9377{
9378    int saved_line = ruby_sourceline;
9379    struct rb_args_info *args;
9380    NODE *kw_rest_arg = 0;
9381    NODE *node;
9382
9383    args = ALLOC(struct rb_args_info);
9384    MEMZERO(args, struct rb_args_info, 1);
9385    node = NEW_NODE(NODE_ARGS, 0, 0, args);
9386
9387    args->block_arg      = b;
9388    args->kw_args        = k;
9389    if (k && !kr) kr = internal_id();
9390    if (kr) {
9391	arg_var(kr);
9392	kw_rest_arg  = NEW_DVAR(kr);
9393    }
9394    args->kw_rest_arg    = kw_rest_arg;
9395
9396    ruby_sourceline = saved_line;
9397    return node;
9398}
9399
9400static NODE*
9401dsym_node_gen(struct parser_params *parser, NODE *node)
9402{
9403    VALUE lit;
9404
9405    if (!node) {
9406	return NEW_LIT(ID2SYM(idNULL));
9407    }
9408
9409    switch (nd_type(node)) {
9410      case NODE_DSTR:
9411	nd_set_type(node, NODE_DSYM);
9412	break;
9413      case NODE_STR:
9414	lit = node->nd_lit;
9415	node->nd_lit = ID2SYM(rb_intern_str(lit));
9416	nd_set_type(node, NODE_LIT);
9417	break;
9418      default:
9419	node = NEW_NODE(NODE_DSYM, Qnil, 1, NEW_LIST(node));
9420	break;
9421    }
9422    return node;
9423}
9424#endif /* !RIPPER */
9425
9426#ifndef RIPPER
9427static NODE *
9428new_op_assign_gen(struct parser_params *parser, NODE *lhs, ID op, NODE *rhs)
9429{
9430    NODE *asgn;
9431
9432    if (lhs) {
9433	ID vid = lhs->nd_vid;
9434	if (op == tOROP) {
9435	    lhs->nd_value = rhs;
9436	    asgn = NEW_OP_ASGN_OR(gettable(vid), lhs);
9437	    if (is_asgn_or_id(vid)) {
9438		asgn->nd_aid = vid;
9439	    }
9440	}
9441	else if (op == tANDOP) {
9442	    lhs->nd_value = rhs;
9443	    asgn = NEW_OP_ASGN_AND(gettable(vid), lhs);
9444	}
9445	else {
9446	    asgn = lhs;
9447	    asgn->nd_value = NEW_CALL(gettable(vid), op, NEW_LIST(rhs));
9448	}
9449    }
9450    else {
9451	asgn = NEW_BEGIN(0);
9452    }
9453    return asgn;
9454}
9455
9456static NODE *
9457new_attr_op_assign_gen(struct parser_params *parser, NODE *lhs, ID attr, ID op, NODE *rhs)
9458{
9459    NODE *asgn;
9460
9461    if (op == tOROP) {
9462	op = 0;
9463    }
9464    else if (op == tANDOP) {
9465	op = 1;
9466    }
9467    asgn = NEW_OP_ASGN2(lhs, attr, op, rhs);
9468    fixpos(asgn, lhs);
9469    return asgn;
9470}
9471
9472static NODE *
9473new_const_op_assign_gen(struct parser_params *parser, NODE *lhs, ID op, NODE *rhs)
9474{
9475    NODE *asgn;
9476
9477    if (op == tOROP) {
9478	op = 0;
9479    }
9480    else if (op == tANDOP) {
9481	op = 1;
9482    }
9483    if (lhs) {
9484	asgn = NEW_OP_CDECL(lhs, op, rhs);
9485    }
9486    else {
9487	asgn = NEW_BEGIN(0);
9488    }
9489    fixpos(asgn, lhs);
9490    return asgn;
9491}
9492#else
9493static VALUE
9494new_op_assign_gen(struct parser_params *parser, VALUE lhs, VALUE op, VALUE rhs)
9495{
9496    return dispatch3(opassign, lhs, op, rhs);
9497}
9498
9499static VALUE
9500new_attr_op_assign_gen(struct parser_params *parser, VALUE lhs, VALUE type, VALUE attr, VALUE op, VALUE rhs)
9501{
9502    VALUE recv = dispatch3(field, lhs, type, attr);
9503    return dispatch3(opassign, recv, op, rhs);
9504}
9505#endif
9506
9507static void
9508warn_unused_var(struct parser_params *parser, struct local_vars *local)
9509{
9510    int i, cnt;
9511    ID *v, *u;
9512
9513    if (!local->used) return;
9514    v = local->vars->tbl;
9515    u = local->used->tbl;
9516    cnt = local->used->pos;
9517    if (cnt != local->vars->pos) {
9518	rb_bug("local->used->pos != local->vars->pos");
9519    }
9520    for (i = 0; i < cnt; ++i) {
9521	if (!v[i] || (u[i] & LVAR_USED)) continue;
9522	if (is_private_local_id(v[i])) continue;
9523	rb_warn4S(ruby_sourcefile, (int)u[i], "assigned but unused variable - %s", rb_id2name(v[i]));
9524    }
9525}
9526
9527static void
9528local_push_gen(struct parser_params *parser, int inherit_dvars)
9529{
9530    struct local_vars *local;
9531
9532    local = ALLOC(struct local_vars);
9533    local->prev = lvtbl;
9534    local->args = vtable_alloc(0);
9535    local->vars = vtable_alloc(inherit_dvars ? DVARS_INHERIT : DVARS_TOPSCOPE);
9536    local->used = !(inherit_dvars &&
9537		    (ifndef_ripper(compile_for_eval || e_option_supplied(parser))+0)) &&
9538	RTEST(ruby_verbose) ? vtable_alloc(0) : 0;
9539    local->cmdargs = cmdarg_stack;
9540    cmdarg_stack = 0;
9541    lvtbl = local;
9542}
9543
9544static void
9545local_pop_gen(struct parser_params *parser)
9546{
9547    struct local_vars *local = lvtbl->prev;
9548    if (lvtbl->used) {
9549	warn_unused_var(parser, lvtbl);
9550	vtable_free(lvtbl->used);
9551    }
9552    vtable_free(lvtbl->args);
9553    vtable_free(lvtbl->vars);
9554    cmdarg_stack = lvtbl->cmdargs;
9555    xfree(lvtbl);
9556    lvtbl = local;
9557}
9558
9559#ifndef RIPPER
9560static ID*
9561vtable_tblcpy(ID *buf, const struct vtable *src)
9562{
9563    int i, cnt = vtable_size(src);
9564
9565    if (cnt > 0) {
9566        buf[0] = cnt;
9567        for (i = 0; i < cnt; i++) {
9568            buf[i] = src->tbl[i];
9569        }
9570        return buf;
9571    }
9572    return 0;
9573}
9574
9575static ID*
9576local_tbl_gen(struct parser_params *parser)
9577{
9578    int cnt = vtable_size(lvtbl->args) + vtable_size(lvtbl->vars);
9579    ID *buf;
9580
9581    if (cnt <= 0) return 0;
9582    buf = ALLOC_N(ID, cnt + 1);
9583    vtable_tblcpy(buf+1, lvtbl->args);
9584    vtable_tblcpy(buf+vtable_size(lvtbl->args)+1, lvtbl->vars);
9585    buf[0] = cnt;
9586    return buf;
9587}
9588#endif
9589
9590static int
9591arg_var_gen(struct parser_params *parser, ID id)
9592{
9593    vtable_add(lvtbl->args, id);
9594    return vtable_size(lvtbl->args) - 1;
9595}
9596
9597static int
9598local_var_gen(struct parser_params *parser, ID id)
9599{
9600    vtable_add(lvtbl->vars, id);
9601    if (lvtbl->used) {
9602	vtable_add(lvtbl->used, (ID)ruby_sourceline);
9603    }
9604    return vtable_size(lvtbl->vars) - 1;
9605}
9606
9607static int
9608local_id_gen(struct parser_params *parser, ID id)
9609{
9610    struct vtable *vars, *args, *used;
9611
9612    vars = lvtbl->vars;
9613    args = lvtbl->args;
9614    used = lvtbl->used;
9615
9616    while (vars && POINTER_P(vars->prev)) {
9617	vars = vars->prev;
9618	args = args->prev;
9619	if (used) used = used->prev;
9620    }
9621
9622    if (vars && vars->prev == DVARS_INHERIT) {
9623	return rb_local_defined(id);
9624    }
9625    else if (vtable_included(args, id)) {
9626	return 1;
9627    }
9628    else {
9629	int i = vtable_included(vars, id);
9630	if (i && used) used->tbl[i-1] |= LVAR_USED;
9631	return i != 0;
9632    }
9633}
9634
9635static const struct vtable *
9636dyna_push_gen(struct parser_params *parser)
9637{
9638    lvtbl->args = vtable_alloc(lvtbl->args);
9639    lvtbl->vars = vtable_alloc(lvtbl->vars);
9640    if (lvtbl->used) {
9641	lvtbl->used = vtable_alloc(lvtbl->used);
9642    }
9643    return lvtbl->args;
9644}
9645
9646static void
9647dyna_pop_1(struct parser_params *parser)
9648{
9649    struct vtable *tmp;
9650
9651    if ((tmp = lvtbl->used) != 0) {
9652	warn_unused_var(parser, lvtbl);
9653	lvtbl->used = lvtbl->used->prev;
9654	vtable_free(tmp);
9655    }
9656    tmp = lvtbl->args;
9657    lvtbl->args = lvtbl->args->prev;
9658    vtable_free(tmp);
9659    tmp = lvtbl->vars;
9660    lvtbl->vars = lvtbl->vars->prev;
9661    vtable_free(tmp);
9662}
9663
9664static void
9665dyna_pop_gen(struct parser_params *parser, const struct vtable *lvargs)
9666{
9667    while (lvtbl->args != lvargs) {
9668	dyna_pop_1(parser);
9669	if (!lvtbl->args) {
9670	    struct local_vars *local = lvtbl->prev;
9671	    xfree(lvtbl);
9672	    lvtbl = local;
9673	}
9674    }
9675    dyna_pop_1(parser);
9676}
9677
9678static int
9679dyna_in_block_gen(struct parser_params *parser)
9680{
9681    return POINTER_P(lvtbl->vars) && lvtbl->vars->prev != DVARS_TOPSCOPE;
9682}
9683
9684static int
9685dvar_defined_gen(struct parser_params *parser, ID id, int get)
9686{
9687    struct vtable *vars, *args, *used;
9688    int i;
9689
9690    args = lvtbl->args;
9691    vars = lvtbl->vars;
9692    used = lvtbl->used;
9693
9694    while (POINTER_P(vars)) {
9695	if (vtable_included(args, id)) {
9696	    return 1;
9697	}
9698	if ((i = vtable_included(vars, id)) != 0) {
9699	    if (used) used->tbl[i-1] |= LVAR_USED;
9700	    return 1;
9701	}
9702	args = args->prev;
9703	vars = vars->prev;
9704	if (get) used = 0;
9705	if (used) used = used->prev;
9706    }
9707
9708    if (vars == DVARS_INHERIT) {
9709        return rb_dvar_defined(id);
9710    }
9711
9712    return 0;
9713}
9714
9715static int
9716dvar_curr_gen(struct parser_params *parser, ID id)
9717{
9718    return (vtable_included(lvtbl->args, id) ||
9719	    vtable_included(lvtbl->vars, id));
9720}
9721
9722#ifndef RIPPER
9723static void
9724reg_fragment_setenc_gen(struct parser_params* parser, VALUE str, int options)
9725{
9726    int c = RE_OPTION_ENCODING_IDX(options);
9727
9728    if (c) {
9729	int opt, idx;
9730	rb_char_to_option_kcode(c, &opt, &idx);
9731	if (idx != ENCODING_GET(str) &&
9732	    rb_enc_str_coderange(str) != ENC_CODERANGE_7BIT) {
9733            goto error;
9734	}
9735	ENCODING_SET(str, idx);
9736    }
9737    else if (RE_OPTION_ENCODING_NONE(options)) {
9738        if (!ENCODING_IS_ASCII8BIT(str) &&
9739            rb_enc_str_coderange(str) != ENC_CODERANGE_7BIT) {
9740            c = 'n';
9741            goto error;
9742        }
9743	rb_enc_associate(str, rb_ascii8bit_encoding());
9744    }
9745    else if (current_enc == rb_usascii_encoding()) {
9746	if (rb_enc_str_coderange(str) != ENC_CODERANGE_7BIT) {
9747	    /* raise in re.c */
9748	    rb_enc_associate(str, rb_usascii_encoding());
9749	}
9750	else {
9751	    rb_enc_associate(str, rb_ascii8bit_encoding());
9752	}
9753    }
9754    return;
9755
9756  error:
9757    compile_error(PARSER_ARG
9758        "regexp encoding option '%c' differs from source encoding '%s'",
9759        c, rb_enc_name(rb_enc_get(str)));
9760}
9761
9762static int
9763reg_fragment_check_gen(struct parser_params* parser, VALUE str, int options)
9764{
9765    VALUE err;
9766    reg_fragment_setenc(str, options);
9767    err = rb_reg_check_preprocess(str);
9768    if (err != Qnil) {
9769        err = rb_obj_as_string(err);
9770        compile_error(PARSER_ARG "%s", RSTRING_PTR(err));
9771	RB_GC_GUARD(err);
9772	return 0;
9773    }
9774    return 1;
9775}
9776
9777typedef struct {
9778    struct parser_params* parser;
9779    rb_encoding *enc;
9780    NODE *succ_block;
9781    NODE *fail_block;
9782    int num;
9783} reg_named_capture_assign_t;
9784
9785static int
9786reg_named_capture_assign_iter(const OnigUChar *name, const OnigUChar *name_end,
9787          int back_num, int *back_refs, OnigRegex regex, void *arg0)
9788{
9789    reg_named_capture_assign_t *arg = (reg_named_capture_assign_t*)arg0;
9790    struct parser_params* parser = arg->parser;
9791    rb_encoding *enc = arg->enc;
9792    long len = name_end - name;
9793    const char *s = (const char *)name;
9794    ID var;
9795
9796    arg->num++;
9797
9798    if (arg->succ_block == 0) {
9799        arg->succ_block = NEW_BEGIN(0);
9800        arg->fail_block = NEW_BEGIN(0);
9801    }
9802
9803    if (!len || (*name != '_' && ISASCII(*name) && !rb_enc_islower(*name, enc)) ||
9804	(len < MAX_WORD_LENGTH && rb_reserved_word(s, (int)len)) ||
9805	!rb_enc_symname2_p(s, len, enc)) {
9806        return ST_CONTINUE;
9807    }
9808    var = rb_intern3(s, len, enc);
9809    if (dvar_defined(var) || local_id(var)) {
9810        rb_warningS("named capture conflicts a local variable - %s",
9811                    rb_id2name(var));
9812    }
9813    arg->succ_block = block_append(arg->succ_block,
9814        newline_node(node_assign(assignable(var,0),
9815            NEW_CALL(
9816              gettable(rb_intern("$~")),
9817              idAREF,
9818              NEW_LIST(NEW_LIT(ID2SYM(var))))
9819            )));
9820    arg->fail_block = block_append(arg->fail_block,
9821        newline_node(node_assign(assignable(var,0), NEW_LIT(Qnil))));
9822    return ST_CONTINUE;
9823}
9824
9825static NODE *
9826reg_named_capture_assign_gen(struct parser_params* parser, VALUE regexp, NODE *match)
9827{
9828    reg_named_capture_assign_t arg;
9829
9830    arg.parser = parser;
9831    arg.enc = rb_enc_get(regexp);
9832    arg.succ_block = 0;
9833    arg.fail_block = 0;
9834    arg.num = 0;
9835    onig_foreach_name(RREGEXP(regexp)->ptr, reg_named_capture_assign_iter, (void*)&arg);
9836
9837    if (arg.num == 0)
9838        return match;
9839
9840    return
9841        block_append(
9842            newline_node(match),
9843            NEW_IF(gettable(rb_intern("$~")),
9844                block_append(
9845                    newline_node(arg.succ_block),
9846                    newline_node(
9847                        NEW_CALL(
9848                          gettable(rb_intern("$~")),
9849                          rb_intern("begin"),
9850                          NEW_LIST(NEW_LIT(INT2FIX(0)))))),
9851                block_append(
9852                    newline_node(arg.fail_block),
9853                    newline_node(
9854                        NEW_LIT(Qnil)))));
9855}
9856
9857static VALUE
9858reg_compile_gen(struct parser_params* parser, VALUE str, int options)
9859{
9860    VALUE re;
9861    VALUE err;
9862
9863    reg_fragment_setenc(str, options);
9864    err = rb_errinfo();
9865    re = rb_reg_compile(str, options & RE_OPTION_MASK, ruby_sourcefile, ruby_sourceline);
9866    if (NIL_P(re)) {
9867	ID mesg = rb_intern("mesg");
9868	VALUE m = rb_attr_get(rb_errinfo(), mesg);
9869	rb_set_errinfo(err);
9870	if (!NIL_P(err)) {
9871	    rb_str_append(rb_str_cat(rb_attr_get(err, mesg), "\n", 1), m);
9872	}
9873	else {
9874	    compile_error(PARSER_ARG "%s", RSTRING_PTR(m));
9875	}
9876	return Qnil;
9877    }
9878    return re;
9879}
9880
9881void
9882rb_gc_mark_parser(void)
9883{
9884}
9885
9886NODE*
9887rb_parser_append_print(VALUE vparser, NODE *node)
9888{
9889    NODE *prelude = 0;
9890    NODE *scope = node;
9891    struct parser_params *parser;
9892
9893    if (!node) return node;
9894
9895    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
9896
9897    node = node->nd_body;
9898
9899    if (nd_type(node) == NODE_PRELUDE) {
9900	prelude = node;
9901	node = node->nd_body;
9902    }
9903
9904    node = block_append(node,
9905			NEW_FCALL(rb_intern("print"),
9906				  NEW_ARRAY(NEW_GVAR(rb_intern("$_")))));
9907    if (prelude) {
9908	prelude->nd_body = node;
9909	scope->nd_body = prelude;
9910    }
9911    else {
9912	scope->nd_body = node;
9913    }
9914
9915    return scope;
9916}
9917
9918NODE *
9919rb_parser_while_loop(VALUE vparser, NODE *node, int chop, int split)
9920{
9921    NODE *prelude = 0;
9922    NODE *scope = node;
9923    struct parser_params *parser;
9924
9925    if (!node) return node;
9926
9927    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
9928
9929    node = node->nd_body;
9930
9931    if (nd_type(node) == NODE_PRELUDE) {
9932	prelude = node;
9933	node = node->nd_body;
9934    }
9935    if (split) {
9936	node = block_append(NEW_GASGN(rb_intern("$F"),
9937				      NEW_CALL(NEW_GVAR(rb_intern("$_")),
9938					       rb_intern("split"), 0)),
9939			    node);
9940    }
9941    if (chop) {
9942	node = block_append(NEW_CALL(NEW_GVAR(rb_intern("$_")),
9943				     rb_intern("chop!"), 0), node);
9944    }
9945
9946    node = NEW_OPT_N(node);
9947
9948    if (prelude) {
9949	prelude->nd_body = node;
9950	scope->nd_body = prelude;
9951    }
9952    else {
9953	scope->nd_body = node;
9954    }
9955
9956    return scope;
9957}
9958
9959static const struct {
9960    ID token;
9961    const char *name;
9962} op_tbl[] = {
9963    {tDOT2,	".."},
9964    {tDOT3,	"..."},
9965    {tPOW,	"**"},
9966    {tDSTAR,	"**"},
9967    {tUPLUS,	"+@"},
9968    {tUMINUS,	"-@"},
9969    {tCMP,	"<=>"},
9970    {tGEQ,	">="},
9971    {tLEQ,	"<="},
9972    {tEQ,	"=="},
9973    {tEQQ,	"==="},
9974    {tNEQ,	"!="},
9975    {tMATCH,	"=~"},
9976    {tNMATCH,	"!~"},
9977    {tAREF,	"[]"},
9978    {tASET,	"[]="},
9979    {tLSHFT,	"<<"},
9980    {tRSHFT,	">>"},
9981    {tCOLON2,	"::"},
9982};
9983
9984#define op_tbl_count numberof(op_tbl)
9985
9986#ifndef ENABLE_SELECTOR_NAMESPACE
9987#define ENABLE_SELECTOR_NAMESPACE 0
9988#endif
9989
9990static struct symbols {
9991    ID last_id;
9992    st_table *sym_id;
9993    st_table *id_str;
9994#if ENABLE_SELECTOR_NAMESPACE
9995    st_table *ivar2_id;
9996    st_table *id_ivar2;
9997#endif
9998    VALUE op_sym[tLAST_OP_ID];
9999} global_symbols = {tLAST_TOKEN};
10000
10001static const struct st_hash_type symhash = {
10002    rb_str_hash_cmp,
10003    rb_str_hash,
10004};
10005
10006#if ENABLE_SELECTOR_NAMESPACE
10007struct ivar2_key {
10008    ID id;
10009    VALUE klass;
10010};
10011
10012static int
10013ivar2_cmp(struct ivar2_key *key1, struct ivar2_key *key2)
10014{
10015    if (key1->id == key2->id && key1->klass == key2->klass) {
10016	return 0;
10017    }
10018    return 1;
10019}
10020
10021static int
10022ivar2_hash(struct ivar2_key *key)
10023{
10024    return (key->id << 8) ^ (key->klass >> 2);
10025}
10026
10027static const struct st_hash_type ivar2_hash_type = {
10028    ivar2_cmp,
10029    ivar2_hash,
10030};
10031#endif
10032
10033void
10034Init_sym(void)
10035{
10036    global_symbols.sym_id = st_init_table_with_size(&symhash, 1000);
10037    global_symbols.id_str = st_init_numtable_with_size(1000);
10038#if ENABLE_SELECTOR_NAMESPACE
10039    global_symbols.ivar2_id = st_init_table_with_size(&ivar2_hash_type, 1000);
10040    global_symbols.id_ivar2 = st_init_numtable_with_size(1000);
10041#endif
10042
10043    (void)nodetype;
10044    (void)nodeline;
10045#if PARSER_DEBUG
10046    (void)lex_state_name(-1);
10047#endif
10048
10049    Init_id();
10050}
10051
10052void
10053rb_gc_mark_symbols(void)
10054{
10055    rb_mark_tbl(global_symbols.id_str);
10056    rb_gc_mark_locations(global_symbols.op_sym,
10057			 global_symbols.op_sym + numberof(global_symbols.op_sym));
10058}
10059#endif /* !RIPPER */
10060
10061static ID
10062internal_id_gen(struct parser_params *parser)
10063{
10064    ID id = (ID)vtable_size(lvtbl->args) + (ID)vtable_size(lvtbl->vars);
10065    id += ((tLAST_TOKEN - ID_INTERNAL) >> ID_SCOPE_SHIFT) + 1;
10066    return ID_INTERNAL | (id << ID_SCOPE_SHIFT);
10067}
10068
10069#ifndef RIPPER
10070static int
10071is_special_global_name(const char *m, const char *e, rb_encoding *enc)
10072{
10073    int mb = 0;
10074
10075    if (m >= e) return 0;
10076    if (is_global_name_punct(*m)) {
10077	++m;
10078    }
10079    else if (*m == '-') {
10080	++m;
10081	if (m < e && is_identchar(m, e, enc)) {
10082	    if (!ISASCII(*m)) mb = 1;
10083	    m += rb_enc_mbclen(m, e, enc);
10084	}
10085    }
10086    else {
10087	if (!rb_enc_isdigit(*m, enc)) return 0;
10088	do {
10089	    if (!ISASCII(*m)) mb = 1;
10090	    ++m;
10091	} while (m < e && rb_enc_isdigit(*m, enc));
10092    }
10093    return m == e ? mb + 1 : 0;
10094}
10095
10096int
10097rb_symname_p(const char *name)
10098{
10099    return rb_enc_symname_p(name, rb_ascii8bit_encoding());
10100}
10101
10102int
10103rb_enc_symname_p(const char *name, rb_encoding *enc)
10104{
10105    return rb_enc_symname2_p(name, strlen(name), enc);
10106}
10107
10108#define IDSET_ATTRSET_FOR_SYNTAX ((1U<<ID_LOCAL)|(1U<<ID_CONST))
10109#define IDSET_ATTRSET_FOR_INTERN (~(~0U<<(1<<ID_SCOPE_SHIFT)) & ~(1U<<ID_ATTRSET))
10110
10111static int
10112rb_enc_symname_type(const char *name, long len, rb_encoding *enc, unsigned int allowed_atttset)
10113{
10114    const char *m = name;
10115    const char *e = m + len;
10116    int type = ID_JUNK;
10117
10118    if (!m || len <= 0) return -1;
10119    switch (*m) {
10120      case '\0':
10121	return -1;
10122
10123      case '$':
10124	type = ID_GLOBAL;
10125	if (is_special_global_name(++m, e, enc)) return type;
10126	goto id;
10127
10128      case '@':
10129	type = ID_INSTANCE;
10130	if (*++m == '@') {
10131	    ++m;
10132	    type = ID_CLASS;
10133	}
10134	goto id;
10135
10136      case '<':
10137	switch (*++m) {
10138	  case '<': ++m; break;
10139	  case '=': if (*++m == '>') ++m; break;
10140	  default: break;
10141	}
10142	break;
10143
10144      case '>':
10145	switch (*++m) {
10146	  case '>': case '=': ++m; break;
10147	}
10148	break;
10149
10150      case '=':
10151	switch (*++m) {
10152	  case '~': ++m; break;
10153	  case '=': if (*++m == '=') ++m; break;
10154	  default: return -1;
10155	}
10156	break;
10157
10158      case '*':
10159	if (*++m == '*') ++m;
10160	break;
10161
10162      case '+': case '-':
10163	if (*++m == '@') ++m;
10164	break;
10165
10166      case '|': case '^': case '&': case '/': case '%': case '~': case '`':
10167	++m;
10168	break;
10169
10170      case '[':
10171	if (*++m != ']') return -1;
10172	if (*++m == '=') ++m;
10173	break;
10174
10175      case '!':
10176	if (len == 1) return ID_JUNK;
10177	switch (*++m) {
10178	  case '=': case '~': ++m; break;
10179	  default: return -1;
10180	}
10181	break;
10182
10183      default:
10184	type = rb_enc_isupper(*m, enc) ? ID_CONST : ID_LOCAL;
10185      id:
10186	if (m >= e || (*m != '_' && !rb_enc_isalpha(*m, enc) && ISASCII(*m)))
10187	    return -1;
10188	while (m < e && is_identchar(m, e, enc)) m += rb_enc_mbclen(m, e, enc);
10189	if (m >= e) break;
10190	switch (*m) {
10191	  case '!': case '?':
10192	    if (type == ID_GLOBAL || type == ID_CLASS || type == ID_INSTANCE) return -1;
10193	    type = ID_JUNK;
10194	    ++m;
10195	    if (m + 1 < e || *m != '=') break;
10196	    /* fall through */
10197	  case '=':
10198	    if (!(allowed_atttset & (1U << type))) return -1;
10199	    type = ID_ATTRSET;
10200	    ++m;
10201	    break;
10202	}
10203	break;
10204    }
10205    return m == e ? type : -1;
10206}
10207
10208int
10209rb_enc_symname2_p(const char *name, long len, rb_encoding *enc)
10210{
10211    return rb_enc_symname_type(name, len, enc, IDSET_ATTRSET_FOR_SYNTAX) != -1;
10212}
10213
10214static int
10215rb_str_symname_type(VALUE name, unsigned int allowed_atttset)
10216{
10217    const char *ptr = StringValuePtr(name);
10218    long len = RSTRING_LEN(name);
10219    int type = rb_enc_symname_type(ptr, len, rb_enc_get(name), allowed_atttset);
10220    RB_GC_GUARD(name);
10221    return type;
10222}
10223
10224static ID
10225register_symid(ID id, const char *name, long len, rb_encoding *enc)
10226{
10227    VALUE str = rb_enc_str_new(name, len, enc);
10228    return register_symid_str(id, str);
10229}
10230
10231static ID
10232register_symid_str(ID id, VALUE str)
10233{
10234    OBJ_FREEZE(str);
10235    st_add_direct(global_symbols.sym_id, (st_data_t)str, id);
10236    st_add_direct(global_symbols.id_str, id, (st_data_t)str);
10237    return id;
10238}
10239
10240static int
10241sym_check_asciionly(VALUE str)
10242{
10243    if (!rb_enc_asciicompat(rb_enc_get(str))) return FALSE;
10244    switch (rb_enc_str_coderange(str)) {
10245      case ENC_CODERANGE_BROKEN:
10246	rb_raise(rb_eEncodingError, "invalid encoding symbol");
10247      case ENC_CODERANGE_7BIT:
10248	return TRUE;
10249    }
10250    return FALSE;
10251}
10252
10253/*
10254 * _str_ itself will be registered at the global symbol table.  _str_
10255 * can be modified before the registration, since the encoding will be
10256 * set to ASCII-8BIT if it is a special global name.
10257 */
10258static ID intern_str(VALUE str);
10259
10260ID
10261rb_intern3(const char *name, long len, rb_encoding *enc)
10262{
10263    VALUE str;
10264    st_data_t data;
10265    struct RString fake_str;
10266    fake_str.basic.flags = T_STRING|RSTRING_NOEMBED;
10267    fake_str.basic.klass = rb_cString;
10268    fake_str.as.heap.len = len;
10269    fake_str.as.heap.ptr = (char *)name;
10270    fake_str.as.heap.aux.capa = len;
10271    str = (VALUE)&fake_str;
10272    rb_enc_associate(str, enc);
10273    OBJ_FREEZE(str);
10274
10275    if (st_lookup(global_symbols.sym_id, str, &data))
10276	return (ID)data;
10277
10278    str = rb_enc_str_new(name, len, enc); /* make true string */
10279    return intern_str(str);
10280}
10281
10282static ID
10283intern_str(VALUE str)
10284{
10285    const char *name, *m, *e;
10286    long len, last;
10287    rb_encoding *enc, *symenc;
10288    unsigned char c;
10289    ID id;
10290    int mb;
10291
10292    RSTRING_GETMEM(str, name, len);
10293    m = name;
10294    e = m + len;
10295    enc = rb_enc_get(str);
10296    symenc = enc;
10297
10298    if (!len || (rb_cString && !rb_enc_asciicompat(enc))) {
10299      junk:
10300	id = ID_JUNK;
10301	goto new_id;
10302    }
10303    last = len-1;
10304    id = 0;
10305    switch (*m) {
10306      case '$':
10307	if (len < 2) goto junk;
10308	id |= ID_GLOBAL;
10309	if ((mb = is_special_global_name(++m, e, enc)) != 0) {
10310	    if (!--mb) symenc = rb_usascii_encoding();
10311	    goto new_id;
10312	}
10313	break;
10314      case '@':
10315	if (m[1] == '@') {
10316	    if (len < 3) goto junk;
10317	    m++;
10318	    id |= ID_CLASS;
10319	}
10320	else {
10321	    if (len < 2) goto junk;
10322	    id |= ID_INSTANCE;
10323	}
10324	m++;
10325	break;
10326      default:
10327	c = m[0];
10328	if (c != '_' && rb_enc_isascii(c, enc) && rb_enc_ispunct(c, enc)) {
10329	    /* operators */
10330	    int i;
10331
10332	    if (len == 1) {
10333		id = c;
10334		goto id_register;
10335	    }
10336	    for (i = 0; i < op_tbl_count; i++) {
10337		if (*op_tbl[i].name == *m &&
10338		    strcmp(op_tbl[i].name, m) == 0) {
10339		    id = op_tbl[i].token;
10340		    goto id_register;
10341		}
10342	    }
10343	}
10344	break;
10345    }
10346    if (name[last] == '=') {
10347	/* attribute assignment */
10348	if (last > 1 && name[last-1] == '=')
10349	    goto junk;
10350	id = rb_intern3(name, last, enc);
10351	if (id > tLAST_OP_ID && !is_attrset_id(id)) {
10352	    enc = rb_enc_get(rb_id2str(id));
10353	    id = rb_id_attrset(id);
10354	    goto id_register;
10355	}
10356	id = ID_ATTRSET;
10357    }
10358    else if (id == 0) {
10359	if (rb_enc_isupper(m[0], enc)) {
10360	    id = ID_CONST;
10361	}
10362	else {
10363	    id = ID_LOCAL;
10364	}
10365    }
10366    if (!rb_enc_isdigit(*m, enc)) {
10367	while (m <= name + last && is_identchar(m, e, enc)) {
10368	    if (ISASCII(*m)) {
10369		m++;
10370	    }
10371	    else {
10372		m += rb_enc_mbclen(m, e, enc);
10373	    }
10374	}
10375    }
10376    if (id != ID_ATTRSET && m - name < len) id = ID_JUNK;
10377    if (sym_check_asciionly(str)) symenc = rb_usascii_encoding();
10378  new_id:
10379    if (symenc != enc) rb_enc_associate(str, symenc);
10380    if (global_symbols.last_id >= ~(ID)0 >> (ID_SCOPE_SHIFT+RUBY_SPECIAL_SHIFT)) {
10381	if (len > 20) {
10382	    rb_raise(rb_eRuntimeError, "symbol table overflow (symbol %.20s...)",
10383		     name);
10384	}
10385	else {
10386	    rb_raise(rb_eRuntimeError, "symbol table overflow (symbol %.*s)",
10387		     (int)len, name);
10388	}
10389    }
10390    id |= ++global_symbols.last_id << ID_SCOPE_SHIFT;
10391  id_register:
10392    return register_symid_str(id, str);
10393}
10394
10395ID
10396rb_intern2(const char *name, long len)
10397{
10398    return rb_intern3(name, len, rb_usascii_encoding());
10399}
10400
10401#undef rb_intern
10402ID
10403rb_intern(const char *name)
10404{
10405    return rb_intern2(name, strlen(name));
10406}
10407
10408ID
10409rb_intern_str(VALUE str)
10410{
10411    st_data_t id;
10412
10413    if (st_lookup(global_symbols.sym_id, str, &id))
10414	return (ID)id;
10415    return intern_str(rb_str_dup(str));
10416}
10417
10418VALUE
10419rb_id2str(ID id)
10420{
10421    st_data_t data;
10422
10423    if (id < tLAST_TOKEN) {
10424	int i = 0;
10425
10426	if (id < INT_MAX && rb_ispunct((int)id)) {
10427	    VALUE str = global_symbols.op_sym[i = (int)id];
10428	    if (!str) {
10429		char name[2];
10430		name[0] = (char)id;
10431		name[1] = 0;
10432		str = rb_usascii_str_new(name, 1);
10433		OBJ_FREEZE(str);
10434		global_symbols.op_sym[i] = str;
10435	    }
10436	    return str;
10437	}
10438	for (i = 0; i < op_tbl_count; i++) {
10439	    if (op_tbl[i].token == id) {
10440		VALUE str = global_symbols.op_sym[i];
10441		if (!str) {
10442		    str = rb_usascii_str_new2(op_tbl[i].name);
10443		    OBJ_FREEZE(str);
10444		    global_symbols.op_sym[i] = str;
10445		}
10446		return str;
10447	    }
10448	}
10449    }
10450
10451    if (st_lookup(global_symbols.id_str, id, &data)) {
10452        VALUE str = (VALUE)data;
10453        if (RBASIC(str)->klass == 0)
10454            RBASIC(str)->klass = rb_cString;
10455	return str;
10456    }
10457
10458    if (is_attrset_id(id)) {
10459	ID id_stem = (id & ~ID_SCOPE_MASK);
10460	VALUE str;
10461
10462	do {
10463	    if (!!(str = rb_id2str(id_stem | ID_LOCAL))) break;
10464	    if (!!(str = rb_id2str(id_stem | ID_CONST))) break;
10465	    if (!!(str = rb_id2str(id_stem | ID_INSTANCE))) break;
10466	    if (!!(str = rb_id2str(id_stem | ID_GLOBAL))) break;
10467	    if (!!(str = rb_id2str(id_stem | ID_CLASS))) break;
10468	    if (!!(str = rb_id2str(id_stem | ID_JUNK))) break;
10469	    return 0;
10470	} while (0);
10471	str = rb_str_dup(str);
10472	rb_str_cat(str, "=", 1);
10473	register_symid_str(id, str);
10474	if (st_lookup(global_symbols.id_str, id, &data)) {
10475            VALUE str = (VALUE)data;
10476            if (RBASIC(str)->klass == 0)
10477                RBASIC(str)->klass = rb_cString;
10478            return str;
10479        }
10480    }
10481    return 0;
10482}
10483
10484const char *
10485rb_id2name(ID id)
10486{
10487    VALUE str = rb_id2str(id);
10488
10489    if (!str) return 0;
10490    return RSTRING_PTR(str);
10491}
10492
10493static int
10494symbols_i(VALUE sym, ID value, VALUE ary)
10495{
10496    rb_ary_push(ary, ID2SYM(value));
10497    return ST_CONTINUE;
10498}
10499
10500/*
10501 *  call-seq:
10502 *     Symbol.all_symbols    => array
10503 *
10504 *  Returns an array of all the symbols currently in Ruby's symbol
10505 *  table.
10506 *
10507 *     Symbol.all_symbols.size    #=> 903
10508 *     Symbol.all_symbols[1,20]   #=> [:floor, :ARGV, :Binding, :symlink,
10509 *                                     :chown, :EOFError, :$;, :String,
10510 *                                     :LOCK_SH, :"setuid?", :$<,
10511 *                                     :default_proc, :compact, :extend,
10512 *                                     :Tms, :getwd, :$=, :ThreadGroup,
10513 *                                     :wait2, :$>]
10514 */
10515
10516VALUE
10517rb_sym_all_symbols(void)
10518{
10519    VALUE ary = rb_ary_new2(global_symbols.sym_id->num_entries);
10520
10521    st_foreach(global_symbols.sym_id, symbols_i, ary);
10522    return ary;
10523}
10524
10525int
10526rb_is_const_id(ID id)
10527{
10528    return is_const_id(id);
10529}
10530
10531int
10532rb_is_class_id(ID id)
10533{
10534    return is_class_id(id);
10535}
10536
10537int
10538rb_is_global_id(ID id)
10539{
10540    return is_global_id(id);
10541}
10542
10543int
10544rb_is_instance_id(ID id)
10545{
10546    return is_instance_id(id);
10547}
10548
10549int
10550rb_is_attrset_id(ID id)
10551{
10552    return is_attrset_id(id);
10553}
10554
10555int
10556rb_is_local_id(ID id)
10557{
10558    return is_local_id(id);
10559}
10560
10561int
10562rb_is_junk_id(ID id)
10563{
10564    return is_junk_id(id);
10565}
10566
10567/**
10568 * Returns ID for the given name if it is interned already, or 0.
10569 *
10570 * \param namep   the pointer to the name object
10571 * \return        the ID for *namep
10572 * \pre           the object referred by \p namep must be a Symbol or
10573 *                a String, or possible to convert with to_str method.
10574 * \post          the object referred by \p namep is a Symbol or a
10575 *                String if non-zero value is returned, or is a String
10576 *                if 0 is returned.
10577 */
10578ID
10579rb_check_id(volatile VALUE *namep)
10580{
10581    st_data_t id;
10582    VALUE tmp;
10583    VALUE name = *namep;
10584
10585    if (SYMBOL_P(name)) {
10586	return SYM2ID(name);
10587    }
10588    else if (!RB_TYPE_P(name, T_STRING)) {
10589	tmp = rb_check_string_type(name);
10590	if (NIL_P(tmp)) {
10591	    tmp = rb_inspect(name);
10592	    rb_raise(rb_eTypeError, "%s is not a symbol",
10593		     RSTRING_PTR(tmp));
10594	}
10595	name = tmp;
10596	*namep = name;
10597    }
10598
10599    sym_check_asciionly(name);
10600
10601    if (st_lookup(global_symbols.sym_id, (st_data_t)name, &id))
10602	return (ID)id;
10603
10604    if (rb_is_attrset_name(name)) {
10605	struct RString fake_str;
10606	const VALUE localname = (VALUE)&fake_str;
10607	/* make local name by chopping '=' */
10608	fake_str.basic.flags = T_STRING|RSTRING_NOEMBED;
10609	fake_str.basic.klass = rb_cString;
10610	fake_str.as.heap.len = RSTRING_LEN(name) - 1;
10611	fake_str.as.heap.ptr = RSTRING_PTR(name);
10612	fake_str.as.heap.aux.capa = fake_str.as.heap.len;
10613	rb_enc_copy(localname, name);
10614	OBJ_FREEZE(localname);
10615
10616	if (st_lookup(global_symbols.sym_id, (st_data_t)localname, &id)) {
10617	    return rb_id_attrset((ID)id);
10618	}
10619	RB_GC_GUARD(name);
10620    }
10621
10622    return (ID)0;
10623}
10624
10625ID
10626rb_check_id_cstr(const char *ptr, long len, rb_encoding *enc)
10627{
10628    st_data_t id;
10629    struct RString fake_str;
10630    const VALUE name = (VALUE)&fake_str;
10631    fake_str.basic.flags = T_STRING|RSTRING_NOEMBED;
10632    fake_str.basic.klass = rb_cString;
10633    fake_str.as.heap.len = len;
10634    fake_str.as.heap.ptr = (char *)ptr;
10635    fake_str.as.heap.aux.capa = len;
10636    rb_enc_associate(name, enc);
10637
10638    sym_check_asciionly(name);
10639
10640    if (st_lookup(global_symbols.sym_id, (st_data_t)name, &id))
10641	return (ID)id;
10642
10643    if (rb_is_attrset_name(name)) {
10644	fake_str.as.heap.len = len - 1;
10645	if (st_lookup(global_symbols.sym_id, (st_data_t)name, &id)) {
10646	    return rb_id_attrset((ID)id);
10647	}
10648    }
10649
10650    return (ID)0;
10651}
10652
10653int
10654rb_is_const_name(VALUE name)
10655{
10656    return rb_str_symname_type(name, 0) == ID_CONST;
10657}
10658
10659int
10660rb_is_class_name(VALUE name)
10661{
10662    return rb_str_symname_type(name, 0) == ID_CLASS;
10663}
10664
10665int
10666rb_is_global_name(VALUE name)
10667{
10668    return rb_str_symname_type(name, 0) == ID_GLOBAL;
10669}
10670
10671int
10672rb_is_instance_name(VALUE name)
10673{
10674    return rb_str_symname_type(name, 0) == ID_INSTANCE;
10675}
10676
10677int
10678rb_is_attrset_name(VALUE name)
10679{
10680    return rb_str_symname_type(name, IDSET_ATTRSET_FOR_INTERN) == ID_ATTRSET;
10681}
10682
10683int
10684rb_is_local_name(VALUE name)
10685{
10686    return rb_str_symname_type(name, 0) == ID_LOCAL;
10687}
10688
10689int
10690rb_is_method_name(VALUE name)
10691{
10692    switch (rb_str_symname_type(name, 0)) {
10693      case ID_LOCAL: case ID_ATTRSET: case ID_JUNK:
10694	return TRUE;
10695    }
10696    return FALSE;
10697}
10698
10699int
10700rb_is_junk_name(VALUE name)
10701{
10702    return rb_str_symname_type(name, IDSET_ATTRSET_FOR_SYNTAX) == -1;
10703}
10704
10705#endif /* !RIPPER */
10706
10707static void
10708parser_initialize(struct parser_params *parser)
10709{
10710    parser->eofp = Qfalse;
10711
10712    parser->parser_lex_strterm = 0;
10713    parser->parser_cond_stack = 0;
10714    parser->parser_cmdarg_stack = 0;
10715    parser->parser_class_nest = 0;
10716    parser->parser_paren_nest = 0;
10717    parser->parser_lpar_beg = 0;
10718    parser->parser_brace_nest = 0;
10719    parser->parser_in_single = 0;
10720    parser->parser_in_def = 0;
10721    parser->parser_in_defined = 0;
10722    parser->parser_compile_for_eval = 0;
10723    parser->parser_cur_mid = 0;
10724    parser->parser_tokenbuf = NULL;
10725    parser->parser_tokidx = 0;
10726    parser->parser_toksiz = 0;
10727    parser->parser_heredoc_end = 0;
10728    parser->parser_command_start = TRUE;
10729    parser->parser_deferred_nodes = 0;
10730    parser->parser_lex_pbeg = 0;
10731    parser->parser_lex_p = 0;
10732    parser->parser_lex_pend = 0;
10733    parser->parser_lvtbl = 0;
10734    parser->parser_ruby__end__seen = 0;
10735    parser->parser_ruby_sourcefile = 0;
10736    parser->parser_ruby_sourcefile_string = Qnil;
10737#ifndef RIPPER
10738    parser->is_ripper = 0;
10739    parser->parser_eval_tree_begin = 0;
10740    parser->parser_eval_tree = 0;
10741#else
10742    parser->is_ripper = 1;
10743    parser->delayed = Qnil;
10744
10745    parser->result = Qnil;
10746    parser->parsing_thread = Qnil;
10747    parser->toplevel_p = TRUE;
10748#endif
10749#ifdef YYMALLOC
10750    parser->heap = NULL;
10751#endif
10752    parser->enc = rb_utf8_encoding();
10753}
10754
10755#ifdef RIPPER
10756#define parser_mark ripper_parser_mark
10757#define parser_free ripper_parser_free
10758#endif
10759
10760static void
10761parser_mark(void *ptr)
10762{
10763    struct parser_params *p = (struct parser_params*)ptr;
10764
10765    rb_gc_mark((VALUE)p->parser_lex_strterm);
10766    rb_gc_mark((VALUE)p->parser_deferred_nodes);
10767    rb_gc_mark(p->parser_lex_input);
10768    rb_gc_mark(p->parser_lex_lastline);
10769    rb_gc_mark(p->parser_lex_nextline);
10770    rb_gc_mark(p->parser_ruby_sourcefile_string);
10771#ifndef RIPPER
10772    rb_gc_mark((VALUE)p->parser_eval_tree_begin) ;
10773    rb_gc_mark((VALUE)p->parser_eval_tree) ;
10774    rb_gc_mark(p->debug_lines);
10775#else
10776    rb_gc_mark(p->delayed);
10777    rb_gc_mark(p->value);
10778    rb_gc_mark(p->result);
10779    rb_gc_mark(p->parsing_thread);
10780#endif
10781#ifdef YYMALLOC
10782    rb_gc_mark((VALUE)p->heap);
10783#endif
10784}
10785
10786static void
10787parser_free(void *ptr)
10788{
10789    struct parser_params *p = (struct parser_params*)ptr;
10790    struct local_vars *local, *prev;
10791
10792    if (p->parser_tokenbuf) {
10793        xfree(p->parser_tokenbuf);
10794    }
10795    for (local = p->parser_lvtbl; local; local = prev) {
10796	if (local->vars) xfree(local->vars);
10797	prev = local->prev;
10798	xfree(local);
10799    }
10800    xfree(p);
10801}
10802
10803static size_t
10804parser_memsize(const void *ptr)
10805{
10806    struct parser_params *p = (struct parser_params*)ptr;
10807    struct local_vars *local;
10808    size_t size = sizeof(*p);
10809
10810    if (!ptr) return 0;
10811    size += p->parser_toksiz;
10812    for (local = p->parser_lvtbl; local; local = local->prev) {
10813	size += sizeof(*local);
10814	if (local->vars) size += local->vars->capa * sizeof(ID);
10815    }
10816    return size;
10817}
10818
10819static
10820#ifndef RIPPER
10821const
10822#endif
10823rb_data_type_t parser_data_type = {
10824    "parser",
10825    {
10826	parser_mark,
10827	parser_free,
10828	parser_memsize,
10829    },
10830};
10831
10832#ifndef RIPPER
10833#undef rb_reserved_word
10834
10835const struct kwtable *
10836rb_reserved_word(const char *str, unsigned int len)
10837{
10838    return reserved_word(str, len);
10839}
10840
10841static struct parser_params *
10842parser_new(void)
10843{
10844    struct parser_params *p;
10845
10846    p = ALLOC_N(struct parser_params, 1);
10847    MEMZERO(p, struct parser_params, 1);
10848    parser_initialize(p);
10849    return p;
10850}
10851
10852VALUE
10853rb_parser_new(void)
10854{
10855    struct parser_params *p = parser_new();
10856
10857    return TypedData_Wrap_Struct(0, &parser_data_type, p);
10858}
10859
10860/*
10861 *  call-seq:
10862 *    ripper#end_seen?   -> Boolean
10863 *
10864 *  Return true if parsed source ended by +\_\_END\_\_+.
10865 */
10866VALUE
10867rb_parser_end_seen_p(VALUE vparser)
10868{
10869    struct parser_params *parser;
10870
10871    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
10872    return ruby__end__seen ? Qtrue : Qfalse;
10873}
10874
10875/*
10876 *  call-seq:
10877 *    ripper#encoding   -> encoding
10878 *
10879 *  Return encoding of the source.
10880 */
10881VALUE
10882rb_parser_encoding(VALUE vparser)
10883{
10884    struct parser_params *parser;
10885
10886    TypedData_Get_Struct(vparser, struct parser_params, &parser_data_type, parser);
10887    return rb_enc_from_encoding(current_enc);
10888}
10889
10890/*
10891 *  call-seq:
10892 *    ripper.yydebug   -> true or false
10893 *
10894 *  Get yydebug.
10895 */
10896VALUE
10897rb_parser_get_yydebug(VALUE self)
10898{
10899    struct parser_params *parser;
10900
10901    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
10902    return yydebug ? Qtrue : Qfalse;
10903}
10904
10905/*
10906 *  call-seq:
10907 *    ripper.yydebug = flag
10908 *
10909 *  Set yydebug.
10910 */
10911VALUE
10912rb_parser_set_yydebug(VALUE self, VALUE flag)
10913{
10914    struct parser_params *parser;
10915
10916    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
10917    yydebug = RTEST(flag);
10918    return flag;
10919}
10920
10921#ifdef YYMALLOC
10922#define HEAPCNT(n, size) ((n) * (size) / sizeof(YYSTYPE))
10923#define NEWHEAP() rb_node_newnode(NODE_ALLOCA, 0, (VALUE)parser->heap, 0)
10924#define ADD2HEAP(n, c, p) ((parser->heap = (n))->u1.node = (p), \
10925			   (n)->u3.cnt = (c), (p))
10926
10927void *
10928rb_parser_malloc(struct parser_params *parser, size_t size)
10929{
10930    size_t cnt = HEAPCNT(1, size);
10931    NODE *n = NEWHEAP();
10932    void *ptr = xmalloc(size);
10933
10934    return ADD2HEAP(n, cnt, ptr);
10935}
10936
10937void *
10938rb_parser_calloc(struct parser_params *parser, size_t nelem, size_t size)
10939{
10940    size_t cnt = HEAPCNT(nelem, size);
10941    NODE *n = NEWHEAP();
10942    void *ptr = xcalloc(nelem, size);
10943
10944    return ADD2HEAP(n, cnt, ptr);
10945}
10946
10947void *
10948rb_parser_realloc(struct parser_params *parser, void *ptr, size_t size)
10949{
10950    NODE *n;
10951    size_t cnt = HEAPCNT(1, size);
10952
10953    if (ptr && (n = parser->heap) != NULL) {
10954	do {
10955	    if (n->u1.node == ptr) {
10956		n->u1.node = ptr = xrealloc(ptr, size);
10957		if (n->u3.cnt) n->u3.cnt = cnt;
10958		return ptr;
10959	    }
10960	} while ((n = n->u2.node) != NULL);
10961    }
10962    n = NEWHEAP();
10963    ptr = xrealloc(ptr, size);
10964    return ADD2HEAP(n, cnt, ptr);
10965}
10966
10967void
10968rb_parser_free(struct parser_params *parser, void *ptr)
10969{
10970    NODE **prev = &parser->heap, *n;
10971
10972    while ((n = *prev) != NULL) {
10973	if (n->u1.node == ptr) {
10974	    *prev = n->u2.node;
10975	    rb_gc_force_recycle((VALUE)n);
10976	    break;
10977	}
10978	prev = &n->u2.node;
10979    }
10980    xfree(ptr);
10981}
10982#endif
10983#endif
10984
10985#ifdef RIPPER
10986#ifdef RIPPER_DEBUG
10987extern int rb_is_pointer_to_heap(VALUE);
10988
10989/* :nodoc: */
10990static VALUE
10991ripper_validate_object(VALUE self, VALUE x)
10992{
10993    if (x == Qfalse) return x;
10994    if (x == Qtrue) return x;
10995    if (x == Qnil) return x;
10996    if (x == Qundef)
10997        rb_raise(rb_eArgError, "Qundef given");
10998    if (FIXNUM_P(x)) return x;
10999    if (SYMBOL_P(x)) return x;
11000    if (!rb_is_pointer_to_heap(x))
11001        rb_raise(rb_eArgError, "invalid pointer: %p", x);
11002    switch (TYPE(x)) {
11003      case T_STRING:
11004      case T_OBJECT:
11005      case T_ARRAY:
11006      case T_BIGNUM:
11007      case T_FLOAT:
11008        return x;
11009      case T_NODE:
11010	if (nd_type(x) != NODE_LASGN) {
11011	    rb_raise(rb_eArgError, "NODE given: %p", x);
11012	}
11013	return ((NODE *)x)->nd_rval;
11014      default:
11015        rb_raise(rb_eArgError, "wrong type of ruby object: %p (%s)",
11016                 x, rb_obj_classname(x));
11017    }
11018    return x;
11019}
11020#endif
11021
11022#define validate(x) ((x) = get_value(x))
11023
11024static VALUE
11025ripper_dispatch0(struct parser_params *parser, ID mid)
11026{
11027    return rb_funcall(parser->value, mid, 0);
11028}
11029
11030static VALUE
11031ripper_dispatch1(struct parser_params *parser, ID mid, VALUE a)
11032{
11033    validate(a);
11034    return rb_funcall(parser->value, mid, 1, a);
11035}
11036
11037static VALUE
11038ripper_dispatch2(struct parser_params *parser, ID mid, VALUE a, VALUE b)
11039{
11040    validate(a);
11041    validate(b);
11042    return rb_funcall(parser->value, mid, 2, a, b);
11043}
11044
11045static VALUE
11046ripper_dispatch3(struct parser_params *parser, ID mid, VALUE a, VALUE b, VALUE c)
11047{
11048    validate(a);
11049    validate(b);
11050    validate(c);
11051    return rb_funcall(parser->value, mid, 3, a, b, c);
11052}
11053
11054static VALUE
11055ripper_dispatch4(struct parser_params *parser, ID mid, VALUE a, VALUE b, VALUE c, VALUE d)
11056{
11057    validate(a);
11058    validate(b);
11059    validate(c);
11060    validate(d);
11061    return rb_funcall(parser->value, mid, 4, a, b, c, d);
11062}
11063
11064static VALUE
11065ripper_dispatch5(struct parser_params *parser, ID mid, VALUE a, VALUE b, VALUE c, VALUE d, VALUE e)
11066{
11067    validate(a);
11068    validate(b);
11069    validate(c);
11070    validate(d);
11071    validate(e);
11072    return rb_funcall(parser->value, mid, 5, a, b, c, d, e);
11073}
11074
11075static VALUE
11076ripper_dispatch7(struct parser_params *parser, ID mid, VALUE a, VALUE b, VALUE c, VALUE d, VALUE e, VALUE f, VALUE g)
11077{
11078    validate(a);
11079    validate(b);
11080    validate(c);
11081    validate(d);
11082    validate(e);
11083    validate(f);
11084    validate(g);
11085    return rb_funcall(parser->value, mid, 7, a, b, c, d, e, f, g);
11086}
11087
11088static const struct kw_assoc {
11089    ID id;
11090    const char *name;
11091} keyword_to_name[] = {
11092    {keyword_class,	"class"},
11093    {keyword_module,	"module"},
11094    {keyword_def,	"def"},
11095    {keyword_undef,	"undef"},
11096    {keyword_begin,	"begin"},
11097    {keyword_rescue,	"rescue"},
11098    {keyword_ensure,	"ensure"},
11099    {keyword_end,	"end"},
11100    {keyword_if,	"if"},
11101    {keyword_unless,	"unless"},
11102    {keyword_then,	"then"},
11103    {keyword_elsif,	"elsif"},
11104    {keyword_else,	"else"},
11105    {keyword_case,	"case"},
11106    {keyword_when,	"when"},
11107    {keyword_while,	"while"},
11108    {keyword_until,	"until"},
11109    {keyword_for,	"for"},
11110    {keyword_break,	"break"},
11111    {keyword_next,	"next"},
11112    {keyword_redo,	"redo"},
11113    {keyword_retry,	"retry"},
11114    {keyword_in,	"in"},
11115    {keyword_do,	"do"},
11116    {keyword_do_cond,	"do"},
11117    {keyword_do_block,	"do"},
11118    {keyword_return,	"return"},
11119    {keyword_yield,	"yield"},
11120    {keyword_super,	"super"},
11121    {keyword_self,	"self"},
11122    {keyword_nil,	"nil"},
11123    {keyword_true,	"true"},
11124    {keyword_false,	"false"},
11125    {keyword_and,	"and"},
11126    {keyword_or,	"or"},
11127    {keyword_not,	"not"},
11128    {modifier_if,	"if"},
11129    {modifier_unless,	"unless"},
11130    {modifier_while,	"while"},
11131    {modifier_until,	"until"},
11132    {modifier_rescue,	"rescue"},
11133    {keyword_alias,	"alias"},
11134    {keyword_defined,	"defined?"},
11135    {keyword_BEGIN,	"BEGIN"},
11136    {keyword_END,	"END"},
11137    {keyword__LINE__,	"__LINE__"},
11138    {keyword__FILE__,	"__FILE__"},
11139    {keyword__ENCODING__, "__ENCODING__"},
11140    {0, NULL}
11141};
11142
11143static const char*
11144keyword_id_to_str(ID id)
11145{
11146    const struct kw_assoc *a;
11147
11148    for (a = keyword_to_name; a->id; a++) {
11149        if (a->id == id)
11150            return a->name;
11151    }
11152    return NULL;
11153}
11154
11155#undef ripper_id2sym
11156static VALUE
11157ripper_id2sym(ID id)
11158{
11159    const char *name;
11160    char buf[8];
11161
11162    if (id <= 256) {
11163        buf[0] = (char)id;
11164        buf[1] = '\0';
11165        return ID2SYM(rb_intern2(buf, 1));
11166    }
11167    if ((name = keyword_id_to_str(id))) {
11168        return ID2SYM(rb_intern(name));
11169    }
11170    switch (id) {
11171      case tOROP:
11172        name = "||";
11173        break;
11174      case tANDOP:
11175        name = "&&";
11176        break;
11177      default:
11178        name = rb_id2name(id);
11179        if (!name) {
11180            rb_bug("cannot convert ID to string: %ld", (unsigned long)id);
11181        }
11182        return ID2SYM(id);
11183    }
11184    return ID2SYM(rb_intern(name));
11185}
11186
11187static ID
11188ripper_get_id(VALUE v)
11189{
11190    NODE *nd;
11191    if (!RB_TYPE_P(v, T_NODE)) return 0;
11192    nd = (NODE *)v;
11193    if (nd_type(nd) != NODE_LASGN) return 0;
11194    return nd->nd_vid;
11195}
11196
11197static VALUE
11198ripper_get_value(VALUE v)
11199{
11200    NODE *nd;
11201    if (v == Qundef) return Qnil;
11202    if (!RB_TYPE_P(v, T_NODE)) return v;
11203    nd = (NODE *)v;
11204    if (nd_type(nd) != NODE_LASGN) return Qnil;
11205    return nd->nd_rval;
11206}
11207
11208static void
11209ripper_compile_error(struct parser_params *parser, const char *fmt, ...)
11210{
11211    VALUE str;
11212    va_list args;
11213
11214    va_start(args, fmt);
11215    str = rb_vsprintf(fmt, args);
11216    va_end(args);
11217    rb_funcall(parser->value, rb_intern("compile_error"), 1, str);
11218}
11219
11220static void
11221ripper_warn0(struct parser_params *parser, const char *fmt)
11222{
11223    rb_funcall(parser->value, rb_intern("warn"), 1, STR_NEW2(fmt));
11224}
11225
11226static void
11227ripper_warnI(struct parser_params *parser, const char *fmt, int a)
11228{
11229    rb_funcall(parser->value, rb_intern("warn"), 2,
11230               STR_NEW2(fmt), INT2NUM(a));
11231}
11232
11233static void
11234ripper_warnS(struct parser_params *parser, const char *fmt, const char *str)
11235{
11236    rb_funcall(parser->value, rb_intern("warn"), 2,
11237               STR_NEW2(fmt), STR_NEW2(str));
11238}
11239
11240static void
11241ripper_warning0(struct parser_params *parser, const char *fmt)
11242{
11243    rb_funcall(parser->value, rb_intern("warning"), 1, STR_NEW2(fmt));
11244}
11245
11246static void
11247ripper_warningS(struct parser_params *parser, const char *fmt, const char *str)
11248{
11249    rb_funcall(parser->value, rb_intern("warning"), 2,
11250               STR_NEW2(fmt), STR_NEW2(str));
11251}
11252
11253static VALUE
11254ripper_lex_get_generic(struct parser_params *parser, VALUE src)
11255{
11256    return rb_io_gets(src);
11257}
11258
11259static VALUE
11260ripper_s_allocate(VALUE klass)
11261{
11262    struct parser_params *p;
11263    VALUE self;
11264
11265    p = ALLOC_N(struct parser_params, 1);
11266    MEMZERO(p, struct parser_params, 1);
11267    self = TypedData_Wrap_Struct(klass, &parser_data_type, p);
11268    p->value = self;
11269    return self;
11270}
11271
11272#define ripper_initialized_p(r) ((r)->parser_lex_input != 0)
11273
11274/*
11275 *  call-seq:
11276 *    Ripper.new(src, filename="(ripper)", lineno=1) -> ripper
11277 *
11278 *  Create a new Ripper object.
11279 *  _src_ must be a String, an IO, or an Object which has #gets method.
11280 *
11281 *  This method does not starts parsing.
11282 *  See also Ripper#parse and Ripper.parse.
11283 */
11284static VALUE
11285ripper_initialize(int argc, VALUE *argv, VALUE self)
11286{
11287    struct parser_params *parser;
11288    VALUE src, fname, lineno;
11289
11290    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
11291    rb_scan_args(argc, argv, "12", &src, &fname, &lineno);
11292    if (RB_TYPE_P(src, T_FILE)) {
11293        parser->parser_lex_gets = ripper_lex_get_generic;
11294    }
11295    else {
11296        StringValue(src);
11297        parser->parser_lex_gets = lex_get_str;
11298    }
11299    parser->parser_lex_input = src;
11300    parser->eofp = Qfalse;
11301    if (NIL_P(fname)) {
11302        fname = STR_NEW2("(ripper)");
11303    }
11304    else {
11305        StringValue(fname);
11306    }
11307    parser_initialize(parser);
11308
11309    parser->parser_ruby_sourcefile_string = fname;
11310    parser->parser_ruby_sourcefile = RSTRING_PTR(fname);
11311    parser->parser_ruby_sourceline = NIL_P(lineno) ? 0 : NUM2INT(lineno) - 1;
11312
11313    return Qnil;
11314}
11315
11316struct ripper_args {
11317    struct parser_params *parser;
11318    int argc;
11319    VALUE *argv;
11320};
11321
11322static VALUE
11323ripper_parse0(VALUE parser_v)
11324{
11325    struct parser_params *parser;
11326
11327    TypedData_Get_Struct(parser_v, struct parser_params, &parser_data_type, parser);
11328    parser_prepare(parser);
11329    ripper_yyparse((void*)parser);
11330    return parser->result;
11331}
11332
11333static VALUE
11334ripper_ensure(VALUE parser_v)
11335{
11336    struct parser_params *parser;
11337
11338    TypedData_Get_Struct(parser_v, struct parser_params, &parser_data_type, parser);
11339    parser->parsing_thread = Qnil;
11340    return Qnil;
11341}
11342
11343/*
11344 *  call-seq:
11345 *    ripper#parse
11346 *
11347 *  Start parsing and returns the value of the root action.
11348 */
11349static VALUE
11350ripper_parse(VALUE self)
11351{
11352    struct parser_params *parser;
11353
11354    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
11355    if (!ripper_initialized_p(parser)) {
11356        rb_raise(rb_eArgError, "method called for uninitialized object");
11357    }
11358    if (!NIL_P(parser->parsing_thread)) {
11359        if (parser->parsing_thread == rb_thread_current())
11360            rb_raise(rb_eArgError, "Ripper#parse is not reentrant");
11361        else
11362            rb_raise(rb_eArgError, "Ripper#parse is not multithread-safe");
11363    }
11364    parser->parsing_thread = rb_thread_current();
11365    rb_ensure(ripper_parse0, self, ripper_ensure, self);
11366
11367    return parser->result;
11368}
11369
11370/*
11371 *  call-seq:
11372 *    ripper#column   -> Integer
11373 *
11374 *  Return column number of current parsing line.
11375 *  This number starts from 0.
11376 */
11377static VALUE
11378ripper_column(VALUE self)
11379{
11380    struct parser_params *parser;
11381    long col;
11382
11383    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
11384    if (!ripper_initialized_p(parser)) {
11385        rb_raise(rb_eArgError, "method called for uninitialized object");
11386    }
11387    if (NIL_P(parser->parsing_thread)) return Qnil;
11388    col = parser->tokp - parser->parser_lex_pbeg;
11389    return LONG2NUM(col);
11390}
11391
11392/*
11393 *  call-seq:
11394 *    ripper#filename   -> String
11395 *
11396 *  Return current parsing filename.
11397 */
11398static VALUE
11399ripper_filename(VALUE self)
11400{
11401    struct parser_params *parser;
11402
11403    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
11404    if (!ripper_initialized_p(parser)) {
11405        rb_raise(rb_eArgError, "method called for uninitialized object");
11406    }
11407    return parser->parser_ruby_sourcefile_string;
11408}
11409
11410/*
11411 *  call-seq:
11412 *    ripper#lineno   -> Integer
11413 *
11414 *  Return line number of current parsing line.
11415 *  This number starts from 1.
11416 */
11417static VALUE
11418ripper_lineno(VALUE self)
11419{
11420    struct parser_params *parser;
11421
11422    TypedData_Get_Struct(self, struct parser_params, &parser_data_type, parser);
11423    if (!ripper_initialized_p(parser)) {
11424        rb_raise(rb_eArgError, "method called for uninitialized object");
11425    }
11426    if (NIL_P(parser->parsing_thread)) return Qnil;
11427    return INT2NUM(parser->parser_ruby_sourceline);
11428}
11429
11430#ifdef RIPPER_DEBUG
11431/* :nodoc: */
11432static VALUE
11433ripper_assert_Qundef(VALUE self, VALUE obj, VALUE msg)
11434{
11435    StringValue(msg);
11436    if (obj == Qundef) {
11437        rb_raise(rb_eArgError, "%s", RSTRING_PTR(msg));
11438    }
11439    return Qnil;
11440}
11441
11442/* :nodoc: */
11443static VALUE
11444ripper_value(VALUE self, VALUE obj)
11445{
11446    return ULONG2NUM(obj);
11447}
11448#endif
11449
11450
11451void
11452Init_ripper(void)
11453{
11454    parser_data_type.parent = RTYPEDDATA_TYPE(rb_parser_new());
11455
11456    ripper_init_eventids1();
11457    ripper_init_eventids2();
11458    /* ensure existing in symbol table */
11459    (void)rb_intern("||");
11460    (void)rb_intern("&&");
11461
11462    InitVM(ripper);
11463}
11464
11465void
11466InitVM_ripper(void)
11467{
11468    VALUE Ripper;
11469
11470    Ripper = rb_define_class("Ripper", rb_cObject);
11471    rb_define_const(Ripper, "Version", rb_usascii_str_new2(RIPPER_VERSION));
11472    rb_define_alloc_func(Ripper, ripper_s_allocate);
11473    rb_define_method(Ripper, "initialize", ripper_initialize, -1);
11474    rb_define_method(Ripper, "parse", ripper_parse, 0);
11475    rb_define_method(Ripper, "column", ripper_column, 0);
11476    rb_define_method(Ripper, "filename", ripper_filename, 0);
11477    rb_define_method(Ripper, "lineno", ripper_lineno, 0);
11478    rb_define_method(Ripper, "end_seen?", rb_parser_end_seen_p, 0);
11479    rb_define_method(Ripper, "encoding", rb_parser_encoding, 0);
11480    rb_define_method(Ripper, "yydebug", rb_parser_get_yydebug, 0);
11481    rb_define_method(Ripper, "yydebug=", rb_parser_set_yydebug, 1);
11482#ifdef RIPPER_DEBUG
11483    rb_define_method(rb_mKernel, "assert_Qundef", ripper_assert_Qundef, 2);
11484    rb_define_method(rb_mKernel, "rawVALUE", ripper_value, 1);
11485    rb_define_method(rb_mKernel, "validate_object", ripper_validate_object, 1);
11486#endif
11487
11488    ripper_init_eventids1_table(Ripper);
11489    ripper_init_eventids2_table(Ripper);
11490
11491# if 0
11492    /* Hack to let RDoc document SCRIPT_LINES__ */
11493
11494    /*
11495     * When a Hash is assigned to +SCRIPT_LINES__+ the contents of files loaded
11496     * after the assignment will be added as an Array of lines with the file
11497     * name as the key.
11498     */
11499    rb_define_global_const("SCRIPT_LINES__", Qnil);
11500#endif
11501
11502}
11503#endif /* RIPPER */
11504