1#include <antlr/config.hpp>
2
3#include <string>
4#include <list>
5#include <vector>
6#include <map>
7#include <utility>
8//#include <iostream>
9#include <iterator>
10#include <sstream>
11#include <cassert>
12
13#include <antlr/TokenStream.hpp>
14#include <antlr/TokenWithIndex.hpp>
15#include <antlr/BitSet.hpp>
16#include <antlr/TokenStreamRewriteEngine.hpp>
17
18#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
19namespace antlr {
20#endif
21
22#ifndef NO_STATIC_CONSTS
23const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0;
24const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100;
25#endif
26
27const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";
28
29namespace {
30
31	struct compareOperationIndex {
32		typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
33		bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
34		{
35			return a->getIndex() < b->getIndex();
36		}
37	};
38	struct dumpTokenWithIndex {
39		dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
40		void operator() ( const RefTokenWithIndex& t ) {
41			out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
42		}
43		ANTLR_USE_NAMESPACE(std)ostream& out;
44	};
45}
46
47TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
48: stream(upstream)
49, index(MIN_TOKEN_INDEX)
50, tokens()
51, programs()
52, discardMask()
53{
54}
55
56TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
57: stream(upstream)
58, index(MIN_TOKEN_INDEX)
59, tokens(initialSize)
60, programs()
61, discardMask()
62{
63}
64
65RefToken TokenStreamRewriteEngine::nextToken( void )
66{
67	RefTokenWithIndex t;
68	// suck tokens until end of stream or we find a non-discarded token
69	do {
70		t = RefTokenWithIndex(stream.nextToken());
71		if ( t )
72		{
73			t->setIndex(index);  // what is t's index in list?
74			if ( t->getType() != Token::EOF_TYPE ) {
75				tokens.push_back(t);  // track all tokens except EOF
76			}
77			index++;			// move to next position
78		}
79	} while ( t && discardMask.member(t->getType()) );
80	return RefToken(t);
81}
82
83void TokenStreamRewriteEngine::rollback( const std::string& programName,
84													  size_t instructionIndex )
85{
86	program_map::iterator rewrite = programs.find(programName);
87	if( rewrite != programs.end() )
88	{
89		operation_list& prog = rewrite->second;
90		operation_list::iterator
91			j = prog.begin(),
92			end = prog.end();
93
94		std::advance(j,instructionIndex);
95		if( j != end )
96			prog.erase(j, end);
97	}
98}
99
100void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
101																 size_t start,
102																 size_t end ) const
103{
104	token_list::const_iterator s = tokens.begin();
105	std::advance( s, start );
106	token_list::const_iterator e = s;
107	std::advance( e, end-start );
108	std::for_each( s, e, tokenToStream(out) );
109}
110
111void TokenStreamRewriteEngine::toStream( std::ostream& out,
112													  const std::string& programName,
113													  size_t firstToken,
114													  size_t lastToken ) const
115{
116	if( tokens.size() == 0 )
117		return;
118
119	program_map::const_iterator rewriter = programs.find(programName);
120
121	if ( rewriter == programs.end() )
122		return;
123
124	// get the prog and some iterators in it...
125	const operation_list& prog = rewriter->second;
126	operation_list::const_iterator
127		rewriteOpIndex = prog.begin(),
128		rewriteOpEnd = prog.end();
129
130	size_t tokenCursor = firstToken;
131	// make sure we don't run out of the tokens we have...
132	if( lastToken > (tokens.size() - 1) )
133		lastToken = tokens.size() - 1;
134
135		while ( tokenCursor <= lastToken )
136		{
137//			std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;
138
139			if( rewriteOpIndex != rewriteOpEnd )
140			{
141				size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
142				while( tokenCursor < up_to_here )
143					out << tokens[tokenCursor++]->getText();
144			}
145			while ( rewriteOpIndex != rewriteOpEnd &&
146					  tokenCursor == (*rewriteOpIndex)->getIndex() &&
147					  tokenCursor <= lastToken )
148			{
149				tokenCursor = (*rewriteOpIndex)->execute(out);
150				++rewriteOpIndex;
151			}
152			if( tokenCursor <= lastToken )
153				out << tokens[tokenCursor++]->getText();
154		}
155	// std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
156	// now see if there are operations (append) beyond last token index
157	std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
158	rewriteOpIndex = rewriteOpEnd;
159}
160
161void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
162															 size_t start,
163															 size_t end ) const
164{
165	token_list::const_iterator s = tokens.begin();
166	std::advance( s, start );
167	token_list::const_iterator e = s;
168	std::advance( e, end-start );
169	std::for_each( s, e, dumpTokenWithIndex(out) );
170}
171
172void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
173																		 RewriteOperation* op )
174{
175	program_map::iterator rewrites = programs.find(programName);
176	// check if we got the program already..
177	if ( rewrites == programs.end() )
178	{
179		// no prog make a new one...
180		operation_list ops;
181		ops.push_back(op);
182		programs.insert(std::make_pair(programName,ops));
183		return;
184	}
185	operation_list& prog = rewrites->second;
186
187	if( prog.empty() )
188	{
189		prog.push_back(op);
190		return;
191	}
192
193	operation_list::iterator i, end = prog.end();
194	i = end;
195	--i;
196	// if at or beyond last op's index, just append
197	if ( op->getIndex() >= (*i)->getIndex() ) {
198		prog.push_back(op); // append to list of operations
199		return;
200	}
201	i = prog.begin();
202
203	if( i != end )
204	{
205		operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
206		prog.insert(pos,op);
207	}
208	else
209		prog.push_back(op);
210}
211
212#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
213}
214#endif
215