1#include <antlr/config.hpp> 2 3#include <string> 4#include <list> 5#include <vector> 6#include <map> 7#include <utility> 8//#include <iostream> 9#include <iterator> 10#include <sstream> 11#include <cassert> 12 13#include <antlr/TokenStream.hpp> 14#include <antlr/TokenWithIndex.hpp> 15#include <antlr/BitSet.hpp> 16#include <antlr/TokenStreamRewriteEngine.hpp> 17 18#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE 19namespace antlr { 20#endif 21 22#ifndef NO_STATIC_CONSTS 23const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0; 24const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100; 25#endif 26 27const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default"; 28 29namespace { 30 31 struct compareOperationIndex { 32 typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation; 33 bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const 34 { 35 return a->getIndex() < b->getIndex(); 36 } 37 }; 38 struct dumpTokenWithIndex { 39 dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {} 40 void operator() ( const RefTokenWithIndex& t ) { 41 out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n"; 42 } 43 ANTLR_USE_NAMESPACE(std)ostream& out; 44 }; 45} 46 47TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream) 48: stream(upstream) 49, index(MIN_TOKEN_INDEX) 50, tokens() 51, programs() 52, discardMask() 53{ 54} 55 56TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize ) 57: stream(upstream) 58, index(MIN_TOKEN_INDEX) 59, tokens(initialSize) 60, programs() 61, discardMask() 62{ 63} 64 65RefToken TokenStreamRewriteEngine::nextToken( void ) 66{ 67 RefTokenWithIndex t; 68 // suck tokens until end of stream or we find a non-discarded token 69 do { 70 t = RefTokenWithIndex(stream.nextToken()); 71 if ( t ) 72 { 73 t->setIndex(index); // what is t's index in list? 74 if ( t->getType() != Token::EOF_TYPE ) { 75 tokens.push_back(t); // track all tokens except EOF 76 } 77 index++; // move to next position 78 } 79 } while ( t && discardMask.member(t->getType()) ); 80 return RefToken(t); 81} 82 83void TokenStreamRewriteEngine::rollback( const std::string& programName, 84 size_t instructionIndex ) 85{ 86 program_map::iterator rewrite = programs.find(programName); 87 if( rewrite != programs.end() ) 88 { 89 operation_list& prog = rewrite->second; 90 operation_list::iterator 91 j = prog.begin(), 92 end = prog.end(); 93 94 std::advance(j,instructionIndex); 95 if( j != end ) 96 prog.erase(j, end); 97 } 98} 99 100void TokenStreamRewriteEngine::originalToStream( std::ostream& out, 101 size_t start, 102 size_t end ) const 103{ 104 token_list::const_iterator s = tokens.begin(); 105 std::advance( s, start ); 106 token_list::const_iterator e = s; 107 std::advance( e, end-start ); 108 std::for_each( s, e, tokenToStream(out) ); 109} 110 111void TokenStreamRewriteEngine::toStream( std::ostream& out, 112 const std::string& programName, 113 size_t firstToken, 114 size_t lastToken ) const 115{ 116 if( tokens.size() == 0 ) 117 return; 118 119 program_map::const_iterator rewriter = programs.find(programName); 120 121 if ( rewriter == programs.end() ) 122 return; 123 124 // get the prog and some iterators in it... 125 const operation_list& prog = rewriter->second; 126 operation_list::const_iterator 127 rewriteOpIndex = prog.begin(), 128 rewriteOpEnd = prog.end(); 129 130 size_t tokenCursor = firstToken; 131 // make sure we don't run out of the tokens we have... 132 if( lastToken > (tokens.size() - 1) ) 133 lastToken = tokens.size() - 1; 134 135 while ( tokenCursor <= lastToken ) 136 { 137// std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl; 138 139 if( rewriteOpIndex != rewriteOpEnd ) 140 { 141 size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex()); 142 while( tokenCursor < up_to_here ) 143 out << tokens[tokenCursor++]->getText(); 144 } 145 while ( rewriteOpIndex != rewriteOpEnd && 146 tokenCursor == (*rewriteOpIndex)->getIndex() && 147 tokenCursor <= lastToken ) 148 { 149 tokenCursor = (*rewriteOpIndex)->execute(out); 150 ++rewriteOpIndex; 151 } 152 if( tokenCursor <= lastToken ) 153 out << tokens[tokenCursor++]->getText(); 154 } 155 // std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl; 156 // now see if there are operations (append) beyond last token index 157 std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) ); 158 rewriteOpIndex = rewriteOpEnd; 159} 160 161void TokenStreamRewriteEngine::toDebugStream( std::ostream& out, 162 size_t start, 163 size_t end ) const 164{ 165 token_list::const_iterator s = tokens.begin(); 166 std::advance( s, start ); 167 token_list::const_iterator e = s; 168 std::advance( e, end-start ); 169 std::for_each( s, e, dumpTokenWithIndex(out) ); 170} 171 172void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName, 173 RewriteOperation* op ) 174{ 175 program_map::iterator rewrites = programs.find(programName); 176 // check if we got the program already.. 177 if ( rewrites == programs.end() ) 178 { 179 // no prog make a new one... 180 operation_list ops; 181 ops.push_back(op); 182 programs.insert(std::make_pair(programName,ops)); 183 return; 184 } 185 operation_list& prog = rewrites->second; 186 187 if( prog.empty() ) 188 { 189 prog.push_back(op); 190 return; 191 } 192 193 operation_list::iterator i, end = prog.end(); 194 i = end; 195 --i; 196 // if at or beyond last op's index, just append 197 if ( op->getIndex() >= (*i)->getIndex() ) { 198 prog.push_back(op); // append to list of operations 199 return; 200 } 201 i = prog.begin(); 202 203 if( i != end ) 204 { 205 operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() ); 206 prog.insert(pos,op); 207 } 208 else 209 prog.push_back(op); 210} 211 212#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE 213} 214#endif 215