antlr2/antlr/TokenStreamRewriteEngine.hpp

#ifndef INC_TokenStreamRewriteEngine_hpp__
#define INC_TokenStreamRewriteEngine_hpp__

/* ANTLR Translator Generator
 * Project led by Terence Parr at http://www.jGuru.com
 * Software rights: http://www.antlr.org/license.html
 */

#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <ostream>
#include <iterator>
#include <cassert>
#include <algorithm>

#include <antlr/config.hpp>

#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif

/** This token stream tracks the *entire* token stream coming from
 *	 a lexer, but does not pass on the whitespace (or whatever else
 *	 you want to discard) to the parser.
 *
 *	 This class can then be asked for the ith token in the input stream.
 *	 Useful for dumping out the input stream exactly after doing some
 *	 augmentation or other manipulations.	Tokens are index from 0..n-1
 *
 *	 You can insert stuff, replace, and delete chunks.	 Note that the
 *	 operations are done lazily--only if you convert the buffer to a
 *	 String.	 This is very efficient because you are not moving data around
 *	 all the time.	 As the buffer of tokens is converted to strings, the
 *	 toString() method(s) check to see if there is an operation at the
 *	 current index.  If so, the operation is done and then normal String
 *	 rendering continues on the buffer.	 This is like having multiple Turing
 *	 machine instruction streams (programs) operating on a single input tape. :)
 *
 *	 Since the operations are done lazily at toString-time, operations do not
 *	 screw up the token index values.  That is, an insert operation at token
 *	 index i does not change the index values for tokens i+1..n-1.
 *
 *	 Because operations never actually alter the buffer, you may always get
 *	 the original token stream back without undoing anything.  Since
 *	 the instructions are queued up, you can easily simulate transactions and
 *	 roll back any changes if there is an error just by removing instructions.
 *	 For example,
 *
 *			TokenStreamRewriteEngine rewriteEngine =
 *				new TokenStreamRewriteEngine(lexer);
 *		  JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
 *		  ...
 *		  rewriteEngine.insertAfter("pass1", t, "foobar");}
 *			rewriteEngine.insertAfter("pass2", u, "start");}
 *			System.out.println(rewriteEngine.toString("pass1"));
 *			System.out.println(rewriteEngine.toString("pass2"));
 *
 *	 You can also have multiple "instruction streams" and get multiple
 *	 rewrites from a single pass over the input.	 Just name the instruction
 *	 streams and use that name again when printing the buffer.	This could be
 *	 useful for generating a C file and also its header file--all from the
 *	 same buffer.
 *
 *	 If you don't use named rewrite streams, a "default" stream is used.
 *
 *	 Terence Parr, parrt@cs.usfca.edu
 *	 University of San Francisco
 *	 February 2004
 */
class TokenStreamRewriteEngine : public TokenStream
{
public:
	typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
	static const char* DEFAULT_PROGRAM_NAME;
#ifndef NO_STATIC_CONSTS
	static const size_t MIN_TOKEN_INDEX;
	static const int PROGRAM_INIT_SIZE;
#else
	enum {
		MIN_TOKEN_INDEX = 0,
		PROGRAM_INIT_SIZE = 100
	};
#endif

	struct tokenToStream {
		tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
		template <typename T> void operator() ( const T& t ) {
			out << t->getText();
		}
		ANTLR_USE_NAMESPACE(std)ostream& out;
	};

	class RewriteOperation {
	protected:
		RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
		: index(idx), text(txt)
		{
		}
	public:
		virtual ~RewriteOperation()
		{
		}
		/** Execute the rewrite operation by possibly adding to the buffer.
		 *	 Return the index of the next token to operate on.
		 */
		virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
			return index;
		}
		virtual size_t getIndex() const {
			return index;
		}
		virtual const char* type() const {
			return "RewriteOperation";
		}
	protected:
		size_t index;
		ANTLR_USE_NAMESPACE(std)string text;
	};

	struct executeOperation {
		ANTLR_USE_NAMESPACE(std)ostream& out;
		executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
		void operator () ( RewriteOperation* t ) {
			t->execute(out);
		}
	};

	/// list of rewrite operations
	typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
	/// map program name to <program counter,program> tuple
	typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;

	class InsertBeforeOp : public RewriteOperation
	{
	public:
		InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
		: RewriteOperation(index, text)
		{
		}
		virtual ~InsertBeforeOp() {}
		virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
		{
			out << text;
			return index;
		}
		virtual const char* type() const {
			return "InsertBeforeOp";
		}
	};

	class ReplaceOp : public RewriteOperation
	{
	public:
		ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
		: RewriteOperation(from,text)
		, lastIndex(to)
		{
		}
		virtual ~ReplaceOp() {}
		virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
			out << text;
			return lastIndex+1;
		}
		virtual const char* type() const {
			return "ReplaceOp";
		}
	protected:
		size_t lastIndex;
	};

	class DeleteOp : public ReplaceOp {
	public:
		DeleteOp(size_t from, size_t to)
		: ReplaceOp(from,to,"")
		{
		}
		virtual const char* type() const {
			return "DeleteOp";
		}
	};

	TokenStreamRewriteEngine(TokenStream& upstream);

	TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);

	RefToken nextToken( void );

	void rollback(size_t instructionIndex) {
		rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
	}

	/** Rollback the instruction stream for a program so that
	 *	 the indicated instruction (via instructionIndex) is no
	 *	 longer in the stream.	UNTESTED!
	 */
	void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
					  size_t instructionIndex );

	void deleteProgram() {
		deleteProgram(DEFAULT_PROGRAM_NAME);
	}

	/** Reset the program so that no instructions exist */
	void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
		rollback(programName, MIN_TOKEN_INDEX);
	}

	void insertAfter( RefTokenWithIndex t,
							const ANTLR_USE_NAMESPACE(std)string& text )
	{
		insertAfter(DEFAULT_PROGRAM_NAME, t, text);
	}

	void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
		insertAfter(DEFAULT_PROGRAM_NAME, index, text);
	}

	void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
							RefTokenWithIndex t,
							const ANTLR_USE_NAMESPACE(std)string& text )
	{
		insertAfter(programName, t->getIndex(), text);
	}

	void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
							size_t index,
							const ANTLR_USE_NAMESPACE(std)string& text )
	{
		// to insert after, just insert before next index (even if past end)
		insertBefore(programName,index+1, text);
	}

	void insertBefore( RefTokenWithIndex t,
							 const ANTLR_USE_NAMESPACE(std)string& text )
	{
		// std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
		insertBefore(DEFAULT_PROGRAM_NAME, t, text);
	}

	void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
		insertBefore(DEFAULT_PROGRAM_NAME, index, text);
	}

	void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
							 RefTokenWithIndex t,
							 const ANTLR_USE_NAMESPACE(std)string& text )
	{
		insertBefore(programName, t->getIndex(), text);
	}

	void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
							 size_t index,
							 const ANTLR_USE_NAMESPACE(std)string& text )
	{
		addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
	}

	void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
	{
		replace(DEFAULT_PROGRAM_NAME, index, index, text);
	}

	void replace( size_t from, size_t to,
					  const ANTLR_USE_NAMESPACE(std)string& text)
	{
		replace(DEFAULT_PROGRAM_NAME, from, to, text);
	}

	void replace( RefTokenWithIndex indexT,
					  const ANTLR_USE_NAMESPACE(std)string& text )
	{
		replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
	}

	void replace( RefTokenWithIndex from,
					  RefTokenWithIndex to,
					  const ANTLR_USE_NAMESPACE(std)string& text )
	{
		replace(DEFAULT_PROGRAM_NAME, from, to, text);
	}

	void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
					 size_t from, size_t to,
					 const ANTLR_USE_NAMESPACE(std)string& text )
	{
		addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
	}

	void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
					  RefTokenWithIndex from,
					  RefTokenWithIndex to,
					  const ANTLR_USE_NAMESPACE(std)string& text )
	{
		replace(programName,
				  from->getIndex(),
				  to->getIndex(),
				  text);
	}

	void remove(size_t index) {
		remove(DEFAULT_PROGRAM_NAME, index, index);
	}

	void remove(size_t from, size_t to) {
		remove(DEFAULT_PROGRAM_NAME, from, to);
	}

	void remove(RefTokenWithIndex indexT) {
		remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
	}

	void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
		remove(DEFAULT_PROGRAM_NAME, from, to);
	}

	void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
					 size_t from, size_t to)
	{
		replace(programName,from,to,"");
	}

	void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
					 RefTokenWithIndex from, RefTokenWithIndex to )
	{
		replace(programName,from,to,"");
	}

	void discard(int ttype) {
		discardMask.add(ttype);
	}

	RefToken getToken( size_t i )
	{
		return RefToken(tokens.at(i));
	}

	size_t getTokenStreamSize() const {
		return tokens.size();
	}

	void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
		ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
	}

	void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
								  size_t start, size_t end ) const;

	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
		toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
	}

	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
						const ANTLR_USE_NAMESPACE(std)string& programName ) const
	{
		toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
	}

	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
						size_t start, size_t end ) const
	{
		toStream(out, DEFAULT_PROGRAM_NAME, start, end);
	}

	void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
						const ANTLR_USE_NAMESPACE(std)string& programName,
						size_t firstToken, size_t lastToken ) const;

	void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
		toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
	}

	void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
							  size_t start, size_t end ) const;

	size_t getLastRewriteTokenIndex() const {
		return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
	}

	/** Return the last index for the program named programName
	 * return 0 if the program does not exist or the program is empty.
	 * (Note this is different from the java implementation that returns -1)
	 */
	size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
		program_map::const_iterator rewrites = programs.find(programName);

		if( rewrites == programs.end() )
			return 0;

		const operation_list& prog = rewrites->second;
		if( !prog.empty() )
		{
			operation_list::const_iterator last = prog.end();
			--last;
			return (*last)->getIndex();
		}
		return 0;
	}

protected:
	/** If op.index > lastRewriteTokenIndexes, just add to the end.
	 *	 Otherwise, do linear */
	void addToSortedRewriteList(RewriteOperation* op) {
		addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
	}

	void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
										  RewriteOperation* op );

protected:
	/** Who do we suck tokens from? */
	TokenStream& stream;
	/** track index of tokens */
	size_t index;

	/** Track the incoming list of tokens */
	token_list tokens;

	/** You may have multiple, named streams of rewrite operations.
	 *  I'm calling these things "programs."
	 *  Maps String (name) -> rewrite (List)
	 */
	program_map programs;

	/** Which (whitespace) token(s) to throw out */
	BitSet discardMask;
};

#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif

#endif