TokenStreamRewriteEngine.hpp [plain text]
#ifndef INC_TokenStreamRewriteEngine_hpp__
#define INC_TokenStreamRewriteEngine_hpp__
/* ANTLR Translator Generator
* Project led by Terence Parr at http://www.jGuru.com
* Software rights: http://www.antlr.org/license.html
*/
#include <string>
#include <list>
#include <vector>
#include <map>
#include <utility>
#include <ostream>
#include <iterator>
#include <cassert>
#include <algorithm>
#include <antlr/config.hpp>
#include <antlr/TokenStream.hpp>
#include <antlr/TokenWithIndex.hpp>
#include <antlr/BitSet.hpp>
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
namespace antlr {
#endif
/** This token stream tracks the *entire* token stream coming from
* a lexer, but does not pass on the whitespace (or whatever else
* you want to discard) to the parser.
*
* This class can then be asked for the ith token in the input stream.
* Useful for dumping out the input stream exactly after doing some
* augmentation or other manipulations. Tokens are index from 0..n-1
*
* You can insert stuff, replace, and delete chunks. Note that the
* operations are done lazily--only if you convert the buffer to a
* String. This is very efficient because you are not moving data around
* all the time. As the buffer of tokens is converted to strings, the
* toString() method(s) check to see if there is an operation at the
* current index. If so, the operation is done and then normal String
* rendering continues on the buffer. This is like having multiple Turing
* machine instruction streams (programs) operating on a single input tape. :)
*
* Since the operations are done lazily at toString-time, operations do not
* screw up the token index values. That is, an insert operation at token
* index i does not change the index values for tokens i+1..n-1.
*
* Because operations never actually alter the buffer, you may always get
* the original token stream back without undoing anything. Since
* the instructions are queued up, you can easily simulate transactions and
* roll back any changes if there is an error just by removing instructions.
* For example,
*
* TokenStreamRewriteEngine rewriteEngine =
* new TokenStreamRewriteEngine(lexer);
* JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
* ...
* rewriteEngine.insertAfter("pass1", t, "foobar");}
* rewriteEngine.insertAfter("pass2", u, "start");}
* System.out.println(rewriteEngine.toString("pass1"));
* System.out.println(rewriteEngine.toString("pass2"));
*
* You can also have multiple "instruction streams" and get multiple
* rewrites from a single pass over the input. Just name the instruction
* streams and use that name again when printing the buffer. This could be
* useful for generating a C file and also its header file--all from the
* same buffer.
*
* If you don't use named rewrite streams, a "default" stream is used.
*
* Terence Parr, parrt@cs.usfca.edu
* University of San Francisco
* February 2004
*/
class TokenStreamRewriteEngine : public TokenStream
{
public:
typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
static const char* DEFAULT_PROGRAM_NAME;
#ifndef NO_STATIC_CONSTS
static const size_t MIN_TOKEN_INDEX;
static const int PROGRAM_INIT_SIZE;
#else
enum {
MIN_TOKEN_INDEX = 0,
PROGRAM_INIT_SIZE = 100
};
#endif
struct tokenToStream {
tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
template <typename T> void operator() ( const T& t ) {
out << t->getText();
}
ANTLR_USE_NAMESPACE(std)ostream& out;
};
class RewriteOperation {
protected:
RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
: index(idx), text(txt)
{
}
public:
virtual ~RewriteOperation()
{
}
/** Execute the rewrite operation by possibly adding to the buffer.
* Return the index of the next token to operate on.
*/
virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
return index;
}
virtual size_t getIndex() const {
return index;
}
virtual const char* type() const {
return "RewriteOperation";
}
protected:
size_t index;
ANTLR_USE_NAMESPACE(std)string text;
};
struct executeOperation {
ANTLR_USE_NAMESPACE(std)ostream& out;
executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
void operator () ( RewriteOperation* t ) {
t->execute(out);
}
};
/// list of rewrite operations
typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
/// map program name to <program counter,program> tuple
typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;
class InsertBeforeOp : public RewriteOperation
{
public:
InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
: RewriteOperation(index, text)
{
}
virtual ~InsertBeforeOp() {}
virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
{
out << text;
return index;
}
virtual const char* type() const {
return "InsertBeforeOp";
}
};
class ReplaceOp : public RewriteOperation
{
public:
ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
: RewriteOperation(from,text)
, lastIndex(to)
{
}
virtual ~ReplaceOp() {}
virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
out << text;
return lastIndex+1;
}
virtual const char* type() const {
return "ReplaceOp";
}
protected:
size_t lastIndex;
};
class DeleteOp : public ReplaceOp {
public:
DeleteOp(size_t from, size_t to)
: ReplaceOp(from,to,"")
{
}
virtual const char* type() const {
return "DeleteOp";
}
};
TokenStreamRewriteEngine(TokenStream& upstream);
TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);
RefToken nextToken( void );
void rollback(size_t instructionIndex) {
rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
}
/** Rollback the instruction stream for a program so that
* the indicated instruction (via instructionIndex) is no
* longer in the stream. UNTESTED!
*/
void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
size_t instructionIndex );
void deleteProgram() {
deleteProgram(DEFAULT_PROGRAM_NAME);
}
/** Reset the program so that no instructions exist */
void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
rollback(programName, MIN_TOKEN_INDEX);
}
void insertAfter( RefTokenWithIndex t,
const ANTLR_USE_NAMESPACE(std)string& text )
{
insertAfter(DEFAULT_PROGRAM_NAME, t, text);
}
void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
insertAfter(DEFAULT_PROGRAM_NAME, index, text);
}
void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
RefTokenWithIndex t,
const ANTLR_USE_NAMESPACE(std)string& text )
{
insertAfter(programName, t->getIndex(), text);
}
void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
size_t index,
const ANTLR_USE_NAMESPACE(std)string& text )
{
// to insert after, just insert before next index (even if past end)
insertBefore(programName,index+1, text);
}
void insertBefore( RefTokenWithIndex t,
const ANTLR_USE_NAMESPACE(std)string& text )
{
// std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
insertBefore(DEFAULT_PROGRAM_NAME, t, text);
}
void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
insertBefore(DEFAULT_PROGRAM_NAME, index, text);
}
void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
RefTokenWithIndex t,
const ANTLR_USE_NAMESPACE(std)string& text )
{
insertBefore(programName, t->getIndex(), text);
}
void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
size_t index,
const ANTLR_USE_NAMESPACE(std)string& text )
{
addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
}
void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
{
replace(DEFAULT_PROGRAM_NAME, index, index, text);
}
void replace( size_t from, size_t to,
const ANTLR_USE_NAMESPACE(std)string& text)
{
replace(DEFAULT_PROGRAM_NAME, from, to, text);
}
void replace( RefTokenWithIndex indexT,
const ANTLR_USE_NAMESPACE(std)string& text )
{
replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
}
void replace( RefTokenWithIndex from,
RefTokenWithIndex to,
const ANTLR_USE_NAMESPACE(std)string& text )
{
replace(DEFAULT_PROGRAM_NAME, from, to, text);
}
void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
size_t from, size_t to,
const ANTLR_USE_NAMESPACE(std)string& text )
{
addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
}
void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
RefTokenWithIndex from,
RefTokenWithIndex to,
const ANTLR_USE_NAMESPACE(std)string& text )
{
replace(programName,
from->getIndex(),
to->getIndex(),
text);
}
void remove(size_t index) {
remove(DEFAULT_PROGRAM_NAME, index, index);
}
void remove(size_t from, size_t to) {
remove(DEFAULT_PROGRAM_NAME, from, to);
}
void remove(RefTokenWithIndex indexT) {
remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
}
void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
remove(DEFAULT_PROGRAM_NAME, from, to);
}
void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
size_t from, size_t to)
{
replace(programName,from,to,"");
}
void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
RefTokenWithIndex from, RefTokenWithIndex to )
{
replace(programName,from,to,"");
}
void discard(int ttype) {
discardMask.add(ttype);
}
RefToken getToken( size_t i )
{
return RefToken(tokens.at(i));
}
size_t getTokenStreamSize() const {
return tokens.size();
}
void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
}
void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
size_t start, size_t end ) const;
void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
}
void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
const ANTLR_USE_NAMESPACE(std)string& programName ) const
{
toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
}
void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
size_t start, size_t end ) const
{
toStream(out, DEFAULT_PROGRAM_NAME, start, end);
}
void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
const ANTLR_USE_NAMESPACE(std)string& programName,
size_t firstToken, size_t lastToken ) const;
void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
}
void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
size_t start, size_t end ) const;
size_t getLastRewriteTokenIndex() const {
return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
}
/** Return the last index for the program named programName
* return 0 if the program does not exist or the program is empty.
* (Note this is different from the java implementation that returns -1)
*/
size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
program_map::const_iterator rewrites = programs.find(programName);
if( rewrites == programs.end() )
return 0;
const operation_list& prog = rewrites->second;
if( !prog.empty() )
{
operation_list::const_iterator last = prog.end();
--last;
return (*last)->getIndex();
}
return 0;
}
protected:
/** If op.index > lastRewriteTokenIndexes, just add to the end.
* Otherwise, do linear */
void addToSortedRewriteList(RewriteOperation* op) {
addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
}
void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
RewriteOperation* op );
protected:
/** Who do we suck tokens from? */
TokenStream& stream;
/** track index of tokens */
size_t index;
/** Track the incoming list of tokens */
token_list tokens;
/** You may have multiple, named streams of rewrite operations.
* I'm calling these things "programs."
* Maps String (name) -> rewrite (List)
*/
program_map programs;
/** Which (whitespace) token(s) to throw out */
BitSet discardMask;
};
#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
}
#endif
#endif