content: package csep.parser;
  
  import java.util.ArrayList;
  import java.util.List;
  
  import org.antlr.runtime.ANTLRStringStream;
  import org.antlr.runtime.CharStream;
  import org.antlr.runtime.CommonToken;
  import org.antlr.runtime.Token;
  import org.apache.log4j.Logger;
  
  import com.aptana.editor.coffee.parsing.Terminals;
  import com.aptana.editor.coffee.parsing.lexer.CoffeeScanner;
  import com.aptana.editor.coffee.parsing.lexer.CoffeeSymbol;
  
  public class Lexer extends csep.parser.antlr.internal.InternalCoffeeScriptLexer {
  	private final static Logger logger = Logger.getLogger(Lexer.class);
  	private CoffeeScanner aptanaScanner;
  	private CommonToken prevToken = null;
  	/**
  	 * We have to keep a track of it, because the aptana scanner doesn't do that
  	 */
  	private int tokenIndex = 0;
  	private String content;
  
  	public Lexer(CharStream in) {
  		super(in);
  		aptanaScanner = new CoffeeScanner(true);
  		content = in.substring(0,  in.size() - 1);
  		checkTrailingZeroBytes(content);
  		aptanaScanner.setSource(content);
  	}
  
  	public Lexer(String str) {
  		this(new ANTLRStringStream(str));
  	}
  
  	/**
  	 * To enable xtend test cases with multiline strings
  	 * @param str
  	 */
  	public Lexer(CharSequence str) {
  		this(str.toString());
  	}
  
  	/**
  	 * Workaround for a probable bug somewhere in the xtext-aptana toolchain:
  	 * we may get a string with trailing zero bytes
  	 */
  	protected void checkTrailingZeroBytes(String buggyString) {
  		int startZeroes = buggyString.indexOf(0);
  		if (startZeroes >= 0) {
  			logger.error("Trailing zeroes in " + buggyString, new Exception());
  		}
  	}
  
  	/**
  	 * Get next token.  If an exception is thrown by the underlying lexer,
  	 * keep calling it, and append an invalid token at the very end.
  	 */
  	@Override
  	public Token nextToken() {
  		Token token = null;
  		CoffeeSymbol symbol = null;
  		try {
  			symbol = aptanaScanner.nextToken();
  			if (symbol == null || symbol.getId() < 0) {
  				logger.warn("Unexpected symbol " + symbol, new Exception());
  				token = CommonToken.INVALID_TOKEN;
  			}
  			else if (symbol.getId() == Terminals.EOF) {
  				token = CommonToken.EOF_TOKEN;
  			}
  			else {
  				token = new BeaverToken(symbol);
  				if (((CommonToken) token).getStopIndex() >= input.size()) {
  					assert false: "Token stop index overflows " + symbol + " in:\n<<<" + content + ">>>";
  				}
  			}
  		}
  		catch (Exception e) {
  			// Xtext wants token to be CommonToken, INVALID_TOKEN_TYPE, and HIDDEN_CHANNEL
  			String text = e.getLocalizedMessage();
  			if (text == null)
  				text = "simply " + e.getClass().getSimpleName();
  			CommonToken ct = new CommonToken(Token.INVALID_TOKEN_TYPE,
  					text);
  			ct.setChannel(Token.HIDDEN_CHANNEL);
  			if (prevToken != null) {
  				int start = prevToken.getStopIndex() + 1;
  				int stop = start + 1; // TODO: get more informative errors with length of token
  				ct.setStartIndex(start);
  				ct.setStopIndex(stop);
  			}
  			token = ct;
  		}
  		token.setTokenIndex(tokenIndex);
  		if (symbol != null && symbol.hidden)
  			token.setChannel(Token.HIDDEN_CHANNEL);
  		tokenIndex++;
  		if (token instanceof CommonToken) {
  			if (prevToken != null && token.getType() > 0) {
  				if (((CommonToken)token).getStartIndex() < prevToken.getStartIndex()) {
  					assert false: "Position not follows, prevToken: " + prevToken + ", token: " + token;
  				}
  			}			
  			prevToken = (CommonToken)token;
  		}
  		logger.debug("token: " + token);
  		return token;
  	}
  	
  	/**
  	 * Read the whole input and tokenize it
  	 * 
  	 * @return a list of tokens
  	 * @throws Exception
  	 */
  	public List<CommonToken> tokenize() throws Exception {
  		List<CommonToken> symbols = new ArrayList<CommonToken>();
  		while (true) {
  			CommonToken token = (CommonToken)nextToken();
  			if (Token.EOF == token.getType()) {
  				break;
  			} else {
  				symbols.add(token);
  			}
  		}
  		return symbols;
  	}
  
  	/**
  	 * Tokenize the whole input, skipping hidden terminals
  	 * 
  	 * @return a simplified string representation of the token stream in the
  	 *         form of "tokenName:text"
  	 * @throws Exception
  	 */
  	public List<String> tokenizeToStrings() throws Exception {
  		List<String> strings = new ArrayList<String>();
  		for (Token t : tokenize()) {
  			if (t.getChannel() == Token.DEFAULT_CHANNEL) {
  				strings.add(Helper.getNameAndText(t));
  			}
  		}
  		return strings;
  	}
  
  	@Override
  	/**
  	 * Because super gets the message from an empty map
  	 */
  	public String getErrorMessage(Token t) {
  		String message = super.getErrorMessage(t);
  		if (message == null)
  			message = t.getText();
  		if (message == null)
  			message = t.toString();
  		return message;
  	}
  }
  