<commit_before>/***************************************************************************
 *   Copyright (C) 2006-2012 by <NAME> <<EMAIL>>     *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU Library General Public License as       *
 *   published by the Free Software Foundation; either version 2 of the    *
 *   License, or (at your option) any later version.                       *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU Library General Public     *
 *   License along with this program; if not, write to the                 *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 *                                                                         *
 *   For details about the authors of this software, see the AUTHORS file. *
 ***************************************************************************/

package jolie.lang.parse;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import jolie.lang.NativeType;

/**
 * Scanner implementation for the Jolie language parser.
 * 
 * @author <NAME>
 *
 */
public class Scanner
{
	/** Token types */
	public enum TokenType {
		EOF,				///< End Of File
		ID,					///< [a-z][a-zA-Z0-9]*
		COMMA,				///< ,
		DOT,				///< .
		INT,				///< [0-9]+
		TRUE,				///< true
		FALSE,				///< false
		LONG,				///< [0-9]+(l|L)
		DOUBLE,				///< [0-9]*"."[0-9]+(e|E)[0-9]+
		LPAREN,				///< (
		RPAREN,				///< )
		LSQUARE,			///< [
		RSQUARE,			///< ]
		LCURLY,				///< {
		RCURLY,				///< }
		//DOLLAR,			///< $
		STRING,				///< "[[:graph:]]*"
		INCREMENT,			///< ++
		MINUS,				///< The minus sign -
		ASTERISK,			///< *
		DIVIDE,				///< /
		ASSIGN,				///< =
		PLUS,				///< +
		ADD_ASSIGN,			///< +=
		MINUS_ASSIGN,		///< -=
		MULTIPLY_ASSIGN,	///< *=
		DIVIDE_ASSIGN,		///< %=
		SEQUENCE,			///< ;
		IF,					///< if
		ELSE,				///< else
		LANGLE,				///< <
		RANGLE,				///< >
		AT,					///< @
		LINKIN,				///< linkIn
		LINKOUT,			///< linkOut
		INSTANCE_OF,		///< instanceof
		EQUAL,				///< ==
		AND,				///< &&
		OR,					///< ||
		PARALLEL,			///< |
		NOT,				///< !
		CARET,				///< ^
		COLON,				///< :
		OP_OW,				///< OneWay
		OP_RR,				///< RequestResponse
		DEFINE, 			///< define
		MAJOR_OR_EQUAL,		///< >=
		MINOR_OR_EQUAL,		///< <=
		NOT_EQUAL,			///< !=
		NULL_PROCESS,		///< nullProcess
		WHILE,				///< while
		EXECUTION,			///< execution
		THROW,				///< throw
		DOCUMENTATION_COMMENT,
		INSTALL,				///< install
		SCOPE,				///< scope
		SPAWN,				///< spawn
		THIS,				///< this
		COMPENSATE,			///< comp
		EXIT,				///< exit
		INCLUDE,			///< include
		CONSTANTS,			///< constants
		POINTS_TO,			///< ->
		QUESTION_MARK,		///< ?
		ARROW,				///< =>
		DEEP_COPY_LEFT,		///< <<
		RUN,				///< run
		UNDEF,				///< undef
		HASH,				///< #
		PERCENT_SIGN,		///< %
		FOR,					///< for
		FOREACH,			///< foreach
		WITH,				///< with
		DECREMENT,			///< --
		IS_STRING,			///< is_string
		IS_INT,				///< is_int
		IS_DOUBLE,			///< is_double
		IS_BOOL,				///< is_bool
		IS_LONG,			///< is_long
		IS_DEFINED,			///< is_defined
		CAST_INT,			///< int
		CAST_STRING,		///< string
		CAST_DOUBLE,		///< double
		CAST_BOOL,			///< bool
		CAST_LONG,			///< long
		SYNCHRONIZED,		///< synchronized
		THROWS,				///< throws
		CURRENT_HANDLER,	///< cH
		INIT,				///< init
		PROVIDE,			///< provide
		ERROR				///< Scanner error
	}
	
	/*
	 * Map of unreserved keywords,
	 * which can be considered as IDs in certain places (e.g. variables).
	 */
	private static final Map< String, TokenType > unreservedKeywords = new HashMap<>();
	
	static {
		// Initialise the unreserved keywords map.
		unreservedKeywords.put( "OneWay", TokenType.OP_OW );
		unreservedKeywords.put( "RequestResponse", TokenType.OP_RR );
		unreservedKeywords.put( "linkIn", TokenType.LINKIN );
		unreservedKeywords.put( "linkOut", TokenType.LINKOUT );
		unreservedKeywords.put( "if", TokenType.IF );
		unreservedKeywords.put( "else", TokenType.ELSE );
		unreservedKeywords.put( "include", TokenType.INCLUDE );
		unreservedKeywords.put( "define", TokenType.DEFINE );
		unreservedKeywords.put( "nullProcess", TokenType.NULL_PROCESS );
		unreservedKeywords.put( "while", TokenType.WHILE );
		unreservedKeywords.put( "execution", TokenType.EXECUTION );
		unreservedKeywords.put( "install", TokenType.INSTALL );
		unreservedKeywords.put( "this", TokenType.THIS );
		unreservedKeywords.put( "synchronized", TokenType.SYNCHRONIZED );
		unreservedKeywords.put( "throw", TokenType.THROW );
		unreservedKeywords.put( "scope", TokenType.SCOPE );
		unreservedKeywords.put( "spawn", TokenType.SPAWN );
		unreservedKeywords.put( "comp", TokenType.COMPENSATE );
		unreservedKeywords.put( "exit", TokenType.EXIT );
		unreservedKeywords.put( "constants", TokenType.CONSTANTS );
		unreservedKeywords.put( "undef", TokenType.UNDEF );
		unreservedKeywords.put( "for", TokenType.FOR );
		unreservedKeywords.put( "foreach", TokenType.FOREACH );
		unreservedKeywords.put( "is_defined", TokenType.IS_DEFINED );
		unreservedKeywords.put( "is_string", TokenType.IS_STRING );
		unreservedKeywords.put( "is_int", TokenType.IS_INT );
		unreservedKeywords.put( "is_bool", TokenType.IS_BOOL );
		unreservedKeywords.put( "is_long", TokenType.IS_LONG );
		unreservedKeywords.put( "is_double", TokenType.IS_DOUBLE );
		unreservedKeywords.put( "instanceof", TokenType.INSTANCE_OF );
		unreservedKeywords.put( NativeType.INT.id(), TokenType.CAST_INT );
		unreservedKeywords.put( NativeType.STRING.id(), TokenType.CAST_STRING );
		unreservedKeywords.put( NativeType.BOOL.id(), TokenType.CAST_BOOL );
		unreservedKeywords.put( NativeType.DOUBLE.id(), TokenType.CAST_DOUBLE );
		unreservedKeywords.put( NativeType.LONG.id(), TokenType.CAST_LONG );
		unreservedKeywords.put( "throws", TokenType.THROWS );
		unreservedKeywords.put( "cH", TokenType.CURRENT_HANDLER );
		unreservedKeywords.put( "init", TokenType.INIT );
		unreservedKeywords.put( "with", TokenType.WITH );
		unreservedKeywords.put( "true", TokenType.TRUE );
		unreservedKeywords.put( "false", TokenType.FALSE );
		unreservedKeywords.put( "provide", TokenType.PROVIDE );
	}
	
	/**
	 * This class represents an input token read by the Scanner class.
	 * 
	 * @see Scanner
	 * @author <NAME>
	 * @version 1.0
	 *
	 */
	public static class Token
	{
		private final TokenType type;
		private final String content;
		private final boolean isUnreservedKeyword;

		/**
		 * Constructor. The content of the token will be set to "".
		 * @param type the type of this token
		 */
		public Token( TokenType type )
		{
			this.type = type;
			this.content = "";
			this.isUnreservedKeyword = false;
		}

		/**
		 * Constructor.
		 * @param type the type of this token
		 * @param content the content of this token
		 */
		public Token( TokenType type, String content )
		{
			this.type = type;
			this.content = content;
			this.isUnreservedKeyword = false;
		}
		
		/**
		 * Constructor.
		 * @param type the type of this token
		 * @param content the content of this token
		 * @param isUnreservedKeyword specifies whether this token is an unreserved keyword
		 */
		public Token( TokenType type, String content, boolean isUnreservedKeyword )
		{
			this.type = type;
			this.content = content;
			this.isUnreservedKeyword = isUnreservedKeyword;
		}

		/**
		 * Returns the content of this token.
		 * @return the content of this token
		 */
		public String content()
		{
			return content;
		}

		/**
		 * Returns the type of this token.
		 * @return the type of this token
		 */
		public TokenType type()
		{
			return type;
		}

		/**
		 * Returns <code>true</code> if this token can be considered as a valid
		 * value for a constant, <code>false</code> otherwise.
		 * @return <code>true</code> if this token can be considered as a valid
		 * value for a constant, <code>false</code> otherwise
		 */
		public boolean isValidConstant()
		{
			return	type == TokenType.STRING ||
					type == TokenType.INT ||
					type == TokenType.ID ||
					type == TokenType.LONG ||
					type == TokenType.TRUE ||
					type == TokenType.FALSE ||
					type == TokenType.DOUBLE;
		}

		/**
		 * Equivalent to <code>is(TokenType.EOF)</code>
		 * @return <code>true</code> if this token has type <code>TokenType.EOF</code>, false otherwise
		 */
		public boolean isEOF()
		{
			return ( type == TokenType.EOF );
		}

		/**
		 * Returns <code>true</code> if this token has the passed type, <code>false</code> otherwise.
		 * @param compareType the type to compare the type of this token with
		 * @return <code>true</code> if this token has the passed type, <code>false</code> otherwise
		 */
		public boolean is( TokenType compareType )
		{
			return ( type == compareType );
		}

		/**
		 * Returns <code>true</code> if this token has a different type from the passed one, <code>false</code> otherwise.
		 * @param compareType the type to compare the type of this token with
		 * @return <code>true</code> if this token has a different type from the passed one, <code>false</code> otherwise
		 */
		public boolean isNot( TokenType compareType )
		{
			return ( type != compareType );
		}

		/**
		 * Returns <code>true</code> if this token has type <code>TokenType.ID</code>
		 * and its content is equal to the passed parameter, <code>false</code> otherwise.
		 * @param keyword the keyword to check the content of this token against
		 * @return <code>true</code> if this token has type <code>TokenType.ID</code>
		 * and its content is equal to the passed parameter, <code>false</code> otherwise
		 */
		public boolean isKeyword( String keyword )
		{
			return( type == TokenType.ID && content.equals( keyword ) ); 
		}
		
		/**
		 * Returns <code>true</code> if this token has type <code>TokenType.ID</code>
		 * or is a token for an unreserved keyword, <code>false</code> otherwise.
		 * @return <code>true</code> if this token has type <code>TokenType.ID</code>
		 * or is a token for an unreserved keyword, <code>false</code> otherwise.
		 */
		public boolean isIdentifier()
		{
			return( type == TokenType.ID || isUnreservedKeyword );
		}

		/**
		 * This method behaves as {@link #isKeyword(java.lang.String) isKeyword}, except that
		 * it is case insensitive.
		 * @param keyword the keyword to check the content of this token against
		 * @return
		 */
		public boolean isKeywordIgnoreCase( String keyword )
		{
			return( type == TokenType.ID && content.equalsIgnoreCase( keyword ) );
		}
	}
	

	private final InputStream stream;		// input stream
	private final InputStreamReader reader;			// data input
	protected char ch;						// current character
	protected int currInt;					// current stream int
	protected int state;					// current state
	private int line;						// current line
	private final URI source;				// source name

	/**
	 * Constructor
	 * @param stream the <code>InputStream</code> to use for input reading
	 * @param source the source URI of the stream
	 * @param charset the character encoding
	 * @throws java.io.IOException if the input reading initialization fails
	 */
	public Scanner( InputStream stream, URI source, String charset )
		throws IOException
	{
		this.stream = stream;
		this.reader = charset != null ? new InputStreamReader( stream, charset ) : new InputStreamReader( stream );
		this.source = source;
		line = 1;
		readChar();
	}

	/* public String readWord()
		throws IOException
	{
		return readWord( true );
	}
	
	public String readWord( boolean readChar )
		throws IOException
	{
		StringBuilder buffer = new StringBuilder();
		if ( readChar ) {
			readChar();
		}

		do {
			buffer.append( ch );
			readChar();
		} while( !isSeparator( ch ) );
		return buffer.toString();
	} */
	
	private final StringBuilder tokenBuilder = new StringBuilder( 64 );

	private void resetTokenBuilder()
	{
		tokenBuilder.setLength( 0 );
	}
	
	public String readLine()
		throws IOException
	{
		resetTokenBuilder();
		readChar();
		while( !isNewLineChar( ch ) ) {
			tokenBuilder.append( ch );
			readChar();
		}
		return tokenBuilder.toString();
	}
	
	public static String addStringTerminator( String str )
	{
		return str + -1;
	}
	
	public InputStream inputStream()
	{
		return stream;
	}

	/**
	 * Returns character encoding
	 * @return character encoding
	 */
	public String charset()
	{
		return reader.getEncoding();
	}
	
	/**
	 * Returns the current line the scanner is reading.
	 * 
	 * @return the current line the scanner is reading.
	 */
	public int line()
	{
		return line;
	}
	
	/**
	 * Returns the source URI the scanner is reading.
	 * 
	 * @return the source URI the scanner is reading
	 */
	public URI source()
	{
		return source;
	}

	public String sourceName()
	{
		return source.getSchemeSpecificPart();
	}
	
	/**
	 * Eats all separators (whitespace) until the next input.
	 * @throws IOException 
	 */
	public void eatSeparators()
		throws IOException
	{
		while( isSeparator( ch ) ) {
			readChar();
		}
	}
	
	public void eatSeparatorsUntilEOF()
		throws IOException
	{
		while( isSeparator( ch ) && stream.available() > 0 ) {
			readChar();
		}
	}
	
	/**
	 * Checks whether a character is a separator (whitespace).
	 * @param c the character to check as a whitespace
	 * @return <code>true</code> if <code>c</code> is a separator (whitespace)
	 */
	public static boolean isSeparator( char c )<commit_msg>Remove unused method Scanner.getSourceName<commit_after>/***************************************************************************
 *   Copyright (C) 2006-2012 by <NAME> <<EMAIL>>     *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU Library General Public License as       *
 *   published by the Free Software Foundation; either version 2 of the    *
 *   License, or (at your option) any later version.                       *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU Library General Public     *
 *   License along with this program; if not, write to the                 *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 *                                                                         *
 *   For details about the authors of this software, see the AUTHORS file. *
 ***************************************************************************/

package jolie.lang.parse;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
import jolie.lang.NativeType;

/**
 * Scanner implementation for the Jolie language parser.
 * 
 * @author <NAME>
 *
 */
public class Scanner
{
	/** Token types */
	public enum TokenType {
		EOF,				///< End Of File
		ID,					///< [a-z][a-zA-Z0-9]*
		COMMA,				///< ,
		DOT,				///< .
		INT,				///< [0-9]+
		TRUE,				///< true
		FALSE,				///< false
		LONG,				///< [0-9]+(l|L)
		DOUBLE,				///< [0-9]*"."[0-9]+(e|E)[0-9]+
		LPAREN,				///< (
		RPAREN,				///< )
		LSQUARE,			///< [
		RSQUARE,			///< ]
		LCURLY,				///< {
		RCURLY,				///< }
		//DOLLAR,			///< $
		STRING,				///< "[[:graph:]]*"
		INCREMENT,			///< ++
		MINUS,				///< The minus sign -
		ASTERISK,			///< *
		DIVIDE,				///< /
		ASSIGN,				///< =
		PLUS,				///< +
		ADD_ASSIGN,			///< +=
		MINUS_ASSIGN,		///< -=
		MULTIPLY_ASSIGN,	///< *=
		DIVIDE_ASSIGN,		///< %=
		SEQUENCE,			///< ;
		IF,					///< if
		ELSE,				///< else
		LANGLE,				///< <
		RANGLE,				///< >
		AT,					///< @
		LINKIN,				///< linkIn
		LINKOUT,			///< linkOut
		INSTANCE_OF,		///< instanceof
		EQUAL,				///< ==
		AND,				///< &&
		OR,					///< ||
		PARALLEL,			///< |
		NOT,				///< !
		CARET,				///< ^
		COLON,				///< :
		OP_OW,				///< OneWay
		OP_RR,				///< RequestResponse
		DEFINE, 			///< define
		MAJOR_OR_EQUAL,		///< >=
		MINOR_OR_EQUAL,		///< <=
		NOT_EQUAL,			///< !=
		NULL_PROCESS,		///< nullProcess
		WHILE,				///< while
		EXECUTION,			///< execution
		THROW,				///< throw
		DOCUMENTATION_COMMENT,
		INSTALL,				///< install
		SCOPE,				///< scope
		SPAWN,				///< spawn
		THIS,				///< this
		COMPENSATE,			///< comp
		EXIT,				///< exit
		INCLUDE,			///< include
		CONSTANTS,			///< constants
		POINTS_TO,			///< ->
		QUESTION_MARK,		///< ?
		ARROW,				///< =>
		DEEP_COPY_LEFT,		///< <<
		RUN,				///< run
		UNDEF,				///< undef
		HASH,				///< #
		PERCENT_SIGN,		///< %
		FOR,					///< for
		FOREACH,			///< foreach
		WITH,				///< with
		DECREMENT,			///< --
		IS_STRING,			///< is_string
		IS_INT,				///< is_int
		IS_DOUBLE,			///< is_double
		IS_BOOL,				///< is_bool
		IS_LONG,			///< is_long
		IS_DEFINED,			///< is_defined
		CAST_INT,			///< int
		CAST_STRING,		///< string
		CAST_DOUBLE,		///< double
		CAST_BOOL,			///< bool
		CAST_LONG,			///< long
		SYNCHRONIZED,		///< synchronized
		THROWS,				///< throws
		CURRENT_HANDLER,	///< cH
		INIT,				///< init
		PROVIDE,			///< provide
		ERROR				///< Scanner error
	}
	
	/*
	 * Map of unreserved keywords,
	 * which can be considered as IDs in certain places (e.g. variables).
	 */
	private static final Map< String, TokenType > unreservedKeywords = new HashMap<>();
	
	static {
		// Initialise the unreserved keywords map.
		unreservedKeywords.put( "OneWay", TokenType.OP_OW );
		unreservedKeywords.put( "RequestResponse", TokenType.OP_RR );
		unreservedKeywords.put( "linkIn", TokenType.LINKIN );
		unreservedKeywords.put( "linkOut", TokenType.LINKOUT );
		unreservedKeywords.put( "if", TokenType.IF );
		unreservedKeywords.put( "else", TokenType.ELSE );
		unreservedKeywords.put( "include", TokenType.INCLUDE );
		unreservedKeywords.put( "define", TokenType.DEFINE );
		unreservedKeywords.put( "nullProcess", TokenType.NULL_PROCESS );
		unreservedKeywords.put( "while", TokenType.WHILE );
		unreservedKeywords.put( "execution", TokenType.EXECUTION );
		unreservedKeywords.put( "install", TokenType.INSTALL );
		unreservedKeywords.put( "this", TokenType.THIS );
		unreservedKeywords.put( "synchronized", TokenType.SYNCHRONIZED );
		unreservedKeywords.put( "throw", TokenType.THROW );
		unreservedKeywords.put( "scope", TokenType.SCOPE );
		unreservedKeywords.put( "spawn", TokenType.SPAWN );
		unreservedKeywords.put( "comp", TokenType.COMPENSATE );
		unreservedKeywords.put( "exit", TokenType.EXIT );
		unreservedKeywords.put( "constants", TokenType.CONSTANTS );
		unreservedKeywords.put( "undef", TokenType.UNDEF );
		unreservedKeywords.put( "for", TokenType.FOR );
		unreservedKeywords.put( "foreach", TokenType.FOREACH );
		unreservedKeywords.put( "is_defined", TokenType.IS_DEFINED );
		unreservedKeywords.put( "is_string", TokenType.IS_STRING );
		unreservedKeywords.put( "is_int", TokenType.IS_INT );
		unreservedKeywords.put( "is_bool", TokenType.IS_BOOL );
		unreservedKeywords.put( "is_long", TokenType.IS_LONG );
		unreservedKeywords.put( "is_double", TokenType.IS_DOUBLE );
		unreservedKeywords.put( "instanceof", TokenType.INSTANCE_OF );
		unreservedKeywords.put( NativeType.INT.id(), TokenType.CAST_INT );
		unreservedKeywords.put( NativeType.STRING.id(), TokenType.CAST_STRING );
		unreservedKeywords.put( NativeType.BOOL.id(), TokenType.CAST_BOOL );
		unreservedKeywords.put( NativeType.DOUBLE.id(), TokenType.CAST_DOUBLE );
		unreservedKeywords.put( NativeType.LONG.id(), TokenType.CAST_LONG );
		unreservedKeywords.put( "throws", TokenType.THROWS );
		unreservedKeywords.put( "cH", TokenType.CURRENT_HANDLER );
		unreservedKeywords.put( "init", TokenType.INIT );
		unreservedKeywords.put( "with", TokenType.WITH );
		unreservedKeywords.put( "true", TokenType.TRUE );
		unreservedKeywords.put( "false", TokenType.FALSE );
		unreservedKeywords.put( "provide", TokenType.PROVIDE );
	}
	
	/**
	 * This class represents an input token read by the Scanner class.
	 * 
	 * @see Scanner
	 * @author <NAME>
	 * @version 1.0
	 *
	 */
	public static class Token
	{
		private final TokenType type;
		private final String content;
		private final boolean isUnreservedKeyword;

		/**
		 * Constructor. The content of the token will be set to "".
		 * @param type the type of this token
		 */
		public Token( TokenType type )
		{
			this.type = type;
			this.content = "";
			this.isUnreservedKeyword = false;
		}

		/**
		 * Constructor.
		 * @param type the type of this token
		 * @param content the content of this token
		 */
		public Token( TokenType type, String content )
		{
			this.type = type;
			this.content = content;
			this.isUnreservedKeyword = false;
		}
		
		/**
		 * Constructor.
		 * @param type the type of this token
		 * @param content the content of this token
		 * @param isUnreservedKeyword specifies whether this token is an unreserved keyword
		 */
		public Token( TokenType type, String content, boolean isUnreservedKeyword )
		{
			this.type = type;
			this.content = content;
			this.isUnreservedKeyword = isUnreservedKeyword;
		}

		/**
		 * Returns the content of this token.
		 * @return the content of this token
		 */
		public String content()
		{
			return content;
		}

		/**
		 * Returns the type of this token.
		 * @return the type of this token
		 */
		public TokenType type()
		{
			return type;
		}

		/**
		 * Returns <code>true</code> if this token can be considered as a valid
		 * value for a constant, <code>false</code> otherwise.
		 * @return <code>true</code> if this token can be considered as a valid
		 * value for a constant, <code>false</code> otherwise
		 */
		public boolean isValidConstant()
		{
			return	type == TokenType.STRING ||
					type == TokenType.INT ||
					type == TokenType.ID ||
					type == TokenType.LONG ||
					type == TokenType.TRUE ||
					type == TokenType.FALSE ||
					type == TokenType.DOUBLE;
		}

		/**
		 * Equivalent to <code>is(TokenType.EOF)</code>
		 * @return <code>true</code> if this token has type <code>TokenType.EOF</code>, false otherwise
		 */
		public boolean isEOF()
		{
			return ( type == TokenType.EOF );
		}

		/**
		 * Returns <code>true</code> if this token has the passed type, <code>false</code> otherwise.
		 * @param compareType the type to compare the type of this token with
		 * @return <code>true</code> if this token has the passed type, <code>false</code> otherwise
		 */
		public boolean is( TokenType compareType )
		{
			return ( type == compareType );
		}

		/**
		 * Returns <code>true</code> if this token has a different type from the passed one, <code>false</code> otherwise.
		 * @param compareType the type to compare the type of this token with
		 * @return <code>true</code> if this token has a different type from the passed one, <code>false</code> otherwise
		 */
		public boolean isNot( TokenType compareType )
		{
			return ( type != compareType );
		}

		/**
		 * Returns <code>true</code> if this token has type <code>TokenType.ID</code>
		 * and its content is equal to the passed parameter, <code>false</code> otherwise.
		 * @param keyword the keyword to check the content of this token against
		 * @return <code>true</code> if this token has type <code>TokenType.ID</code>
		 * and its content is equal to the passed parameter, <code>false</code> otherwise
		 */
		public boolean isKeyword( String keyword )
		{
			return( type == TokenType.ID && content.equals( keyword ) ); 
		}
		
		/**
		 * Returns <code>true</code> if this token has type <code>TokenType.ID</code>
		 * or is a token for an unreserved keyword, <code>false</code> otherwise.
		 * @return <code>true</code> if this token has type <code>TokenType.ID</code>
		 * or is a token for an unreserved keyword, <code>false</code> otherwise.
		 */
		public boolean isIdentifier()
		{
			return( type == TokenType.ID || isUnreservedKeyword );
		}

		/**
		 * This method behaves as {@link #isKeyword(java.lang.String) isKeyword}, except that
		 * it is case insensitive.
		 * @param keyword the keyword to check the content of this token against
		 * @return
		 */
		public boolean isKeywordIgnoreCase( String keyword )
		{
			return( type == TokenType.ID && content.equalsIgnoreCase( keyword ) );
		}
	}
	

	private final InputStream stream;		// input stream
	private final InputStreamReader reader;			// data input
	protected char ch;						// current character
	protected int currInt;					// current stream int
	protected int state;					// current state
	private int line;						// current line
	private final URI source;				// source name

	/**
	 * Constructor
	 * @param stream the <code>InputStream</code> to use for input reading
	 * @param source the source URI of the stream
	 * @param charset the character encoding
	 * @throws java.io.IOException if the input reading initialization fails
	 */
	public Scanner( InputStream stream, URI source, String charset )
		throws IOException
	{
		this.stream = stream;
		this.reader = charset != null ? new InputStreamReader( stream, charset ) : new InputStreamReader( stream );
		this.source = source;
		line = 1;
		readChar();
	}

	/* public String readWord()
		throws IOException
	{
		return readWord( true );
	}
	
	public String readWord( boolean readChar )
		throws IOException
	{
		StringBuilder buffer = new StringBuilder();
		if ( readChar ) {
			readChar();
		}

		do {
			buffer.append( ch );
			readChar();
		} while( !isSeparator( ch ) );
		return buffer.toString();
	} */
	
	private final StringBuilder tokenBuilder = new StringBuilder( 64 );

	private void resetTokenBuilder()
	{
		tokenBuilder.setLength( 0 );
	}
	
	public String readLine()
		throws IOException
	{
		resetTokenBuilder();
		readChar();
		while( !isNewLineChar( ch ) ) {
			tokenBuilder.append( ch );
			readChar();
		}
		return tokenBuilder.toString();
	}
	
	public static String addStringTerminator( String str )
	{
		return str + -1;
	}
	
	public InputStream inputStream()
	{
		return stream;
	}

	/**
	 * Returns character encoding
	 * @return character encoding
	 */
	public String charset()
	{
		return reader.getEncoding();
	}
	
	/**
	 * Returns the current line the scanner is reading.
	 * 
	 * @return the current line the scanner is reading.
	 */
	public int line()
	{
		return line;
	}
	
	/**
	 * Returns the source URI the scanner is reading.
	 * 
	 * @return the source URI the scanner is reading
	 */
	public URI source()
	{
		return source;
	}
	
	/**
	 * Eats all separators (whitespace) until the next input.
	 * @throws IOException 
	 */
	public void eatSeparators()
		throws IOException
	{
		while( isSeparator( ch ) ) {
			readChar();
		}
	}
	
	public void eatSeparatorsUntilEOF()
		throws IOException
	{
		while( isSeparator( ch ) && stream.available() > 0 ) {
			readChar();
		}
	}
	
	/**
	 * Checks whether a character is a separator (whitespace).
	 * @param c the character to check as a whitespace
	 * @return <code>true</code> if <code>c</code> is a separator (whitespace)
	 */
	public static boolean isSeparator( char c )
{'fraction_non_alphanumeric': 0.14040667548154237, 'fraction_numerical': 0.003073294500232284, 'mean_word_length': 2.5927590191295415, 'pattern_counts': {'":': 0, '<': 325, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4429466', 'n_tokens_mistral': 10223, 'n_tokens_neox': 8747, 'n_words': 2436}