<commit_before>package cop5555sp15;

import cop5555sp15.TokenStream.Kind;
import cop5555sp15.TokenStream.Token;
import static cop5555sp15.TokenStream.Kind.*;
import java.util.HashMap;

public class Scanner {
	
	private enum State {
		START, 
		GOT_EQUALS, 
		GOT_DOT,
		GOT_EXCLAM,
		GOT_LANGLE,
		GOT_RANGLE,
		GOT_HYPHEN,
		
		GOT_SLASH, 				//got '/'
		GOT_SLASHSTAR,			//got '/*'
		GOT_SLASH2STAR,			//got '/*......*'
		GOT_BSLASHR,			//got '\r'
		
		IDENT_PART, 
		INT_PART, 
		STRING_PART,
		EOF
	}
	//the current state of the DFA 
	private State state;
	
	//the index of (next) char to process during scanning, or if none, past the end of the array
	private int index = 0; 
	
	//the char to process during scanning
	private char ch;

	//the current line number
	private int line = 1;	
	
    //local references to TokenStream objects for convenience
      //set in constructor
	final TokenStream stream;

	//the keywords hash map
	final HashMap<String, Kind> reservedWord = new HashMap<String, Kind>();			
    
	public void initKeyword(){
		this.reservedWord.put("int", Kind.KW_INT);
		this.reservedWord.put("string", Kind.KW_STRING);
		this.reservedWord.put("boolean", Kind.KW_BOOLEAN);
		this.reservedWord.put("import", Kind.KW_IMPORT);
		this.reservedWord.put("class", Kind.KW_CLASS);
		this.reservedWord.put("def", Kind.KW_DEF);
		this.reservedWord.put("while", Kind.KW_WHILE);
		this.reservedWord.put("if", Kind.KW_IF);
		this.reservedWord.put("else", Kind.KW_ELSE);
		this.reservedWord.put("return", Kind.KW_RETURN);
		this.reservedWord.put("print", Kind.KW_PRINT);
		this.reservedWord.put("true", Kind.BL_TRUE);
		this.reservedWord.put("false", Kind.BL_FALSE);
		this.reservedWord.put("null", Kind.NL_NULL);
	}
	
	public Scanner(TokenStream stream) {
		//initialize input stream
		this.stream = stream;
		//initialize keywords Hashmap
		this.initKeyword();
	}


	
	private char getch(){
		
		if(index < stream.inputChars.length){<commit_msg>rename initKeyword<commit_after>package cop5555sp15;

import cop5555sp15.TokenStream.Kind;
import cop5555sp15.TokenStream.Token;
import static cop5555sp15.TokenStream.Kind.*;
import java.util.HashMap;

public class Scanner {
	
	private enum State {
		START, 
		GOT_EQUALS, 
		GOT_DOT,
		GOT_EXCLAM,
		GOT_LANGLE,
		GOT_RANGLE,
		GOT_HYPHEN,
		
		GOT_SLASH, 				//got '/'
		GOT_SLASHSTAR,			//got '/*'
		GOT_SLASH2STAR,			//got '/*......*'
		GOT_BSLASHR,			//got '\r'
		
		IDENT_PART, 
		INT_PART, 
		STRING_PART,
		EOF
	}
	//the current state of the DFA 
	private State state;
	
	//the index of (next) char to process during scanning, or if none, past the end of the array
	private int index = 0; 
	
	//the char to process during scanning
	private char ch;

	//the current line number
	private int line = 1;	
	
    //local references to TokenStream objects for convenience
      //set in constructor
	final TokenStream stream;

	//the keywords hash map
	final HashMap<String, Kind> reservedWord = new HashMap<String, Kind>();			
    
	public void initReservedWord(){
		this.reservedWord.put("int", Kind.KW_INT);
		this.reservedWord.put("string", Kind.KW_STRING);
		this.reservedWord.put("boolean", Kind.KW_BOOLEAN);
		this.reservedWord.put("import", Kind.KW_IMPORT);
		this.reservedWord.put("class", Kind.KW_CLASS);
		this.reservedWord.put("def", Kind.KW_DEF);
		this.reservedWord.put("while", Kind.KW_WHILE);
		this.reservedWord.put("if", Kind.KW_IF);
		this.reservedWord.put("else", Kind.KW_ELSE);
		this.reservedWord.put("return", Kind.KW_RETURN);
		this.reservedWord.put("print", Kind.KW_PRINT);
		this.reservedWord.put("true", Kind.BL_TRUE);
		this.reservedWord.put("false", Kind.BL_FALSE);
		this.reservedWord.put("null", Kind.NL_NULL);
	}
	
	public Scanner(TokenStream stream) {
		//initialize input stream
		this.stream = stream;
		//initialize reserved words Hashmap
		this.initReservedWord();
	}


	
	private char getch(){
		
		if(index < stream.inputChars.length){
{'fraction_non_alphanumeric': 0.12878595062356835, 'fraction_numerical': 0.01374395520488674, 'mean_word_length': 3.786845310596833, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2298410', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1375, 'n_words': 315}