<commit_before>
/**
 * [[[copyright]]]
 */
package edu.cmu.sphinx.util;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StreamCorruptedException;
import java.io.StreamTokenizer;
import java.io.FileReader;
import java.io.Reader;
import java.io.BufferedReader;


public class ExtendedStreamTokenizer {
    private String path;
    private StreamTokenizer st;
    private Reader reader;
    private boolean atEOF = false;

    /**
     * Creates and returns a stream tokenizer that has
     * been properly configured to parse sphinx3 data
     *
     * @param path the source of the data
     *
     * @throws FileNotFoundException if a file cannot be found
     */
    public ExtendedStreamTokenizer(String path) 
			throws FileNotFoundException {
	this(path, false);
    }

    /**
     * Creates and returns a stream tokenizer that has
     * been properly configured to parse sphinx3 data
     *
     * @param path the source of the data
     * @param eolIsSignificant if true eol is significant
     *
     * @throws FileNotFoundException if a file cannot be found
     */
    public ExtendedStreamTokenizer(String path, boolean eolIsSignificant) 
			throws FileNotFoundException {
	FileReader fileReader = new FileReader(path);
	reader = new BufferedReader(fileReader);
	this.path = path;
	st = new StreamTokenizer(reader);
	st.resetSyntax();
	st.whitespaceChars(0, 32);
	st.wordChars(33, 127);
	st.eolIsSignificant(eolIsSignificant);
	st.commentChar('#');
    }


    /**
     * Closes the tokenizer
     */
    public void close() throws IOException {
	reader.close();
    }

    /**
     * Gets the next word from the tokenizer
     *
     * @throws StreamCorruptedException if the word does not match
     * @throws IOException if an error occurs while loading the data
     */
    public String getString() throws StreamCorruptedException, IOException  {
	st.nextToken();
	if (st.ttype == StreamTokenizer.TT_EOF) {
	    atEOF = true;
	}
	if (st.ttype != StreamTokenizer.TT_WORD &&
	    st.ttype != StreamTokenizer.TT_EOL &&
	    st.ttype != StreamTokenizer.TT_EOF) {
	    corrupt("word expected but not found");
	}
	if (st.ttype == StreamTokenizer.TT_EOL ||
	    st.ttype == StreamTokenizer.TT_EOF) {
	    return null;
	} else {
	    return st.sval;
	}
    }


    /**
     * Determines if the stream is at the end of file
     *
     * @return true if the stream is at EOF
     */
    public boolean isEOF() {
	return atEOF;
    }


    /**<commit_msg>Added an 'unget' capability to allow for applications to read ahead from a stream and put tokens back when they've read too far.  This is used by the Language Model parser.<commit_after>
/**
 * [[[copyright]]]
 */
package edu.cmu.sphinx.util;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StreamCorruptedException;
import java.io.StreamTokenizer;
import java.io.FileReader;
import java.io.Reader;
import java.io.BufferedReader;
import java.util.List;
import java.util.ArrayList;


public class ExtendedStreamTokenizer {
    private String path;
    private StreamTokenizer st;
    private Reader reader;
    private boolean atEOF = false;
    private List putbackList;

    /**
     * Creates and returns a stream tokenizer that has
     * been properly configured to parse sphinx3 data
     *
     * @param path the source of the data
     *
     * @throws FileNotFoundException if a file cannot be found
     */
    public ExtendedStreamTokenizer(String path) 
			throws FileNotFoundException {
	this(path, false);
    }

    /**
     * Creates and returns a stream tokenizer that has
     * been properly configured to parse sphinx3 data
     *
     * @param path the source of the data
     * @param eolIsSignificant if true eol is significant
     *
     * @throws FileNotFoundException if a file cannot be found
     */
    public ExtendedStreamTokenizer(String path, boolean eolIsSignificant) 
			throws FileNotFoundException {
	FileReader fileReader = new FileReader(path);
	reader = new BufferedReader(fileReader);
	this.path = path;
	st = new StreamTokenizer(reader);
	st.resetSyntax();
	st.whitespaceChars(0, 32);
	st.wordChars(33, 127);
	st.eolIsSignificant(eolIsSignificant);
	st.commentChar('#');
	putbackList = new ArrayList();
    }


    /**
     * Closes the tokenizer
     */
    public void close() throws IOException {
	reader.close();
    }

    /**
     * Gets the next word from the tokenizer
     *
     * @throws StreamCorruptedException if the word does not match
     * @throws IOException if an error occurs while loading the data
     */
    public String getString() throws StreamCorruptedException, IOException  {
	if (putbackList.size() > 0) {
	    return (String) putbackList.remove(putbackList.size() - 1);
	} else {
	    st.nextToken();
	    if (st.ttype == StreamTokenizer.TT_EOF) {
		atEOF = true;
	    }
	    if (st.ttype != StreamTokenizer.TT_WORD &&
		st.ttype != StreamTokenizer.TT_EOL &&
		st.ttype != StreamTokenizer.TT_EOF) {
		corrupt("word expected but not found");
	    }
	    if (st.ttype == StreamTokenizer.TT_EOL ||
		st.ttype == StreamTokenizer.TT_EOF) {
		return null;
	    } else {
		return st.sval;
	    }
	}
    }


    /**
     * Puts a string back, the next get will return this string
     *
     * @param string the string to unget
     */
    public void unget(String string) {
	putbackList.add(string);
    }


    /**
     * Determines if the stream is at the end of file
     *
     * @return true if the stream is at EOF
     */
    public boolean isEOF() {
	return atEOF;
    }


    /**
{'fraction_non_alphanumeric': 0.0927484333034915, 'fraction_numerical': 0.003939122649955237, 'mean_word_length': 2.928270042194093, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1363006', 'n_tokens_mistral': 1752, 'n_tokens_neox': 1664, 'n_words': 524}