<commit_before>package itertools.iterator;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;

/**
 * Wraps a BufferedReader to provide iteration over the lines in the file.
 * 
 * @author jelsas
 * 
 */
public class FileLineIterator extends IteratorWrapper<String> {
  BufferedReader in;

  public FileLineIterator(String filename) throws IOException {
    in = new BufferedReader(new FileReader(filename));
  }

  /**
   * Returns the next line of the file or null if there are no more lines, or if
   * an error occurred reading the file. The file is close & cleaned up if an
   * error occurred or the end of the file is reached.
   */
  @Override
  public String getNext() {
    if (in == null)
      return null;
    String next;
    try {
      next = in.readLine();
    } catch (IOException e) {
      cleanup();
      next = null;
    }
    return next;
  }

  private void cleanup() {
    try {
      in.close();
    } catch (IOException e) {
      // ignore
    }
    in = null;
  }
}<commit_msg>adding parameter for setting file buffer size<commit_after>package itertools.iterator;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;

/**
 * Wraps a BufferedReader to provide iteration over the lines in the file.
 * 
 * @author jelsas
 * 
 */
public class FileLineIterator extends IteratorWrapper<String> {
  BufferedReader in;

  public FileLineIterator(String filename) throws IOException {
    in = new BufferedReader(new FileReader(filename));
  }

  public FileLineIterator(String filename, int bufferSize) throws IOException {
    in = new BufferedReader(new FileReader(filename), bufferSize);
  }

  /**
   * Returns the next line of the file or null if there are no more lines, or if
   * an error occurred reading the file. The file is close & cleaned up if an
   * error occurred or the end of the file is reached.
   */
  @Override
  public String getNext() {
    if (in == null) return null;
    String next;
    try {
      next = in.readLine();
    } catch (IOException e) {
      cleanup();
      next = null;
    }
    return next;
  }

  private void cleanup() {
    try {
      in.close();
    } catch (IOException e) {
      // ignore
    }
    in = null;
  }
}
{'fraction_non_alphanumeric': 0.09187279151943463, 'fraction_numerical': 0.0, 'mean_word_length': 2.80672268907563, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5386116', 'n_tokens_mistral': 666, 'n_tokens_neox': 655, 'n_words': 242}