<commit_before>package edu.umass.ciir.galagotools.utils;

import org.lemurproject.galago.tupleflow.FileUtility;
import org.lemurproject.galago.tupleflow.StreamCreator;
import org.lemurproject.galago.tupleflow.Utility;

import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * @author jfoley
 */
public class IO {
  public interface StringFunctor {
    public void process(String input);
  }

  public static void forEachLine(List<File> files, StringFunctor doWhat) {
    for(File fp : files) {
      forEachLine(fp, doWhat);
    }
  }

  public static void forEachLine(File fp, StringFunctor doWhat) {
    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new InputStreamReader(StreamCreator.openInputStream(fp)));

      while(true) {
        String line = reader.readLine();
        if(line == null) break;
        doWhat.process(line);
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      close(reader);
    }
  }

  public static void forEachLine(BufferedReader reader, StringFunctor doWhat) {
    try {
      while (true) {
        String line = reader.readLine();
        if (line == null) break;
        doWhat.process(line);
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      close(reader);
    }
  }

  public static String slurp(File path) throws IOException {
    final StringBuilder contents = new StringBuilder();
    forEachLine(path, new StringFunctor() {
      @Override
      public void process(String input) {
        contents.append(input).append('\n');
      }
    });
    return contents.toString();
  }

  public static String slurp(String path) throws IOException {
    return slurp(new File(path));
  }

  public static List<String> slurpLines(String path) throws IOException {
    final ArrayList<String> lines = new ArrayList<String>();
    forEachLine(new File(path), new StringFunctor() {
      @Override
      public void process(String input) {
        lines.add(input);
      }
    });
    return lines;
  }

  public static void copyFile(String from, String to) {
    InputStream is = null;
    try {
      is = StreamCreator.realInputStream(from);
      Utility.copyStreamToFile(is, new File(to));
      is = null;
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      close(is);
    }
  }

  public static BufferedReader fileReader(String path) throws IOException {
    return fileReader(new File(path));
  }

  public static BufferedReader fileReader(File input) throws IOException {
    return new BufferedReader(new InputStreamReader(StreamCreator.openInputStream(input)));
  }

  public static void forEachLineInStr(String input, StringFunctor doWhat) {
    String[] lines = input.split("\n");
    for(String line : lines) {
      doWhat.process(line);
    }
  }

  public static PrintWriter printWriter(String output) throws IOException {
    FileUtility.makeParentDirectories(output);
    return new PrintWriter(StreamCreator.openOutputStream(output));
  }

  public static InputStream stringStream(String input) {
    return new ByteArrayInputStream(input.getBytes());
  }

  /** Make annoying idiom of Java more tolerable */
  public static void close(Closeable obj) {
    try {
      if(obj != null) obj.close();
    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  }

  public static void close(XMLStreamReader xml) {
    try {
      if(xml != null) xml.close();
    } catch (XMLStreamException e) {
      throw new RuntimeException(e);
    }
  }


  public static File file(String path) {
    return new File(path);
  }
}<commit_msg>stringReader<commit_after>package edu.umass.ciir.galagotools.utils;

import org.lemurproject.galago.tupleflow.FileUtility;
import org.lemurproject.galago.tupleflow.StreamCreator;
import org.lemurproject.galago.tupleflow.Utility;

import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * @author jfoley
 */
public class IO {
  public interface StringFunctor {
    public void process(String input);
  }

  public static void forEachLine(List<File> files, StringFunctor doWhat) {
    for(File fp : files) {
      forEachLine(fp, doWhat);
    }
  }

  public static void forEachLine(File fp, StringFunctor doWhat) {
    BufferedReader reader = null;
    try {
      reader = new BufferedReader(new InputStreamReader(StreamCreator.openInputStream(fp)));

      while(true) {
        String line = reader.readLine();
        if(line == null) break;
        doWhat.process(line);
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      close(reader);
    }
  }

  public static void forEachLine(BufferedReader reader, StringFunctor doWhat) {
    try {
      while (true) {
        String line = reader.readLine();
        if (line == null) break;
        doWhat.process(line);
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    } finally {
      close(reader);
    }
  }

  public static String slurp(File path) throws IOException {
    final StringBuilder contents = new StringBuilder();
    forEachLine(path, new StringFunctor() {
      @Override
      public void process(String input) {
        contents.append(input).append('\n');
      }
    });
    return contents.toString();
  }

  public static String slurp(String path) throws IOException {
    return slurp(new File(path));
  }

  public static List<String> slurpLines(String path) throws IOException {
    final ArrayList<String> lines = new ArrayList<String>();
    forEachLine(new File(path), new StringFunctor() {
      @Override
      public void process(String input) {
        lines.add(input);
      }
    });
    return lines;
  }

  public static void copyFile(String from, String to) {
    InputStream is = null;
    try {
      is = StreamCreator.realInputStream(from);
      Utility.copyStreamToFile(is, new File(to));
      is = null;
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      close(is);
    }
  }

  public static BufferedReader fileReader(String path) throws IOException {
    return fileReader(new File(path));
  }

  public static BufferedReader fileReader(File input) throws IOException {
    return new BufferedReader(new InputStreamReader(StreamCreator.openInputStream(input)));
  }

  public static void forEachLineInStr(String input, StringFunctor doWhat) {
    String[] lines = input.split("\n");
    for(String line : lines) {
      doWhat.process(line);
    }
  }

  public static PrintWriter printWriter(String output) throws IOException {
    FileUtility.makeParentDirectories(output);
    return new PrintWriter(StreamCreator.openOutputStream(output));
  }

  public static InputStream stringStream(String input) {
    return new ByteArrayInputStream(input.getBytes());
  }

  public static BufferedReader stringReader(String input) {
    return new BufferedReader(new StringReader(input));
  }

  /** Make annoying idiom of Java more tolerable */
  public static void close(Closeable obj) {
    try {
      if(obj != null) obj.close();
    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  }

  public static void close(XMLStreamReader xml) {
    try {
      if(xml != null) xml.close();
    } catch (XMLStreamException e) {
      throw new RuntimeException(e);
    }
  }


  public static File file(String path) {
    return new File(path);
  }
}
{'fraction_non_alphanumeric': 0.10435809923864531, 'fraction_numerical': 0.0, 'mean_word_length': 3.17937465715853, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4328941', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2189, 'n_words': 567}