<commit_before>package org.openworm.trackercommons

object ReadWrite {
  def readOne(s: String): Either[String, DataSet] = readOne(new java.io.File(s))

  def readOne(f: java.io.File): Either[String, DataSet] = ???

  def readAll(s: String): Either[String, Array[DataSet]] = readAll(new java.io.File(s))

  def readAll(f: java.io.File): Either[String, Array[DataSet]] = readOne(f) match {
    case Left(err) => Left(err)
    case Right(dset) => ???
  }
}<commit_msg>Nearly have file IO working.<commit_after>package org.openworm.trackercommons

import scala.util.control.NonFatal

object ReadWrite {
  def readOne(s: String): Either[String, DataSet] = readOne(new java.io.File(s))

  def readOne(f: java.io.File): Either[String, DataSet] = {
    try {
      val s = scala.io.Source.fromFile(f)
      try { Parser(s.mkString).left.map(err => s"Could not read ${f.getPath} because\n$err") }
      finally{ s.close }
    }
    catch { case NonFatal(_) => Left("Could not read " + f.getPath) }
  }

  def readAll(s: String): Either[String, Vector[DataSet]] = readAll(new java.io.File(s))

  def readAll(f: java.io.File): Either[String, Vector[DataSet]] = readOne(f) match {
    case Left(err) => Left(err)
    case Right(dset) =>
      if (dset.files.size <= 1) Right(Vector(dset))
      else {
        var ds = Vector(dset)
        var fs = dset.files
        fs.setRootFile(f)
        var i0 = 0
        var i1 = 0
        while (fs.indices.contains(i0) || fs.indices.contains(i1)) {
          val i = if (fs.indices.contains(i0-1)) i0-1 else i1+1
          val fi = fs.lookup(i) match {
            case None => return Left("Extended data set was supposed to contain ${fs.you(i)} but can't find corresponding file")
            case Some(x) => x
          }
          val di = readOne(fi) match {
            case Left(err) => return Left(err)
            case Right(x) => x
          }
          val fsi = di.files
          fsi.setRootFile(fi)
          if (fs.indices.contains(i-1) && !fsi.indices.contains(-1))
            return Left("Expected to find ${fs.you(i-1)} data set but not according to\n  $fi")
          if (fs.indices.contains(i+1) && !fsi.indices.contains(1))
            return Left("Expected to find ${fs.you(i+1)} data set but not according to\n  $fi")
          fs = fs.join(i, fsi) match {
            case Left(err) => return Left(err)
            case Right(x) => x
          }
          if (i < i0) {
            i0 = i
            ds = di +: ds
          }
          else {
            i1 = i
            ds = ds :+ di
          }
        }
        Right(ds)
      }
  }
}
{'fraction_non_alphanumeric': 0.14544055405925355, 'fraction_numerical': 0.008464794151596768, 'mean_word_length': 2.194103194103194, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4503004', 'n_tokens_mistral': 894, 'n_tokens_neox': 881, 'n_words': 218}