<commit_before>val outFile = opts.woFile("outFile","The file to write the merged counts to")
val args = opts.listOfArgs("files","The list of files to merge")
opts.verify

namedTask("Merge counts") {
  val readers = args.map { arg => new util.PeekableIterator(opts.openInput(arg).getLines) }
  val out = opts.openOutput(outFile)

  while(readers.exists(_.hasNext)) {
    val lines = readers.flatMap(_.peek)
    val keyCounts = lines.map( line => {
      (line.substring(0,line.lastIndexOf("|||") + 4), line.substring(line.lastIndexOf("|||")+4).toInt)
    })
    val minKey = keyCounts.map(_._1).min
    val total = keyCounts.filter(_._1==minKey).map(_._2).sum
    val active = readers.filter { 
      reader => {
        reader.peek match {
          case Some(line) => line.startsWith(minKey)
          case None => false
        }
      }
    }
    active.foreach(_.next)
    out.println(minKey + total)
  }
  out.flush
  out.close
}<commit_msg>Fix bug with duplicates in merge<commit_after>val outFile = opts.woFile("outFile","The file to write the merged counts to")
val args = opts.listOfArgs("files","The list of files to merge")
opts.verify

namedTask("Merge counts") {
  val readers = args.map { arg => new util.PeekableIterator(opts.openInput(arg).getLines) }
  val out = opts.openOutput(outFile)

  while(readers.exists(_.hasNext)) {
    val lines = readers.flatMap(_.peek)
    val keyCounts = lines.map( line => {
      (line.substring(0,line.lastIndexOf("|||")), line.substring(line.lastIndexOf("|||") + 4).toInt)
    })
    val minKey = keyCounts.map(_._1).min
    val total = keyCounts.filter(_._1==minKey).map(_._2).sum
    val active = readers.filter { 
      reader => {
        reader.peek match {
          case Some(line) => line.substring(0,line.lastIndexOf("|||")) == minKey
          case None => false
        }
      }
    }
    active.foreach(_.next)
    out.println(minKey + "||| " + total)
  }
  out.flush
  out.close
}
{'fraction_non_alphanumeric': 0.1551901336073998, 'fraction_numerical': 0.006166495375128468, 'mean_word_length': 3.0903361344537816, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430476', 'n_tokens_mistral': 659, 'n_tokens_neox': 650, 'n_words': 134}