<commit_before>#!/bin/sh
exec scala "$0" "$@"
!#

import java.io._
import java.util.regex.Pattern
import scala.collection.mutable.Buffer

object Naive {
  def mapper(pair: (String, String)): (String, Int) = {
    var count = 0
    for (word <- pair._2.split("\\W+")) {
      count += 1
    }
    (pair._1, count)
  }

  def main(args: Array[String]) {
    val file = io.Source.fromFile("enwiki1.xml")

    val recordSeparator = 30.toChar
    val unitSeparator = 31.toChar
    val regex = Pattern.compile(
      recordSeparator + "(.*?)" + unitSeparator + "(.*?)" + recordSeparator)
    val matcher = regex.matcher(file.getLines.mkString)

    val output = Buffer[(String, Int)]()
    while (matcher.find()) {
      output += mapper((matcher.group(1), matcher.group(2)))
    }

    val writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream("naive-output.txt"), "utf-8"))
    for ((key, value) <- output.sortWith(_._1 < _._1)) {
      writer.write(key + " -> " + value + "\n")
    }
    writer.close()
  }
}<commit_msg>Updated naive_map for the new file format.<commit_after>#!/bin/sh
exec scala "$0" "$@"
!#

import java.io._
import java.util.regex.Pattern
import scala.collection.mutable.Buffer

object Naive {
  def mapper(pair: (String, String)): (String, Int) = {
    var count = 0
    for (word <- pair._2.split("\\W+")) {
      count += 1
    }
    (pair._1, count)
  }

  def main(args: Array[String]) {
    val file = io.Source.fromFile("enwiki9.xml")

    val unitSeparator = 31.toChar

    val output = Buffer[(String, Int)]()
    for (line <- file.getLines) {
      val split = line.split(unitSeparator)
      output += mapper((split(0), split(1)))
    }

    val writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream("naive-output.txt"), "utf-8"))
    for ((key, value) <- output.sortWith(_._1 < _._1)) {
      writer.write(key + " -> " + value + "\n")
    }
    writer.close()
  }
}
{'fraction_non_alphanumeric': 0.17335390946502058, 'fraction_numerical': 0.01440329218106996, 'mean_word_length': 2.89, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4076274', 'n_tokens_mistral': 688, 'n_tokens_neox': 667, 'n_words': 148}