<commit_before>package vtpassim

import org.apache.spark.sql.SparkSession

import scala.collection.mutable.StringBuilder
import scala.util.Try
import scala.xml.pull._

object TEIPages {
  def main(args: Array[String]) {
    val spark = SparkSession.builder().appName("TEIPages Import").getOrCreate()
    import spark.implicits._

    spark.sparkContext.hadoopConfiguration
      .set("mapreduce.input.fileinputformat.input.dir.recursive", "true")

    spark.sparkContext.binaryFiles(args(0), spark.sparkContext.defaultParallelism)
      .filter(_._1.endsWith(".xml"))
      .flatMap( in => {
        val buf = new StringBuilder
        var buffering = false
        var seq = -1
        var id = ""

        val pass = new XMLEventReader(scala.io.Source.fromURL(in._1))
        pass.flatMap { event =>
          event match {
            case EvElemStart(_, "idno", attr, _) => {
              if ( id == "" ) {
                buffering = true
                buf.clear
              }
              None
            }
            case EvElemEnd(_, "idno") => {
              if ( id == "" ) {
                val rawid = buf.toString.trim
                id = (if ( rawid != "" ) rawid else in._1)
                buffering = false
                buf.clear
              }
              None
            }
            case EvElemStart(_, "pb", attr, _) => {
              val rec = if ( buffering ) {
                Some((f"$id%s_$seq%04d", id, seq, buf.toString.trim))
              } else {
                None
              }
              buffering = true
              seq += 1
              buf.clear
              rec
            }
            case EvElemEnd(_, "text") => {
              if ( buffering ) {
                seq += 1
                val text = buf.toString.trim
                buffering = false
                buf.clear
                Some((f"$id%s_$seq%04d", id, seq, text))
              } else {
                None
              }
            }
            case EvText(t) => {
              if ( buffering ) buf ++= t
              None
            }
            case EvEntityRef(n) => {
              if ( buffering ) buf ++= "&" + n + ";"
              None
            }
            case _ => None
          }
        }
      })
      .toDF("id", "book", "seq", "text")
      .write.save(args(1))
    spark.stop()
  }
}
<commit_msg>Don't try to guess ID using <idno>; just use filename.<commit_after>package vtpassim

import org.apache.spark.sql.SparkSession

import scala.collection.mutable.StringBuilder
import scala.util.Try
import scala.xml.pull._

object TEIPages {
  def main(args: Array[String]) {
    val spark = SparkSession.builder().appName("TEIPages Import").getOrCreate()
    import spark.implicits._

    spark.sparkContext.hadoopConfiguration
      .set("mapreduce.input.fileinputformat.input.dir.recursive", "true")

    spark.sparkContext.binaryFiles(args(0), spark.sparkContext.defaultParallelism)
      .filter(_._1.endsWith(".xml"))
      .flatMap( in => {
        val fname = new java.io.File(new java.net.URL(in._1).toURI)
        val id = fname.getName.replaceAll(".xml$", "")
        val buf = new StringBuilder
        var buffering = false
        var seq = -1

        val pass = new XMLEventReader(scala.io.Source.fromURL(in._1))
        pass.flatMap { event =>
          event match {
            case EvElemStart(_, "pb", attr, _) => {
              val rec = if ( buffering ) {
                Some((f"$id%s_$seq%04d", id, seq, buf.toString.trim))
              } else {
                None
              }
              buffering = true
              seq += 1
              buf.clear
              rec
            }
            case EvElemEnd(_, "text") => {
              if ( buffering ) {
                seq += 1
                val text = buf.toString.trim
                buffering = false
                buf.clear
                Some((f"$id%s_$seq%04d", id, seq, text))
              } else {
                None
              }
            }
            case EvText(t) => {
              if ( buffering ) buf ++= t
              None
            }
            case EvEntityRef(n) => {
              if ( buffering ) buf ++= "&" + n + ";"
              None
            }
            case _ => None
          }
        }
      })
      .toDF("id", "book", "seq", "text")
      .write.save(args(1))
    spark.stop()
  }
}

{'fraction_non_alphanumeric': 0.12173518055870997, 'fraction_numerical': 0.005450828980240745, 'mean_word_length': 1.299738903394256, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3719319', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1253, 'n_words': 263}