<commit_before>package scoverage

import java.io._
import scala.xml.{XML, Utility, Node}

/** @author <NAME> */
object IOUtils {

  private val MeasurementsPrefix = "scoverage.measurements."
  private val CoverageFileName = "scoverage.coverage"

  def coverageFile(dataDir: File): File = coverageFile(dataDir.getAbsolutePath)
  def coverageFile(dataDir: String): File = new File(dataDir + "/" + CoverageFileName)

  def measurementFile(dataDir: File): File = measurementFile(dataDir.getAbsolutePath)
  def measurementFile(dataDir: String): File = new File(dataDir + "/" + MeasurementsPrefix + Thread.currentThread.getId)

  def findMeasurementFiles(dataDir: String): Array[File] = findMeasurementFiles(new File(dataDir))
  def findMeasurementFiles(dataDir: File): Array[File] = dataDir.listFiles(new FileFilter {
    override def accept(pathname: File): Boolean = pathname.getName.startsWith(MeasurementsPrefix)
  })

  // loads all the invoked statement ids from the given files
  def invoked(files: Seq[File]): Seq[Int] = {
    files.flatMap {
      file =>
      val reader = new BufferedReader(new FileReader(file))
      val line = reader.readLine()
      reader.close()
      line.split(";").filterNot(_.isEmpty).map(_.toInt)
    }
  }

  /**
   * Write out coverage data to the given data directory
   */
  def serialize(coverage: Coverage, dataDir: String): Unit = serialize(coverage, coverageFile(dataDir))
  def serialize(coverage: Coverage, file: File): Unit = {
    val out = new BufferedWriter(new FileWriter(file))
    out.write(serialize(coverage).toString())
    out.close()
  }

  def serialize(coverage: Coverage): Node = {
    val lines = coverage.statements.map(stmt => {
      <statement>
        <source>
          {stmt.source}
        </source>
        <package>
          {stmt.location._package}
        </package>
        <class>
          {stmt.location._class}
        </class>
        <classType>
          {stmt.location.classType.toString}
        </classType>
        <method>
          {stmt.location.method}
        </method>
        <id>
          {stmt.id.toString}
        </id>
        <start>
          {stmt.start.toString}
        </start>
        <end>
          {stmt.end.toString}
        </end>
        <line>
          {stmt.line.toString}
        </line>
        <description>
          {stmt.desc}
        </description>
        <symbolName>
          {stmt.symbolName}
        </symbolName>
        <treeName>
          {stmt.treeName}
        </treeName>
        <branch>
          {stmt.branch.toString}
        </branch>
        <count>
          {stmt.count.toString}
        </count>
      </statement>
    })
    Utility.trim(<statements>
      {lines}
    </statements>)
  }

  def deserialize(str: String): Coverage = {
    val xml = XML.loadString(str)
    val statements = xml \ "statement" map (node => {
      val source = (node \ "source").text
      val count = (node \ "count").text.toInt
      val branch = (node \ "branch").text.toBoolean
      val _package = (node \ "package").text
      val _class = (node \ "class").text
      val method = (node \ "method").text
      val treeName = (node \ "treeName").text
      val symbolName = (node \ "symbolName").text
      val id = (node \ "id").text.toInt
      val line = (node \ "line").text.toInt
      val desc = (node \ "description").text
      val start = (node \ "start").text.toInt
      val end = (node \ "end").text.toInt
      val classType = (node \ "classType").text match {
        case "Trait" => ClassType.Trait
        case "Object" => ClassType.Object
        case _ => ClassType.Class
      }
      MeasuredStatement(source, Location(_package, _class, classType, method), id, start, end, line, desc, symbolName,
        treeName, branch, count)
    })

    val coverage = Coverage()
    for ( statement <- statements )
      coverage.add(statement)
    coverage
  }

  def deserialize(file: File): Coverage = deserialize(new FileReader(file))
  def deserialize(reader: Reader): Coverage = {
    val reader = new BufferedReader(reader)
    val sb = new StringBuilder
    var line = reader.readLine()
    while (line != null) {
      sb.append(line)
      line = reader.readLine()
    }
    val coverage = deserialize(sb.toString)
    reader.close()<commit_msg>Fixed build error<commit_after>package scoverage

import java.io._
import scala.xml.{XML, Utility, Node}

/** @author <NAME> */
object IOUtils {

  private val MeasurementsPrefix = "scoverage.measurements."
  private val CoverageFileName = "scoverage.coverage"

  def coverageFile(dataDir: File): File = coverageFile(dataDir.getAbsolutePath)
  def coverageFile(dataDir: String): File = new File(dataDir + "/" + CoverageFileName)

  def measurementFile(dataDir: File): File = measurementFile(dataDir.getAbsolutePath)
  def measurementFile(dataDir: String): File = new File(dataDir + "/" + MeasurementsPrefix + Thread.currentThread.getId)

  def findMeasurementFiles(dataDir: String): Array[File] = findMeasurementFiles(new File(dataDir))
  def findMeasurementFiles(dataDir: File): Array[File] = dataDir.listFiles(new FileFilter {
    override def accept(pathname: File): Boolean = pathname.getName.startsWith(MeasurementsPrefix)
  })

  // loads all the invoked statement ids from the given files
  def invoked(files: Seq[File]): Seq[Int] = {
    files.flatMap {
      file =>
      val reader = new BufferedReader(new FileReader(file))
      val line = reader.readLine()
      reader.close()
      line.split(";").filterNot(_.isEmpty).map(_.toInt)
    }
  }

  /**
   * Write out coverage data to the given data directory
   */
  def serialize(coverage: Coverage, dataDir: String): Unit = serialize(coverage, coverageFile(dataDir))
  def serialize(coverage: Coverage, file: File): Unit = {
    val out = new BufferedWriter(new FileWriter(file))
    out.write(serialize(coverage).toString())
    out.close()
  }

  def serialize(coverage: Coverage): Node = {
    val lines = coverage.statements.map(stmt => {
      <statement>
        <source>
          {stmt.source}
        </source>
        <package>
          {stmt.location._package}
        </package>
        <class>
          {stmt.location._class}
        </class>
        <classType>
          {stmt.location.classType.toString}
        </classType>
        <method>
          {stmt.location.method}
        </method>
        <id>
          {stmt.id.toString}
        </id>
        <start>
          {stmt.start.toString}
        </start>
        <end>
          {stmt.end.toString}
        </end>
        <line>
          {stmt.line.toString}
        </line>
        <description>
          {stmt.desc}
        </description>
        <symbolName>
          {stmt.symbolName}
        </symbolName>
        <treeName>
          {stmt.treeName}
        </treeName>
        <branch>
          {stmt.branch.toString}
        </branch>
        <count>
          {stmt.count.toString}
        </count>
      </statement>
    })
    Utility.trim(<statements>
      {lines}
    </statements>)
  }

  def deserialize(str: String): Coverage = {
    val xml = XML.loadString(str)
    val statements = xml \ "statement" map (node => {
      val source = (node \ "source").text
      val count = (node \ "count").text.toInt
      val branch = (node \ "branch").text.toBoolean
      val _package = (node \ "package").text
      val _class = (node \ "class").text
      val method = (node \ "method").text
      val treeName = (node \ "treeName").text
      val symbolName = (node \ "symbolName").text
      val id = (node \ "id").text.toInt
      val line = (node \ "line").text.toInt
      val desc = (node \ "description").text
      val start = (node \ "start").text.toInt
      val end = (node \ "end").text.toInt
      val classType = (node \ "classType").text match {
        case "Trait" => ClassType.Trait
        case "Object" => ClassType.Object
        case _ => ClassType.Class
      }
      MeasuredStatement(source, Location(_package, _class, classType, method), id, start, end, line, desc, symbolName,
        treeName, branch, count)
    })

    val coverage = Coverage()
    for ( statement <- statements )
      coverage.add(statement)
    coverage
  }

  def deserialize(file: File): Coverage = deserialize(new FileReader(file))
  def deserialize(reader: Reader): Coverage = {
    val buffered = new BufferedReader(reader)
    val sb = new StringBuilder
    var line = buffered.readLine()
    while (line != null) {
      sb.append(line)
      line = buffered.readLine()
    }
    val coverage = deserialize(sb.toString())
    reader.close()
{'fraction_non_alphanumeric': 0.1293324028843917, 'fraction_numerical': 0.0, 'mean_word_length': 2.4603621730382295, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5093282', 'n_tokens_mistral': 2586, 'n_tokens_neox': 2548, 'n_words': 609}