<commit_before>//===========================================================================
//    Copyright 2014 Delving B.V.
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//===========================================================================

package services

import java.io._
import java.util.zip.{GZIPInputStream, ZipEntry, ZipFile}

import org.apache.commons.io.FileUtils._
import org.apache.commons.io.input.{BOMInputStream, CountingInputStream}
import play.api.Logger

import scala.io.Source
import scala.sys.process._

object FileHandling {

  def clearDir(dir: File) = {
    deleteQuietly(dir)
    dir.mkdir()
    dir
  }

  def ensureGitRepo(directory: File): Boolean = {
    val hiddenGit = new File(directory, ".git")
    if (!hiddenGit.exists()) {
      val init = s"git init ${directory.getAbsolutePath}".!!
      Logger.info(s"git init: $init")
    }
    hiddenGit.exists()
  }

  def gitCommit(file: File, comment: String): Boolean = {
    // assuming all files are in the git root?
    val home = file.getParentFile.getAbsolutePath
    val name = file.getName
    val addCommand = Process(Seq("git", "-C", home, "add", name))
    Logger.info(s"git add: $addCommand")
    val add = addCommand.!!
    val commitCommand = Process(Seq("git", "-C", home, "commit", "-m", comment, name))
    Logger.info(s"git commit: $commitCommand")
    val commit = commitCommand.!!
    true
  }

  def reader(file: File) = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"))

  def writer(file: File) = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"))

  def appender(file: File) = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true), "UTF-8"))

  def writer(outputStream: OutputStream) = new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8"))

  abstract class ReadProgress {
    def getPercentRead: Int
  }

  class FileReadProgress(fileSize: Long, counter: CountingInputStream) extends ReadProgress {
    override def getPercentRead: Int = ((100 * counter.getByteCount) / fileSize).toInt
  }

  def sourceFromFile(file: File): (Source, ReadProgress) = {
    val name = file.getName
    if (name.endsWith(".zip")) {
      val zc = new ZipConcatXML(new ZipFile(file))
      (zc, zc.readProgress)
    }
    else if (name.endsWith(".xml.gz")) {
      val is = new FileInputStream(file)
      val cs = new CountingInputStream(is)
      val gz = new GZIPInputStream(cs)
      val bis = new BOMInputStream(gz)
      (Source.fromInputStream(bis, "UTF-8"), new FileReadProgress(file.length(), cs))
    }
    else if (name.endsWith(".xml")) {
      val is = new FileInputStream(file)
      val cs = new CountingInputStream(is)
      val bis = new BOMInputStream(cs)
      (Source.fromInputStream(bis, "UTF-8"), new FileReadProgress(file.length(), cs))
    }
    else {
      throw new RuntimeException(s"Unrecognized extension: $name")
    }
  }

  def getZipFileSize(zipFile: ZipFile) = {
    val e = zipFile.entries()
    var size = 0L
    while (e.hasMoreElements) {
      size += e.nextElement().getSize
    }
    size
  }

  //  def createDigest = MessageDigest.getInstance("SHA1")
  //  def hex(digest: MessageDigest) = digest.digest().map("%02X" format _).mkString

  private def unzipXML(file: File, inputStream: InputStream) = {
    val stream = if (file.getName.endsWith(".gz")) new GZIPInputStream(inputStream) else inputStream
    new BOMInputStream(stream)
  }

  class ZipConcatXML(val file: ZipFile) extends Source {

    var entries = file.entries()
    var entry: Option[ZipEntry] = None
    var entryInputStream: InputStream = null
    var nextChar: Int = -1
    var charCount = 0L

    class ZipReadProgress(size: Long) extends ReadProgress {
      override def getPercentRead: Int = ((100 * charCount) / size).toInt
    }

    class ZipEntryIterator extends Iterator[Char] {
      override def hasNext: Boolean = {
        entry match {
          case Some(zipEntry) =>
            if (nextChar >= 0) {
              true
            }
            else {
              entry = None
              hasNext
            }

          case None =>
            if (entries.hasMoreElements) {
              // check for .xml
              entry = Some(entries.nextElement())
              val is = file.getInputStream(entry.get)
              entryInputStream = new BOMInputStream(new BufferedInputStream(is))
              nextChar = entryInputStream.read()
              charCount += 1
              hasNext
            }
            else {
              false
            }
        }
      }

      override def next(): Char = {
        val c = nextChar
        nextChar = entryInputStream.read()
        charCount += 1
        c.toChar
      }
    }

    override protected val iter: Iterator[Char] = new ZipEntryIterator

    def readProgress = new ZipReadProgress(getZipFileSize(file))
  }

}<commit_msg>Reading with a UTF-8 InputStreamReader in the ZipConcatXML source.<commit_after>//===========================================================================
//    Copyright 2014 <NAME>.V.
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//===========================================================================

package services

import java.io._
import java.util.zip.{GZIPInputStream, ZipEntry, ZipFile}

import org.apache.commons.io.FileUtils._
import org.apache.commons.io.input.{BOMInputStream, CountingInputStream}
import play.api.Logger

import scala.io.Source
import scala.sys.process._

object FileHandling {

  def clearDir(dir: File) = {
    deleteQuietly(dir)
    dir.mkdir()
    dir
  }

  def ensureGitRepo(directory: File): Boolean = {
    val hiddenGit = new File(directory, ".git")
    if (!hiddenGit.exists()) {
      val init = s"git init ${directory.getAbsolutePath}".!!
      Logger.info(s"git init: $init")
    }
    hiddenGit.exists()
  }

  def gitCommit(file: File, comment: String): Boolean = {
    // assuming all files are in the git root?
    val home = file.getParentFile.getAbsolutePath
    val name = file.getName
    val addCommand = Process(Seq("git", "-C", home, "add", name))
    Logger.info(s"git add: $addCommand")
    val add = addCommand.!!
    val commitCommand = Process(Seq("git", "-C", home, "commit", "-m", comment, name))
    Logger.info(s"git commit: $commitCommand")
    val commit = commitCommand.!!
    true
  }

  def reader(file: File) = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"))

  def writer(file: File) = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"))

  def appender(file: File) = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true), "UTF-8"))

  def writer(outputStream: OutputStream) = new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8"))

  abstract class ReadProgress {
    def getPercentRead: Int
  }

  class FileReadProgress(fileSize: Long, counter: CountingInputStream) extends ReadProgress {
    override def getPercentRead: Int = ((100 * counter.getByteCount) / fileSize).toInt
  }

  def sourceFromFile(file: File): (Source, ReadProgress) = {
    val name = file.getName
    if (name.endsWith(".zip")) {
      val zc = new ZipConcatXML(new ZipFile(file))
      (zc, zc.readProgress)
    }
    else if (name.endsWith(".xml.gz")) {
      val is = new FileInputStream(file)
      val cs = new CountingInputStream(is)
      val gz = new GZIPInputStream(cs)
      val bis = new BOMInputStream(gz)
      (Source.fromInputStream(bis, "UTF-8"), new FileReadProgress(file.length(), cs))
    }
    else if (name.endsWith(".xml")) {
      val is = new FileInputStream(file)
      val cs = new CountingInputStream(is)
      val bis = new BOMInputStream(cs)
      (Source.fromInputStream(bis, "UTF-8"), new FileReadProgress(file.length(), cs))
    }
    else {
      throw new RuntimeException(s"Unrecognized extension: $name")
    }
  }

  def getZipFileSize(zipFile: ZipFile) = {
    val e = zipFile.entries()
    var size = 0L
    while (e.hasMoreElements) {
      size += e.nextElement().getSize
    }
    size
  }

  //  def createDigest = MessageDigest.getInstance("SHA1")
  //  def hex(digest: MessageDigest) = digest.digest().map("%02X" format _).mkString

  private def unzipXML(file: File, inputStream: InputStream) = {
    val stream = if (file.getName.endsWith(".gz")) new GZIPInputStream(inputStream) else inputStream
    new BOMInputStream(stream)
  }

  class ZipConcatXML(val file: ZipFile) extends Source {

    var entries = file.entries()
    var entry: Option[ZipEntry] = None
    var entryReader: Reader = null
    var nextChar: Int = -1
    var charCount = 0L

    class ZipReadProgress(size: Long) extends ReadProgress {
      override def getPercentRead: Int = ((100 * charCount) / size).toInt
    }

    class ZipEntryIterator extends Iterator[Char] {
      override def hasNext: Boolean = {
        entry match {
          case Some(zipEntry) =>
            if (nextChar >= 0) {
              true
            }
            else {
              entry = None
              hasNext
            }

          case None =>
            if (entries.hasMoreElements) {
              // check for .xml
              entry = Some(entries.nextElement())
              val is = file.getInputStream(entry.get)
              entryReader = new InputStreamReader(new BOMInputStream(new BufferedInputStream(is)), "UTF-8")
              nextChar = entryReader.read()
              charCount += 1
              hasNext
            }
            else {
              false
            }
        }
      }

      override def next(): Char = {
        val c = nextChar
        nextChar = entryReader.read()
        charCount += 1
        c.toChar
      }
    }

    override protected val iter: Iterator[Char] = new ZipEntryIterator

    def readProgress = new ZipReadProgress(getZipFileSize(file))
  }

}
{'fraction_non_alphanumeric': 0.13569537028581866, 'fraction_numerical': 0.005478951693909232, 'mean_word_length': 2.8093913043478262, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112018', 'n_tokens_mistral': 3216, 'n_tokens_neox': 3086, 'n_words': 887}