<commit_before>package util

import java.io.File

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.{ ActorMaterializer, IOResult }
import akka.stream.scaladsl._
import akka.util.ByteString
import hmda.parser.fi.lar.LarDatParser
import hmda.parser.fi.ts.TsDatParser

import scala.concurrent.ExecutionContext

/*
Takes a .DAT file and converts to pipe delimited CSV (2017)
 */
object Dat2Csv {

  implicit val system: ActorSystem = ActorSystem()
  implicit val materializer: ActorMaterializer = ActorMaterializer()
  implicit val ec: ExecutionContext = system.dispatcher

  def main(args: Array[String]): Unit = {
    if (args.length != 2) {
      throw new Exception("Please provide input .DAT and output .txt files")
    } else {
      val datFilePath = args(0)
      val txtFilePath = args(1)

      val datFile = new File(datFilePath)
      val txtFile = new File(txtFilePath)

      val source = FileIO.fromPath(datFile.toPath)
      val sink = FileIO.toPath(txtFile.toPath)

      val framing = Framing.delimiter(ByteString("\n"), 2048, allowTruncation = true)

      val dat2CsvFlow: Flow[String, String, NotUsed] = {
        Flow[String]
          .map { s =>
            if (s.charAt(0).toString == "1") {
              TsDatParser(s).toCSV
            } else {
              LarDatParser(s).toCSV
            }
          }
      }

      val convert = source
        .via(framing)<commit_msg>use Paths.get instead of new File(...).toPath<commit_after>package util

import java.nio.file.Paths

import akka.NotUsed
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import akka.stream.scaladsl._
import akka.util.ByteString
import hmda.parser.fi.lar.LarDatParser
import hmda.parser.fi.ts.TsDatParser

import scala.concurrent.ExecutionContext

/*
Takes a .DAT file and converts to pipe delimited CSV (2017)
 */
object Dat2Csv {

  implicit val system: ActorSystem = ActorSystem()
  implicit val materializer: ActorMaterializer = ActorMaterializer()
  implicit val ec: ExecutionContext = system.dispatcher

  def main(args: Array[String]): Unit = {
    if (args.length != 2) {
      throw new Exception("Please provide input .DAT and output .txt files")
    } else {
      val datFilePath = args(0)
      val txtFilePath = args(1)

      val source = FileIO.fromPath(Paths.get(datFilePath))
      val sink = FileIO.toPath(Paths.get(txtFilePath))

      val framing = Framing.delimiter(ByteString("\n"), 2048, allowTruncation = true)

      val dat2CsvFlow: Flow[String, String, NotUsed] = {
        Flow[String]
          .map { s =>
            if (s.charAt(0).toString == "1") {
              TsDatParser(s).toCSV
            } else {
              LarDatParser(s).toCSV
            }
          }
      }

      val convert = source
        .via(framing)
{'fraction_non_alphanumeric': 0.10264663805436337, 'fraction_numerical': 0.01072961373390558, 'mean_word_length': 2.7594086021505375, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3306170', 'n_tokens_mistral': 901, 'n_tokens_neox': 857, 'n_words': 218}