<commit_before>package ru.scalalaz.gen

import java.io.File
import java.nio.file.{Files, Path, Paths}

import cats.data.Validated
import ru.scalalaz.gen.parsing.{EpisodeParseError, EpisodeParser}

import scala.collection.JavaConversions._

object Main extends App {

  val markdownDir   = getClass.getResource("/md").getPath
  val targetPath    = Paths.get("target/site")
  val targetRssPath = Paths.get("target/site/rss")

  cleanDir(targetPath)
  createDir(targetRssPath)

  val parsed = EpisodeParser.fromDirectory(markdownDir)
  if (parsed.exists(_.isInvalid)) {
    reportErrors(parsed)
    sys.exit(1)
  }

//  def mdFiles: Seq[File] = {
//    Paths.get(dir).toFile.list()
//      .map(Paths.get(_).toFile)
//      .filter(f => !f.isDirectory && f.getName.endsWith(".md"))
//  }

  private def reportErrors(results: Seq[Validated[EpisodeParseError, EpisodeFile]]): Unit = {
    parsed.filter(_.isInvalid)
      .foreach(inv => {
        val msg = s"Error occured while parsing file:\n\t $inv"
        println(msg)
      })
  }

  private def cleanDir(path: Path): Unit = {
    val file = path.toFile
    if (file.isDirectory) {
      file.list().map(path.resolve).foreach(cleanDir)
      file.delete()
    } else
      file.delete()<commit_msg>typo fix<commit_after>package ru.scalalaz.gen

import java.io.File
import java.nio.file.{Files, Path, Paths}

import cats.data.Validated
import ru.scalalaz.gen.parsing.{EpisodeParseError, EpisodeParser}

import scala.collection.JavaConversions._

object Main extends App {

  val markdownDir   = getClass.getResource("/md").getPath
  val targetPath    = Paths.get("target/site")
  val targetRssPath = Paths.get("target/site/rss")

  cleanDir(targetPath)
  createDir(targetRssPath)

  val parsed = EpisodeParser.fromDirectory(markdownDir)
  if (parsed.exists(_.isInvalid)) {
    reportErrors(parsed)
    sys.exit(1)
  }

//  def mdFiles: Seq[File] = {
//    Paths.get(dir).toFile.list()
//      .map(Paths.get(_).toFile)
//      .filter(f => !f.isDirectory && f.getName.endsWith(".md"))
//  }

  private def reportErrors(results: Seq[Validated[EpisodeParseError, EpisodeFile]]): Unit = {
    parsed.filter(_.isInvalid)
      .foreach(inv => {
        val msg = s"Error occurred while parsing file:\n\t $inv"
        println(msg)
      })
  }

  private def cleanDir(path: Path): Unit = {
    val file = path.toFile
    if (file.isDirectory) {
      file.list().map(path.resolve).foreach(cleanDir)
      file.delete()
    } else
      file.delete()
{'fraction_non_alphanumeric': 0.14262820512820512, 'fraction_numerical': 0.0008012820512820513, 'mean_word_length': 3.320069204152249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7308523', 'n_tokens_mistral': 858, 'n_tokens_neox': 845, 'n_words': 152}