<commit_before>package com.github.dronegator.nlp.main

import java.io.File

import com.github.dronegator.nlp.utils.CFG
import enumeratum.EnumEntry.Lowercase
import enumeratum._

/**
 * Created by cray on 8/17/16.
 */

object NLPTReplMain
  extends App
  with MainTools {
  lazy val cfg: CFG = CFG()

  sealed trait Command extends EnumEntry with Lowercase with Command.EnumApply

  object Command extends Enum[Command] {
    override def values: Seq[Command] = findValues

    trait EnumApply {
      def unapply(name: String) = withNameOption(name) filter ( x => x == this) isDefined
    }

    case object NGram1 extends Command

    case object NGram2 extends Command

    case object NGram3 extends Command

    case object Phrases extends Command

    case object Tokens extends Command
    
    case object Probability extends Command

    def unapply(name: String) = withNameOption(name)
  }

  sealed trait SubCommand extends EnumEntry with Lowercase with SubCommand.EnumUnaply

  object SubCommand extends Enum[SubCommand] {
    override def values: Seq[SubCommand] = findValues

    trait EnumUnaply {
      def unapply(name: String) = withNameOption(name) filter (this == _) isDefined
    }

    case object Dump extends SubCommand

    case object Stat extends SubCommand

    def unapply(name: String) = withNameOption(name)
  }

  import SubCommand._
  import Command._

  val Array(fileIn) = args

  lazy val vocabulary = load(new File(fileIn))

  val ConsoleReader = new jline.console.ConsoleReader()

  def task(): Unit = ConsoleReader.readLine("> ") match {
    case s : String => try {
      exec(s.split("\\s+").toList);
    } catch {
      case x  => printf("\n* Command failure: %s\n\n",x)
    } finally task()
    case _ => {}
  }

  def exec(args: List[String]) = args match {
    case NGram1() :: Dump() :: _ =>
      println("== ngram1")
      dump(vocabulary.ngrams1)
      println("==")

    case NGram2() :: Dump() :: _ =>
      println("== ngram1")
      dump(vocabulary.ngrams2)
      println("==")

    case NGram3() :: Dump() :: _ =>
      println("== ngram1")
      dump(vocabulary.ngrams3)
      println("==")

    case Phrases() :: Dump() :: _ =>
      println("== phrases")
      dump(vocabulary.phrases)
      println("==")

    case Tokens() :: Dump() :: _ =>
      println("== phrases")
      dump(vocabulary.toToken, vocabulary.toToken.size)
      println("==")

    case NGram1() :: _ =>
      println(s"== ngram1 size = ${vocabulary.ngrams1.size}")

    case NGram2() :: _ =>
      println(s"== ngram1 size = ${vocabulary.ngrams2.size}")

    case NGram3() :: _ =>
      println(s"== ngram1 size = ${vocabulary.ngrams3.size}")

    case Phrases() :: _ =>
      println(s"== phrases size = ${vocabulary.phrases.size}")

    case Tokens() :: _ =>
      println(s"== tokens size = ${vocabulary.toToken.size}")

    case Probability() :: args =>
      val phrase = args.mkString(" ")
      println(s"== phrase = ${phrase}")

    case _ =>
      Command.values foreach { value =>
        println(value)
      }
  }

  try {
    task()
  } finally {
    println("end")
    ConsoleReader.clearScreen()
    ConsoleReader.shutdown()
  }
}<commit_msg>Implement Everything command<commit_after>package com.github.dronegator.nlp.main

import java.io.File

import com.github.dronegator.nlp.utils.CFG
import enumeratum.EnumEntry.Lowercase
import enumeratum._

/**
 * Created by cray on 8/17/16.
 */

object NLPTReplMain
  extends App
  with MainTools {
  lazy val cfg: CFG = CFG()

  sealed trait Command extends EnumEntry with Lowercase with Command.EnumApply

  object Command extends Enum[Command] {
    override def values: Seq[Command] = findValues

    trait EnumApply {
      def unapply(name: String) = withNameOption(name) filter ( x => x == this) isDefined
    }

    case object NGram1 extends Command

    case object NGram2 extends Command

    case object NGram3 extends Command

    case object Phrases extends Command

    case object Tokens extends Command
    
    case object Probability extends Command

    case object Everything extends Command

    def unapply(name: String) = withNameOption(name)
  }

  sealed trait SubCommand extends EnumEntry with Lowercase with SubCommand.EnumUnaply

  object SubCommand extends Enum[SubCommand] {
    override def values: Seq[SubCommand] = findValues

    trait EnumUnaply {
      def unapply(name: String) = withNameOption(name) filter (this == _) isDefined
    }

    case object Dump extends SubCommand

    case object Stat extends SubCommand

    def unapply(name: String) = withNameOption(name)
  }

  import SubCommand._
  import Command._

  val Array(fileIn) = args

  lazy val vocabulary = load(new File(fileIn))

  val ConsoleReader = new jline.console.ConsoleReader()

  def task(): Unit = ConsoleReader.readLine("> ") match {
    case s : String => try {
      exec(s.split("\\s+").toList);
    } catch {
      case x  => printf("\n* Command failure: %s\n\n",x)
    } finally task()
    case _ => {}
  }

  def exec(args: List[String]) = args match {
    case NGram1() :: Dump() :: _ =>
      println("== ngram1")
      dump(vocabulary.ngrams1)
      println("==")

    case NGram2() :: Dump() :: _ =>
      println("== ngram2")
      dump(vocabulary.ngrams2)
      println("==")

    case NGram3() :: Dump() :: _ =>
      println("== ngram3")
      dump(vocabulary.ngrams3)
      println("==")

    case Phrases() :: Dump() :: _ =>
      println("== phrases")
      dump(vocabulary.phrases)
      println("==")

    case Tokens() :: Dump() :: _ =>
      println("== tokens")
      dump(vocabulary.toToken, vocabulary.toToken.size)
      println("==")

    case NGram1() :: _ =>
      println(s"== ngram1 size = ${vocabulary.ngrams1.size}")

    case NGram2() :: _ =>
      println(s"== ngram2 size = ${vocabulary.ngrams2.size}")

    case NGram3() :: _ =>
      println(s"== ngram3 size = ${vocabulary.ngrams3.size}")

    case Phrases() :: _ =>
      println(s"== phrases size = ${vocabulary.phrases.size}")

    case Tokens() :: _ =>
      println(s"== tokens size = ${vocabulary.toToken.size}")

    case Everything() :: _ =>
      println(
        s"""
           |Statistic:
           | - ngram1 size = ${vocabulary.ngrams1.size}
           | - ngram2 size = ${vocabulary.ngrams2.size}
           | - ngram3 size = ${vocabulary.ngrams3.size}
           | - phrases size = ${vocabulary.phrases.size}
           | - tokens size = ${vocabulary.toToken.size}
         """.stripMargin)

    case Probability() :: args =>
      val phrase = args.mkString(" ")
      println(s"== phrase = ${phrase}")

    case _ =>
      Command.values foreach { value =>
        println(value)
      }
  }

  try {
    task()
  } finally {
    println("end")
    ConsoleReader.clearScreen()
    ConsoleReader.shutdown()
  }
}
{'fraction_non_alphanumeric': 0.13338224504768892, 'fraction_numerical': 0.00851063829787234, 'mean_word_length': 2.79933110367893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4555025', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2148, 'n_words': 542}