<commit_before>package org.scalabha.tree

import org.scalabha.log.SimpleLogger
import java.io.{OutputStreamWriter, BufferedWriter}
import org.clapper.argot.{ArgotUsageException, ArgotParser, ArgotConverters}
import org.scalabha.model.TreeNode
import collection.Set
import collection.mutable.HashMap
import grizzled.string
import io.BufferedSource

object TagChecker {

  import ArgotConverters._

  val parser = new ArgotParser("org.scalabha.preproc.Tokenizer", preUsage = Some("Version 0.0"))
  val help = parser.flag[Boolean](List("h", "help"), "print help")
  val input = parser.option[String](List("i", "input"), "FILE", "input inputFile in which to check tags")
  val other = parser.option[String](List("other"), "FILE", "(optional) other inputFile to compare against the input inputFile")
  val log = new SimpleLogger("org.scalabha.tree.TagChecker", SimpleLogger.WARN, new BufferedWriter(new OutputStreamWriter(System.err)))
  val noLog = new SimpleLogger("org.scalabha.tree.TagChecker", SimpleLogger.NONE, new BufferedWriter(new OutputStreamWriter(System.err)))


  abstract class TagCheckResult

  case class Success() extends TagCheckResult {
    override def toString(): String = {
      "OK"
    }

    val bool = true
  }

  case class Fail[T](left: T, right: T) extends TagCheckResult {
    override def toString(): String = {
      "FAIL: <<%s>>!=<<%s>>".format(left, right)
    }

    val bool = false
  }

  def spprintRepr(map: Map[String, Int], join: String): String = {
    val regex = "[^(]+\\((.*)\\)".r
    val regex(string) = map.toList.sorted.toString
    string.replace(", ", join)
  }

  /**
   * Return true iff both trees have the same tag counts
   */
  def apply(left: TreeNode, right: TreeNode): TagCheckResult = {
    val leftCounts = left.getTagCounts()
    val rightCounts = right.getTagCounts()
    if (leftCounts == rightCounts)
      Success()
    else
      Fail[HashMap[String, Int]](leftCounts, rightCounts)
  }

  def combineMaps[K, V](map1: Map[K, V], map2: Map[K, V], op: (V, V) => V): Map[K, V] =
    ((for ((k, v) <- map2) yield (if (map1.contains(k)) (k, op(v, map1(k))) else (k, v)))
      ++
      (for ((k, v) <- map1 if !map2.contains(k)) yield (k, v))
      ).toMap

  def apply(left: Iterator[String], right: Iterator[String]): Map[String, Int] = {
    var resultCounts = Map[String, Int]()
    for ((leftLine, rightLine) <- (left zip right)) {
      Parser(leftLine, Parser.log) match {
        case Some(leftTree: TreeNode) =>
          Parser(rightLine, Parser.log) match {
            case Some(rightTree: TreeNode) =>
              if (leftTree.compareStructure(rightTree))
                resultCounts = combineMaps[String, Int](resultCounts.toMap, leftTree.getTagCounts().toMap, (a: Int, b: Int) => (a + b))
              else
                log.err("the structure of <<%s>> does not the match strucure of <<%s>>\n".format(leftLine, rightLine))
            case None => ()
          }
        case None => ()
      }
    }
    for (line <- left) {
      log.err("Leftover line <<%s>> in input inputFile\n".format(line))
    }
    for (line <- right) {
      log.err("Leftover line <<%s>> in other inputFile\n".format(line))
    }
    resultCounts
  }

  def apply(list: Iterator[String]): HashMap[String, Int] = {
    val tagCounts = HashMap[String, Int]()

    for (line: String <- list) {
      val tree = Parser(line, Parser.log)
      if (tree.isDefined) {
        for ((key, value) <- tree.get.getTagCounts()) {
          if (tagCounts.contains(key))
            tagCounts(key) += value
          else
            tagCounts(key) = value
        }
      }
    }
    tagCounts
  }

  def checkTokensInLine(aList: List[String], bList: List[String]): String = {
    if (aList.length != bList.length) {
      //log.err("Lists should be the same length: %s %s\n".format(aList, bList))
      "Fail: %s is not the same length as %s".format(aList, bList)
    } else if (aList.length == 0) {
      ""
    } else {
      val a :: as = aList
      val b :: bs = bList
      if (a != b) {
        //log.err("%s does not match %s\n".format(a, b))
        "Fail: \"%s\" does not match \"%s\"".format(a, b)
      } else {
        "" + checkTokensInLine(as, bs)
      }
    }
  }

  def checkTokens(infile: List[String], tokfile: List[String]): List[String] = {
    for (((inTreeLine, tokLine),index) <- (infile zip tokfile).toList.zipWithIndex) yield {
      val inTree = Parser(inTreeLine, Parser.log)
      inTree match {
        case Some(root) =>
          val inTreeTokens: List[String] = root.getTokens
          val tokTokens = tokLine.replace("<EOS>", "").split("\\s+").toList
          checkTokensInLine(inTreeTokens, tokTokens) match {
            case "" => "%d: pass".format(index)
            case x => "%d: %s".format(index, x)
          }
        case _ => "%d: Fail"
      }
    }


  }

  def main(args: Array[String]) {
    val parser = new ArgotParser("org.scalabha.preproc.Tokenizer", preUsage = Some("Version 0.0"))
    val help = parser.flag[Boolean](List("h", "help"), "print help")
    val input = parser.option[String](List("i", "input"), "FILE", "input inputFile in which to check tags")
    val tokens = parser.option[String](List("tok"), "FILE", "tags to check")
    val other = parser.option[String](List("other"), "FILE", "(optional) other inputFile to compare against the input inputFile")


    try {
      parser.parse(args)

      if (help.value.isDefined) {
        parser.usage()
      }

      val input_file =
        (if (input.value.isDefined) {
          scala.io.Source.fromFile(input.value.get, "UTF-8")<commit_msg>fixed tagchecker output<commit_after>package org.scalabha.tree

import org.scalabha.log.SimpleLogger
import java.io.{OutputStreamWriter, BufferedWriter}
import org.clapper.argot.{ArgotUsageException, ArgotParser, ArgotConverters}
import org.scalabha.model.TreeNode
import collection.Set
import collection.mutable.HashMap
import grizzled.string
import io.BufferedSource

object TagChecker {

  import ArgotConverters._

  val parser = new ArgotParser("org.scalabha.preproc.Tokenizer", preUsage = Some("Version 0.0"))
  val help = parser.flag[Boolean](List("h", "help"), "print help")
  val input = parser.option[String](List("i", "input"), "FILE", "input inputFile in which to check tags")
  val other = parser.option[String](List("other"), "FILE", "(optional) other inputFile to compare against the input inputFile")
  val log = new SimpleLogger("org.scalabha.tree.TagChecker", SimpleLogger.WARN, new BufferedWriter(new OutputStreamWriter(System.err)))
  val noLog = new SimpleLogger("org.scalabha.tree.TagChecker", SimpleLogger.NONE, new BufferedWriter(new OutputStreamWriter(System.err)))


  abstract class TagCheckResult

  case class Success() extends TagCheckResult {
    override def toString(): String = {
      "OK"
    }

    val bool = true
  }

  case class Fail[T](left: T, right: T) extends TagCheckResult {
    override def toString(): String = {
      "FAIL: <<%s>>!=<<%s>>".format(left, right)
    }

    val bool = false
  }

  def spprintRepr(map: Map[String, Int], join: String): String = {
    val regex = "[^(]+\\((.*)\\)".r
    val regex(string) = map.toList.sorted.toString
    string.replace(", ", join)
  }

  /**
   * Return true iff both trees have the same tag counts
   */
  def apply(left: TreeNode, right: TreeNode): TagCheckResult = {
    val leftCounts = left.getTagCounts()
    val rightCounts = right.getTagCounts()
    if (leftCounts == rightCounts)
      Success()
    else
      Fail[HashMap[String, Int]](leftCounts, rightCounts)
  }

  def combineMaps[K, V](map1: Map[K, V], map2: Map[K, V], op: (V, V) => V): Map[K, V] =
    ((for ((k, v) <- map2) yield (if (map1.contains(k)) (k, op(v, map1(k))) else (k, v)))
      ++
      (for ((k, v) <- map1 if !map2.contains(k)) yield (k, v))
      ).toMap

  def apply(left: Iterator[String], right: Iterator[String]): Map[String, Int] = {
    var resultCounts = Map[String, Int]()
    for ((leftLine, rightLine) <- (left zip right)) {
      Parser(leftLine, Parser.log) match {
        case Some(leftTree: TreeNode) =>
          Parser(rightLine, Parser.log) match {
            case Some(rightTree: TreeNode) =>
              if (leftTree.compareStructure(rightTree))
                resultCounts = combineMaps[String, Int](resultCounts.toMap, leftTree.getTagCounts().toMap, (a: Int, b: Int) => (a + b))
              else
                log.err("the structure of <<%s>> does not the match strucure of <<%s>>\n".format(leftLine, rightLine))
            case None => ()
          }
        case None => ()
      }
    }
    for (line <- left) {
      log.err("Leftover line <<%s>> in input inputFile\n".format(line))
    }
    for (line <- right) {
      log.err("Leftover line <<%s>> in other inputFile\n".format(line))
    }
    resultCounts
  }

  def apply(list: Iterator[String]): HashMap[String, Int] = {
    val tagCounts = HashMap[String, Int]()

    for (line: String <- list) {
      val tree = Parser(line, Parser.log)
      if (tree.isDefined) {
        for ((key, value) <- tree.get.getTagCounts()) {
          if (tagCounts.contains(key))
            tagCounts(key) += value
          else
            tagCounts(key) = value
        }
      }
    }
    tagCounts
  }

  def checkTokensInLine(aList: List[String], bList: List[String]): String = {
    if (aList.length != bList.length) {
      //log.err("Lists should be the same length: %s %s\n".format(aList, bList))
      "Fail: %s is not the same length as %s".format(aList, bList)
    } else if (aList.length == 0) {
      ""
    } else {
      val a :: as = aList
      val b :: bs = bList
      if (a != b) {
        //log.err("%s does not match %s\n".format(a, b))
        "Fail: \"%s\" does not match \"%s\"".format(a, b)
      } else {
        "" + checkTokensInLine(as, bs)
      }
    }
  }

  def checkTokens(infile: List[String], tokfile: List[String]): List[String] = {
    for (((inTreeLine, tokLine),index) <- (infile zip tokfile).toList.zipWithIndex) yield {
      val inTree = Parser(inTreeLine, Parser.log)
      inTree match {
        case Some(root) =>
          val inTreeTokens: List[String] = root.getTokens
          val tokTokens = tokLine.replace("<EOS>", "").split("\\s+").toList
          checkTokensInLine(inTreeTokens, tokTokens) match {
            case "" => "%d: pass".format(index)
            case x => "%d: %s".format(index, x)
          }
        case _ => "%d: Fail - couldn't parse tree. See parser log messages.".format(index)
      }
    }


  }

  def main(args: Array[String]) {
    val parser = new ArgotParser("org.scalabha.preproc.Tokenizer", preUsage = Some("Version 0.0"))
    val help = parser.flag[Boolean](List("h", "help"), "print help")
    val input = parser.option[String](List("i", "input"), "FILE", "input inputFile in which to check tags")
    val tokens = parser.option[String](List("tok"), "FILE", "tags to check")
    val other = parser.option[String](List("other"), "FILE", "(optional) other inputFile to compare against the input inputFile")


    try {
      parser.parse(args)

      if (help.value.isDefined) {
        parser.usage()
      }

      val input_file =
        (if (input.value.isDefined) {
          scala.io.Source.fromFile(input.value.get, "UTF-8")
{'fraction_non_alphanumeric': 0.15514797093744462, 'fraction_numerical': 0.002303739145844409, 'mean_word_length': 2.866735183281946, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5101276', 'n_tokens_mistral': 3765, 'n_tokens_neox': 3695, 'n_words': 938}