<commit_before>package jigg.util

import java.io._

import scala.xml._

import scala.io.Source
import scala.collection.mutable.{StringBuilder, ArrayBuffer}

import org.json4s._
import org.json4s.DefaultFormats
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._

object JSONUtil {

  object XMLParser {

    def hasChild(x: Any): Boolean = x match {
      case x: Elem => if (x.child.length > 0) true else false
      case x: List[_] if x forall (_.isInstanceOf[Node]) =>
        x forall (hasChild(_))
      case _ => false
    }

    def getChildNode(x: Any): List[Node] = x match {
      case x: Elem => (x.child filter (_.label != "#PCDATA")).toList
      case x: List[_] if x forall (_.isInstanceOf[Node]) =>
        (x.map(getChildNode(_))).flatten
      case _ => Nil
    }

    def isTextNode(x: Node): Boolean = !XMLUtil.text(x).trim.isEmpty

    def getAttributionList(x: Node): Seq[(String, String)] = (
      for{
        elem <- x.attributes.seq
        n = elem.key -> elem.value.toString
      } yield n
    ).toList

    def hasAttribution(x: Node): Boolean = !x.attributes.isEmpty
  }

  def readJSON(file: String): String = toJSONFromNode(XML.loadFile(file))

  def toJSON(x: Node): String = toJSONFromNode(x)

  private def toJSONFromNode(node: Node): String = {
    val xml = XML.loadString(node.toString.split("\n").mkString)
    val sb = new StringBuilder

    sb.append('{')
    sb.append(List("\".tag\":\"",node.label,"\",\n").mkString)
    sb.append("\".child\":")
    sb.append("[\n")
    
    sb.append(serializing(xml))

    sb.append("]\n")
    sb.append("}\n")

    pretty(render(parse(sb.toString)))
  }

  private def serializing[T <: Node](x: T): StringBuilder = {
    val subsb = new StringBuilder
    if(XMLParser.hasChild(x)){
      val childNode = XMLParser.getChildNode(x)
      var prefix = ""
      for (i <- childNode){
        val retsb = serializing(i)
        subsb.append(prefix)
        prefix = ",\n"
        subsb.append(List("{\".tag\":\"", i.label, "\",\n").mkString)
        var prefix2 = ""
        if(XMLParser.isTextNode(i)){
          subsb.append(prefix2)
          val text = new StringBuilder<commit_msg>[update] Delete wasted line from JSONUtil.toJSONFromNode<commit_after>package jigg.util

import java.io._

import scala.xml._

import scala.io.Source
import scala.collection.mutable.{StringBuilder, ArrayBuffer}

import org.json4s._
import org.json4s.DefaultFormats
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._

object JSONUtil {

  object XMLParser {

    def hasChild(x: Any): Boolean = x match {
      case x: Elem => if (x.child.length > 0) true else false
      case x: List[_] if x forall (_.isInstanceOf[Node]) =>
        x forall (hasChild(_))
      case _ => false
    }

    def getChildNode(x: Any): List[Node] = x match {
      case x: Elem => (x.child filter (_.label != "#PCDATA")).toList
      case x: List[_] if x forall (_.isInstanceOf[Node]) =>
        (x.map(getChildNode(_))).flatten
      case _ => Nil
    }

    def isTextNode(x: Node): Boolean = !XMLUtil.text(x).trim.isEmpty

    def getAttributionList(x: Node): Seq[(String, String)] = (
      for{
        elem <- x.attributes.seq
        n = elem.key -> elem.value.toString
      } yield n
    ).toList

    def hasAttribution(x: Node): Boolean = !x.attributes.isEmpty
  }

  def readJSON(file: String): String = toJSONFromNode(XML.loadFile(file))

  def toJSON(x: Node): String = toJSONFromNode(x)

  private def toJSONFromNode(node: Node): String = {
    val sb = new StringBuilder

    sb.append('{')
    sb.append(List("\".tag\":\"",node.label,"\",\n").mkString)
    sb.append("\".child\":")
    sb.append("[\n")
    
    sb.append(serializing(node))

    sb.append("]\n")
    sb.append("}\n")

    pretty(render(parse(sb.toString)))
  }

  private def serializing[T <: Node](x: T): StringBuilder = {
    val subsb = new StringBuilder
    if(XMLParser.hasChild(x)){
      val childNode = XMLParser.getChildNode(x)
      var prefix = ""
      for (i <- childNode){
        val retsb = serializing(i)
        subsb.append(prefix)
        prefix = ",\n"
        subsb.append(List("{\".tag\":\"", i.label, "\",\n").mkString)
        var prefix2 = ""
        if(XMLParser.isTextNode(i)){
          subsb.append(prefix2)
          val text = new StringBuilder
{'fraction_non_alphanumeric': 0.15478662053056516, 'fraction_numerical': 0.003229527104959631, 'mean_word_length': 2.763888888888889, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2386992', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1435, 'n_words': 327}