<commit_before>/**
 *   ____    __    ____  ____  ____,,___     ____  __  __  ____
 *  (  _ \  /__\  (_   )(_  _)( ___)/ __)   (  _ \(  )(  )(  _ \           Read
 *   )   / /(__)\  / /_  _)(_  )__) \__ \    )___/ )(__)(  ) _ <     README.txt
 *  (_)\_)(__)(__)(____)(____)(____)(___/   (__)  (______)(____/    LICENSE.txt
 */
package razie.wiki.util

import org.json.{JSONArray, JSONObject}

import scala.collection.mutable.{HashMap, ListBuffer}

/**
 * json helpers
 *
 *  a json is represented as maps of (name,value) and lists of values, either of which can be recursive, representing
 *  a json object via a map and an array as a list.
 *
 *  we do not have a common Node class - but simple Map and List
 */
object js {

  /** turn a map of name,value into json */
  def tojson(x: Map[_, _]): JSONObject = {
    val o = new JSONObject()
    x foreach {t:(_,_) =>
      t._2 match {
        case m: Map[_, _] => o.put(t._1.toString, tojson(m))
        case s: String => o.put(t._1.toString, s)
        case l: List[_] => o.put(t._1.toString, tojson(l))
        case h @ _ => o.put(t._1.toString, h.toString)
      }
    }
    o
  }

  /** turn a list into json */
  def tojson(x: List[_]): JSONArray = {
    val o = new JSONArray()
    x.foreach { t:Any =>
      t match {
        case s: Map[_, _] => o.put(tojson(s))
        case l: List[_] => o.put(tojson(l))
        case s: String => o.put(s)
        case s: JSONObject => o.put(s)
      }
    }
    o
  }

  /** recursively transform a name,value map
    *
    * the transformation is f(path, name, value)
    */
  def jt(map: Map[_, _], path: String = "/")(f: PartialFunction[(String, String, Any), (String, Any)]): Map[String, Any] = {
    val o = new HashMap[String, Any]()
    map.foreach { t:(_,_) =>
      val ts = t._1.toString
      val r = if (f.isDefinedAt(path, ts, t._2)) f(path, ts, t._2) else (ts, t._2)
      if (r._1 != null && r._1.length() > 0)
        r._2 match {
          case s: Map[_, _] => o put (r._1.toString, jt(s, path + "/" + ts)(f))
          case l: List[_] => o put (r._1.toString, jt(l, path + "/" + ts)(f))
          case s @ _ => o put (r._1.toString, s)
        }
    }
    o.toMap
  }

  /** recursively transform a name,value map
    *
    * the transformation is f(path, name, value)
    */
  def jt(x: List[_])(f: PartialFunction[(String, String, Any), (String, Any)]): List[_] = jt(x, "/")(f)

  /** recursively transform a name,value map
    *
    * the transformation is f(path, name, value)
    */
  def jt(x: List[_], path: String)(f: PartialFunction[(String, String, Any), (String, Any)]): List[_] = {
    val o = new ListBuffer[Any]()
    x.foreach { t:Any =>
      t match {
        case m: Map[_, _] => o.append(jt(m, path)(f))
        case l: List[_] => o.append(jt(l, path)(f))
      }
    }
    o.toList
  }

  val q = "\""

  private def q(str:String) = "\""+str+"\""

  /** turn a map of name,value into json */<commit_msg>clean up a little more<commit_after>/**
 *   ____    __    ____  ____  ____,,___     ____  __  __  ____
 *  (  _ \  /__\  (_   )(_  _)( ___)/ __)   (  _ \(  )(  )(  _ \           Read
 *   )   / /(__)\  / /_  _)(_  )__) \__ \    )___/ )(__)(  ) _ <     README.txt
 *  (_)\_)(__)(__)(____)(____)(____)(___/   (__)  (______)(____/    LICENSE.txt
 */
package razie.wiki.util

import org.json.{JSONArray, JSONObject}

import scala.collection.mutable.{HashMap, ListBuffer}

/**
 * json helpers
 *
 *  a json is represented as maps of (name,value) and lists of values, either of which can be recursive, representing
 *  a json object via a map and an array as a list.
 *
 *  we do not have a common Node class - but simple Map and List
 */
object js {

  /** turn a map of name,value into json */
  def tojson(x: Map[_, _]): JSONObject = {
    val o = new JSONObject()
    x foreach {t:(_,_) =>
      t._2 match {
        case m: Map[_, _] => o.put(t._1.toString, tojson(m))
        case s: String => o.put(t._1.toString, s)
        case l: List[_] => o.put(t._1.toString, tojson(l))
        case h @ _ => o.put(t._1.toString, h.toString)
      }
    }
    o
  }

  /** turn a list into json */
  def tojson(x: List[_]): JSONArray = {
    val o = new JSONArray()
    x.foreach { t:Any =>
      t match {
        case s: Map[_, _] => o.put(tojson(s))
        case l: List[_] => o.put(tojson(l))
        case s: String => o.put(s)
        case s: JSONObject => o.put(s)
      }
    }
    o
  }

  /** recursively transform a name,value map
    *
    * the transformation is f(path, name, value) => (name, value)
    */
  def jt(map: Map[_, _], path: String = "/")(f: PartialFunction[(String, String, Any), (String, Any)]): Map[String, Any] = {
    val o = new HashMap[String, Any]()
    map.foreach { t:(_,_) =>
      val ts = t._1.toString
      val r = if (f.isDefinedAt(path, ts, t._2)) f(path, ts, t._2) else (ts, t._2)
      if (r._1 != null && r._1.length() > 0)
        r._2 match {
          case s: Map[_, _] => o put (r._1.toString, jt(s, path + "/" + ts)(f))
          case l: List[_] => o put (r._1.toString, jt(l, path + "/" + ts)(f))
          case s @ _ => o put (r._1.toString, s)
        }
    }
    o.toMap
  }

  /** recursively transform a name,value map
    *
    * the transformation is f(path, name, value) => (name, value)
    */
  def jt(x: List[_], path: String)(f: PartialFunction[(String, String, Any), (String, Any)]): List[_] = {
    val o = new ListBuffer[Any]()
    x.foreach { t:Any =>
      t match {
        case m: Map[_, _] => o.append(jt(m, path)(f))
        case l: List[_] => o.append(jt(l, path)(f))
      }
    }
    o.toList
  }

  /** @see jt */
  def jt(x: List[_])(f: PartialFunction[(String, String, Any), (String, Any)]): List[_] = jt(x, "/")(f)


  val q = "\""

  private def q(str:String) = "\""+str+"\""

  /** turn a map of name,value into json */
{'fraction_non_alphanumeric': 0.18490566037735848, 'fraction_numerical': 0.005488850771869639, 'mean_word_length': 2.153596538669551, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3392201', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2163, 'n_words': 575}