<commit_before>package com.github.jmccrae.yuzu

import com.github.jmccrae.yuzu.YuzuSettings._
import com.hp.hpl.jena.rdf.model.{Literal, Model, RDFNode, Resource, Property}
import com.hp.hpl.jena.vocabulary._
import java.io.Writer
import scala.collection.JavaConversions._
import scala.collection.mutable.{Map => MutableMap}

object JsonLDPrettySerializer {
  sealed trait JsonObj {
    def write(out : Writer, indent : Int) : Unit
    override def toString = {
      val sw = new java.io.StringWriter()
      write(sw, 0)
      sw.toString }
  }

  object JsonObj {
    def apply(value : Any) : JsonObj = value match {
      case o : JsonObj => 
        o
      case s : String =>
        JsonString(s)
      case i : Int =>
        JsonInt(i)
      case d : Double =>
        JsonFloat(d)
      case s : Seq[_] =>
        JsonList(s.map(JsonObj(_)))
      case m : Map[_, _] =>
        JsonMap(MutableMap((m.map {
          case (k, v) => k.toString -> JsonObj(v)
        }).toSeq:_*))
      case m : MutableMap[_, _] =>
        JsonMap(m.map {
          case (k, v) => k.toString -> JsonObj(v)
        })
    }
    def apply(vals : (String, Any)*) : JsonMap = JsonMap((vals.map {
      case (k, v) => k -> JsonObj(v)
    }):_*)
  }
  case class JsonString(value : String) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("\"%s\"" format (value.replaceAll("\\\\","\\\\\\\\").
        replaceAll("\"", "\\\\\""))) }
  }

  case class JsonInt(value : Int) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write(value.toString) }
  }

  case class JsonFloat(value : Double) extends JsonObj {
    def write(out : Writer, index : Int) {
      out.write(value.toString) }
  }

  class JsonMap(val value : MutableMap[String, JsonObj]) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("{\n")
      val elems = value.toSeq.sortBy(_._1)
      if(!elems.isEmpty) {
        val h = elems.head
        out.write("  " * (indent + 1))
        out.write("\"%s\": " format h._1)
        h._2.write(out, indent + 2)
        for((k, v) <- elems.tail) {
          out.write(",\n")
          out.write("  " * (indent + 1))
          out.write("\"%s\": " format k)
          v.write(out, indent + 2) }}
      out.write("\n")
      out.write("  " * indent)
      out.write("}") }

    def update(key : String, v : JsonObj) = value.update(key, v)
    def update(key : String, v : String) = value.update(key, JsonString(v))
    def contains(key : String) = value.contains(key)
    def apply(key : String) = value(key)
    def remove(key : String) = value.remove(key)
    def keys = value.keys
    override def equals(o : Any) = o match {
      case null => 
        false
      case jm : JsonMap =>
        value == jm.value
      case _ =>
        false }
  }

  object JsonMap {
    def apply(value : MutableMap[String, JsonObj]) = new JsonMap(value)
    def apply(vals : (String, JsonObj)*) = new JsonMap(
      MutableMap(vals:_*))
  }

  case class JsonList(value : Seq[JsonObj]) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("[\n")
      if(!value.isEmpty) {
        val h = value.head
        out.write("  " * (indent + 1))
        h.write(out, indent + 1)
        for(v <- value.tail) {
          out.write(",\n")
          out.write("  " * (indent + 1))
          v.write(out, indent + 1) }}
      out.write("\n")
      out.write("  " * indent)
      out.write("]") }

    def :+(elem : JsonObj) = JsonList(value :+ elem)
    def +:(elem : JsonObj) = JsonList(elem +: value)
    def apply(i : Int) = value(i)
  }

  implicit class GraphPimp(graph : Model) {
    def listProperties(subj : Resource = null, obj : RDFNode = null) = {
      val stats = graph.listStatements().toSeq
      (stats.filter { st =>
        (subj == null || st.getSubject() == subj) &&
        (obj == null || st.getObject() == obj) 
      } map { st =>
        st.getPredicate()
      }).toSet }
  }

  private def isAlnum(s : String) = s.matches("\\w+")

  private def propType(p :  Property, graph : Model) : JsonObj = {
    if(graph.listObjectsOfProperty(p).forall(_.isResource())) {
      JsonObj("@id" -> p.getURI(),
              "@type" -> "@id") }
    else {
      JsonString(p.getURI()) }}

  private def splitURI(value : String, graph : Model) : (Option[String], String) = {
    if(value.startsWith(BASE_NAME)) {
      return (None, value.drop(BASE_NAME.length)) }
    for((qn, uri) <- graph.getNsPrefixMap()) {
      if(value.startsWith(uri)) {
        return (Some(qn), value.drop(uri.length)) }}
    return (None, value) }

  private def isPrefixUsed(prefix : String, graph : Model) : Boolean = {
    graph.listStatements().exists { stat =>
      stat.getPredicate().getURI().startsWith(prefix) ||
      (stat.getSubject().isURIResource() &&
       stat.getSubject().asResource().getURI().startsWith(prefix)) ||<commit_msg>Maybe it is not infinite<commit_after>package com.github.jmccrae.yuzu

import com.github.jmccrae.yuzu.YuzuSettings._
import com.hp.hpl.jena.rdf.model.{Literal, Model, RDFNode, Resource, Property}
import com.hp.hpl.jena.vocabulary._
import java.io.Writer
import scala.collection.JavaConversions._
import scala.collection.mutable.{Map => MutableMap}

object JsonLDPrettySerializer {
  sealed trait JsonObj {
    def write(out : Writer, indent : Int) : Unit
    override def toString = {
      val sw = new java.io.StringWriter()
      write(sw, 0)
      sw.toString }
  }

  object JsonObj {
    def apply(value : Any) : JsonObj = value match {
      case o : JsonObj => 
        o
      case s : String =>
        JsonString(s)
      case i : Int =>
        JsonInt(i)
      case d : Double =>
        JsonFloat(d)
      case s : Seq[_] =>
        JsonList(s.map(JsonObj(_)))
      case m : Map[_, _] =>
        JsonMap(MutableMap((m.map {
          case (k, v) => k.toString -> JsonObj(v)
        }).toSeq:_*))
      case m : MutableMap[_, _] =>
        JsonMap(m.map {
          case (k, v) => k.toString -> JsonObj(v)
        })
    }
    def apply(vals : (String, Any)*) : JsonMap = JsonMap((vals.map {
      case (k, v) => k -> JsonObj(v)
    }):_*)
  }
  case class JsonString(value : String) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("\"%s\"" format (value.replaceAll("\\\\","\\\\\\\\").
        replaceAll("\"", "\\\\\""))) }
  }

  case class JsonInt(value : Int) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write(value.toString) }
  }

  case class JsonFloat(value : Double) extends JsonObj {
    def write(out : Writer, index : Int) {
      out.write(value.toString) }
  }

  class JsonMap(val value : MutableMap[String, JsonObj]) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("{\n")
      val elems = value.toSeq.sortBy(_._1)
      if(!elems.isEmpty) {
        val h = elems.head
        out.write("  " * (indent + 1))
        out.write("\"%s\": " format h._1)
        h._2.write(out, indent + 2)
        for((k, v) <- elems.tail) {
          out.write(",\n")
          out.write("  " * (indent + 1))
          out.write("\"%s\": " format k)
          v.write(out, indent + 2) }}
      out.write("\n")
      out.write("  " * indent)
      out.write("}") }

    def update(key : String, v : JsonObj) = value.update(key, v)
    def update(key : String, v : String) = value.update(key, JsonString(v))
    def contains(key : String) = value.contains(key)
    def apply(key : String) = value(key)
    def remove(key : String) = value.remove(key)
    def keys = value.keys
    override def equals(o : Any) = o match {
      case null => 
        false
      case jm : JsonMap =>
        value == jm.value
      case _ =>
        false }
  }

  object JsonMap {
    def apply(value : MutableMap[String, JsonObj]) = new JsonMap(value)
    def apply(vals : (String, JsonObj)*) = new JsonMap(
      MutableMap(vals:_*))
  }

  case class JsonList(value : Seq[JsonObj]) extends JsonObj {
    def write(out : Writer, indent : Int) {
      out.write("[\n")
      if(!value.isEmpty) {
        val h = value.head
        out.write("  " * (indent + 1))
        h.write(out, indent + 1)
        for(v <- value.tail) {
          out.write(",\n")
          out.write("  " * (indent + 1))
          v.write(out, indent + 1) }}
      out.write("\n")
      out.write("  " * indent)
      out.write("]") }

    def :+(elem : JsonObj) = JsonList(value :+ elem)
    def +:(elem : JsonObj) = JsonList(elem +: value)
    def apply(i : Int) = value(i)
  }

  implicit class GraphPimp(graph : Model) {
    def listProperties(subj : Resource = null, obj : RDFNode = null) = {
      (graph.listStatements(subj, Option(obj).map(x => graph.createProperty(x.asResource().getURI())).getOrElse(null), null).map { st =>
//      (stats.filter { st =>
//        (subj == null || st.getSubject() == subj) &&
//        (obj == null || st.getObject() == obj) 
//      } map { st =>
        st.getPredicate()
      }).toSet }
  }

  private def isAlnum(s : String) = s.matches("\\w+")

  private def propType(p :  Property, graph : Model) : JsonObj = {
    if(graph.listObjectsOfProperty(p).forall(_.isResource())) {
      JsonObj("@id" -> p.getURI(),
              "@type" -> "@id") }
    else {
      JsonString(p.getURI()) }}

  private def splitURI(value : String, graph : Model) : (Option[String], String) = {
    if(value.startsWith(BASE_NAME)) {
      return (None, value.drop(BASE_NAME.length)) }
    for((qn, uri) <- graph.getNsPrefixMap()) {
      if(value.startsWith(uri)) {
        return (Some(qn), value.drop(uri.length)) }}
    return (None, value) }

  private def isPrefixUsed(prefix : String, graph : Model) : Boolean = {
    graph.listStatements().exists { stat =>
      stat.getPredicate().getURI().startsWith(prefix) ||
      (stat.getSubject().isURIResource() &&
       stat.getSubject().asResource().getURI().startsWith(prefix)) ||
{'fraction_non_alphanumeric': 0.1682280648735771, 'fraction_numerical': 0.0024176488365064974, 'mean_word_length': 2.333781061114842, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2350733', 'n_tokens_mistral': 3399, 'n_tokens_neox': 3342, 'n_words': 752}