<commit_before>package org.continuumio.bokeh

import play.api.libs.json.{Json,JsValue,JsArray,JsObject,JsNull}

trait Serializer {
    case class Model(id: String, `type`: String, attributes: Map[String, Any], doc: Option[String]) {
        def toJson: JsValue =
            Json.obj("id" -> Json.toJson(id),
                     "type" -> Json.toJson(`type`),
                     "attributes" -> anyToJson(attributes),
                     "doc" -> Json.toJson(doc))
    }

    def anyToJson(obj: Any): JsValue = {
        import Json.toJson
        import Formats._
        obj match {
            case obj: Boolean => toJson(obj)
            case obj: Int => toJson(obj)
            case obj: Double => toJson(obj)
            case obj: String => toJson(obj)
            case obj: Symbol => toJson(obj)
            case obj: macros.EnumType => toJson(obj)
            case obj: CSSColor => toJson(obj)
            case obj: Percent => toJson(obj)
            case obj: breeze.linalg.DenseVector[Double] => toJson(obj)
            case obj: Seq[_] => JsArray(obj.map(anyToJson))
            case obj: Array[_] => JsArray(obj.map(anyToJson))
            case obj: Map[_, _] =>
                JsObject(obj.toList.map {
                    case (key: String, value) => (key,      anyToJson(value))
                    case (key: Symbol, value) => (key.name, anyToJson(value))
                    case _ => throw new IllegalArgumentException(obj.toString)
                })
            case Ref(id, tp) => toJson(Map("id" -> id, "type" -> tp))
            case Some(obj) => anyToJson(obj)
            case None => JsNull
            case obj => throw new IllegalArgumentException(obj.toString)
        }
    }

    def allFieldsWithValues(obj: PlotObject): List[(String, Any)] =
        ("type", obj.viewModel) :: obj.dirtyFieldsWithValues

    def serializeObjs(objs: List[PlotObject]): String = {
        val models = objs.map(getModel).map(_.toJson)
        Json.stringify(JsArray(models))
    }

    def collectObjs(obj: PlotObject): List[PlotObject] = {
        val ids = collection.mutable.HashSet[String]()
        val objs = collection.mutable.ListBuffer[PlotObject]()

        def descend(obj: Any) {
            obj match {
                case obj: PlotObject if !ids.contains(obj.id.value) =>
                    ids += obj.id.value
                    allFieldsWithValues(obj).foreach {
                        case (_, Some(obj)) => descend(obj)
                        case _ =>
                    }
                    obj match {
                        case _: NoRefs =>
                        case _ => objs += obj
                    }
                case obj: List[_] => obj.map(descend)
                case _ =>
            }
        }

        descend(obj)
        objs.toList
    }

    def getModel(obj: PlotObject): Model = {
        val Ref(id, tp) = obj.getRef
        Model(id, tp, replaceWithRefs(allFieldsWithValues(obj)), None)
    }

    def replaceWithRefs(fields: List[(String, Any)]): Map[String, Any] = {
        fields.map {
            case (name, Some(value)) => (name, _replaceWithRefs(value))
            case field => field
        } toMap
    }
<commit_msg>Generalize traversal function in Serializer<commit_after>package org.continuumio.bokeh

import play.api.libs.json.{Json,JsValue,JsArray,JsObject,JsNull}

trait Serializer {
    case class Model(id: String, `type`: String, attributes: Map[String, Any], doc: Option[String]) {
        def toJson: JsValue =
            Json.obj("id" -> Json.toJson(id),
                     "type" -> Json.toJson(`type`),
                     "attributes" -> anyToJson(attributes),
                     "doc" -> Json.toJson(doc))
    }

    def anyToJson(obj: Any): JsValue = {
        import Json.toJson
        import Formats._
        obj match {
            case obj: Boolean => toJson(obj)
            case obj: Int => toJson(obj)
            case obj: Double => toJson(obj)
            case obj: String => toJson(obj)
            case obj: Symbol => toJson(obj)
            case obj: macros.EnumType => toJson(obj)
            case obj: CSSColor => toJson(obj)
            case obj: Percent => toJson(obj)
            case obj: breeze.linalg.DenseVector[Double] => toJson(obj)
            case obj: Seq[_] => JsArray(obj.map(anyToJson))
            case obj: Array[_] => JsArray(obj.map(anyToJson))
            case obj: Map[_, _] =>
                JsObject(obj.toList.map {
                    case (key: String, value) => (key,      anyToJson(value))
                    case (key: Symbol, value) => (key.name, anyToJson(value))
                    case _ => throw new IllegalArgumentException(obj.toString)
                })
            case Ref(id, tp) => toJson(Map("id" -> id, "type" -> tp))
            case Some(obj) => anyToJson(obj)
            case None => JsNull
            case obj => throw new IllegalArgumentException(obj.toString)
        }
    }

    def allFieldsWithValues(obj: PlotObject): List[(String, Any)] =
        ("type", obj.viewModel) :: obj.dirtyFieldsWithValues

    def serializeObjs(objs: List[PlotObject]): String = {
        val models = objs.map(getModel).map(_.toJson)
        Json.stringify(JsArray(models))
    }

    def collectObjs(obj: PlotObject): List[PlotObject] = {
        val objs = collection.mutable.ListBuffer[PlotObject]()

        traverse(obj, obj => obj match {
            case _: NoRefs =>
            case _ => objs += obj
        })

        objs.toList
    }

    def traverse(obj: PlotObject, fn: PlotObject => Unit) {
        val ids = collection.mutable.HashSet[String]()

        def descend(obj: Any) {
            obj match {
                case obj: PlotObject if !ids.contains(obj.id.value) =>
                    ids += obj.id.value
                    allFieldsWithValues(obj).foreach {
                        case (_, Some(obj)) => descend(obj)
                        case _ =>
                    }
                    fn(obj)
                case obj: List[_] => obj.map(descend)
                case _ =>
            }
        }

        descend(obj)
    }

    def getModel(obj: PlotObject): Model = {
        val Ref(id, tp) = obj.getRef
        Model(id, tp, replaceWithRefs(allFieldsWithValues(obj)), None)
    }

    def replaceWithRefs(fields: List[(String, Any)]): Map[String, Any] = {
        fields.map {
            case (name, Some(value)) => (name, _replaceWithRefs(value))
            case field => field
        } toMap
    }

{'fraction_non_alphanumeric': 0.12467372946414862, 'fraction_numerical': 0.0, 'mean_word_length': 1.7198329853862213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4309680', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1869, 'n_words': 423}