<commit_before>package org.interestinglab.waterdrop.filter

import scala.collection.JavaConversions._
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.sql.functions._
import org.json4s.DefaultFormats
import org.json4s.jackson.JsonMethods

class Json(var conf: Config) extends BaseFilter(conf) {

  def this() = {
    this(ConfigFactory.empty())
  }

  override def checkConfig(): (Boolean, String) = {
    conf.hasPath("table_name") && conf.hasPath("sql") match {
      case true => (true, "")
      case false => (false, "please specify [table_name] and [sql] as a non-empty string")
    }
  }

  override def prepare(spark: SparkSession, ssc: StreamingContext): Unit = {
    super.prepare(spark, ssc)

    val defaultConfig = ConfigFactory.parseMap(
      Map(
        "source_field" -> "raw_message",
        "target_field" -> Json.ROOT)
    )
    conf = conf.withFallback(defaultConfig)
  }

  override def process(spark: SparkSession, df: DataFrame): DataFrame = {

    val srcField = conf.getString("source_field")

    conf.getString("target_field") match {
      case Json.ROOT => df
      case targetField: String => {
        val func = udf((s: String) => {
          implicit val formats = DefaultFormats
          JsonMethods.parse(s).extract[Map[String, String]]
        })

        df.withColumn(targetField, func(col(srcField)))
      }
    }
  }
}

object Json {
  val ROOT = "__root__"
}<commit_msg>add jsonParse exception<commit_after>package org.interestinglab.waterdrop.filter

import scala.collection.JavaConversions._
import com.typesafe.config.{Config, ConfigFactory}
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{StructField, StructType}
import org.json4s.DefaultFormats
import org.json4s.jackson.JsonMethods

import scala.util.control.NonFatal

class Json(var conf: Config) extends BaseFilter(conf) {

  def this() = {
    this(ConfigFactory.empty())
  }

  override def checkConfig(): (Boolean, String) = {
    conf.hasPath("source_field") match {
      case true => (true, "")
      case false => (false, "please specify [source_field] as a non-empty string")
    }
  }

  override def prepare(spark: SparkSession, ssc: StreamingContext): Unit = {
    super.prepare(spark, ssc)

    val defaultConfig = ConfigFactory.parseMap(
      Map(
        "source_field" -> "raw_message",
        "target_field" -> Json.ROOT)
    )
    conf = conf.withFallback(defaultConfig)
  }

  override def process(spark: SparkSession, df: DataFrame): DataFrame = {

    val srcField = conf.getString("source_field")

    conf.getString("target_field") match {
      case Json.ROOT => df  // TODO
      case targetField: String => {
        val func = udf((s: String) => {
          implicit val formats = DefaultFormats
          try {
            JsonMethods.parse(s).extract[Map[String, String]]
          } catch {
            // TODO How to set null for Row
            case NonFatal(e) => Map("null" -> "yes")
          }
        })

        df.withColumn(targetField, func(col(srcField)))
      }
    }
  }
}

object Json {
  val ROOT = "__root__"
}
{'fraction_non_alphanumeric': 0.12274909963985595, 'fraction_numerical': 0.0012004801920768306, 'mean_word_length': 3.1714643304130163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1692212', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1003, 'n_words': 227}