<commit_before>package com.holdenkarau.spark.testing

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.scalacheck.Gen
import org.scalacheck.Prop._
import org.scalatest.FunSuite
import org.scalatest.prop.Checkers

class PrettifyTest extends FunSuite with SharedSparkContext with Checkers with Prettify {

  test("pretty output of Dataframe's check") {
    implicit val generatorDrivenConfig =
      PropertyCheckConfig(minSize = 2, maxSize = 2)
    val schema = StructType(List(StructField("name", StringType), StructField("age", IntegerType)))
    val sqlContext = new SQLContext(sc)
    val nameGenerator = new ColumnGenerator("name", Gen.const("<NAME>"))
    val ageGenerator = new ColumnGenerator("age", Gen.const(20))

    val dataframeGen = DataframeGenerator.arbitraryDataFrameWithCustomFields(sqlContext, schema)(nameGenerator, ageGenerator)

    val property =
      forAll(dataframeGen.arbitrary) {
        dataframe => false
      }
    val e = intercept[org.scalatest.exceptions.GeneratorDrivenPropertyCheckFailedException] {
      check(property)
    }
    val expected =
      Some("arg0 = <DataFrame: schema = [name: string, age: int], size = 2, values = ([Holden Hanafy,20], [Holden Hanafy,20])>")
    assert(takeSecondToLastLine(e.message) == expected)
  }

  private def takeSecondToLastLine(msg: Option[String]) =
    msg.flatMap(_.split("\n").toList.reverse.tail.headOption.map(_.trim))

}<commit_msg>Make output of check pretty for generated RDD.<commit_after>package com.holdenkarau.spark.testing

import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.scalacheck.{Arbitrary, Gen}
import org.scalacheck.Prop._
import org.scalatest.FunSuite
import org.scalatest.prop.Checkers

class PrettifyTest extends FunSuite with SharedSparkContext with Checkers with Prettify {

  test("pretty output of Dataframe's check") {
    implicit val generatorDrivenConfig =
      PropertyCheckConfig(minSize = 2, maxSize = 2)
    val schema = StructType(List(StructField("name", StringType), StructField("age", IntegerType)))
    val sqlContext = new SQLContext(sc)
    val nameGenerator = new ColumnGenerator("name", Gen.const("<NAME>"))
    val ageGenerator = new ColumnGenerator("age", Gen.const(20))

    val dataframeGen = DataframeGenerator.arbitraryDataFrameWithCustomFields(sqlContext, schema)(nameGenerator, ageGenerator)

    val property =
      forAll(dataframeGen.arbitrary) {
        dataframe => false
      }
    val e = intercept[org.scalatest.exceptions.GeneratorDrivenPropertyCheckFailedException] {
      check(property)
    }
    val expected =
      Some("arg0 = <DataFrame: schema = [name: string, age: int], size = 2, values = ([Holden Hanafy,20], [Holden Hanafy,20])>")
    assert(takeSecondToLastLine(e.message) == expected)
  }

  test("pretty output of RDD's check") {
    implicit val generatorDrivenConfig =
      PropertyCheckConfig(minSize = 2, maxSize = 2)
    val property =
      forAll(RDDGenerator.genRDD[String](sc)(Arbitrary.arbitrary[String])) {
        rdd => {
          println(rdd.take(10).mkString(","))
          println(rdd.getNumPartitions)
          false
        }
      }
    val e = intercept[org.scalatest.exceptions.GeneratorDrivenPropertyCheckFailedException] {
      check(property)
    }
    val expected =
      Some("arg0 = <DataFrame: schema = [name: string, age: int], size = 2, values = ([Holden Hanafy,20], [Holden Hanafy,20])>")
    assert(takeSecondToLastLine(e.message) == expected)
  }

  private def takeSecondToLastLine(msg: Option[String]) =
    msg.flatMap(_.split("\n").toList.reverse.tail.headOption.map(_.trim))

}
{'fraction_non_alphanumeric': 0.11581469648562301, 'fraction_numerical': 0.007987220447284345, 'mean_word_length': 4.344238975817923, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6889810', 'n_tokens_mistral': 1181, 'n_tokens_neox': 1132, 'n_words': 241}