<commit_before>package org.apache.datacommons.prepbuddy.cluster

import org.apache.datacommons.prepbuddy.SparkTestCase
import org.apache.datacommons.prepbuddy.clusterers.TextFacets
import org.apache.datacommons.prepbuddy.rdds.TransformableRDD
import org.apache.spark.api.java.JavaRDD
import org.junit.Assert.assertTrue

class FacetTests extends SparkTestCase {

    test("should drop the specified column from the given rdd") {
        val initialDataset: JavaRDD[String] = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val expected = ("A", 3)
        assert(textFaceted.highest.size == 1)
        val highest = textFaceted.highest.head

        assert(expected == highest)
    }

    test("highest should give one pair in List if only one lowest pair is found") {
        val initialDataset: JavaRDD[String] = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "X,P"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val listOfHighest = textFaceted.highest

        assert(2 == listOfHighest.size)
        assert(2 == textFaceted.count)

        assertTrue(listOfHighest.contains(("A", 3)))
        assertTrue(listOfHighest.contains(("X", 3)))
    }


    test("Textfacet  lowest should give list of lowest pairs if more than one pair found") {
        val initialDataset = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "Q,E", "Q,R"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val listOfLowest = textFaceted.lowest
        assert(3 == textFaceted.count)
        assert(2 == listOfLowest.size)
        assertTrue(listOfLowest.contains(("X", 2)))
        assertTrue(listOfLowest.contains(("Q", 2)))
    }

    test(" TextFacet_ get Facets Between Should Give List Of Faceted Pair In Given Range ") {
        val initialDataset = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "Q,E", "Q,R", "W,E"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val facetedPair = textFaceted.getFacetsBetween(2, 3)
        assert(4 == textFaceted.count)

        assert(3 == facetedPair.size)
        assertTrue(facetedPair.contains(("X", 2)))
        assertTrue(facetedPair.contains(("Q", 2)))
        assertTrue(facetedPair.contains(("A", 3)))
    }

}<commit_msg>Adding a  method for the cardinal values.<commit_after>package org.apache.datacommons.prepbuddy.cluster

import org.apache.datacommons.prepbuddy.SparkTestCase
import org.apache.datacommons.prepbuddy.clusterers.TextFacets
import org.apache.datacommons.prepbuddy.rdds.TransformableRDD
import org.apache.spark.api.java.JavaRDD
import org.junit.Assert.assertTrue

class FacetTests extends SparkTestCase {

    test("should drop the specified column from the given rdd") {
        val initialDataset: JavaRDD[String] = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val expected = ("A", 3)
        assert(textFaceted.highest.length == 1)
        val highest = textFaceted.highest.head

        assert(expected == highest)
    }

    test("highest should give one pair in List if only one lowest pair is found") {
        val initialDataset: JavaRDD[String] = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "X,P"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val listOfHighest = textFaceted.highest

        assert(2 == listOfHighest.length)
        assert(2 == textFaceted.count)

        assertTrue(listOfHighest.contains(("A", 3)))
        assertTrue(listOfHighest.contains(("X", 3)))
    }


    test("Textfacet  lowest should give list of lowest pairs if more than one pair found") {
        val initialDataset = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "Q,E", "Q,R"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val listOfLowest = textFaceted.lowest
        assert(3 == textFaceted.count)
        assert(2 == listOfLowest.length)
        assertTrue(listOfLowest.contains(("X", 2)))
        assertTrue(listOfLowest.contains(("Q", 2)))
    }

    test(" TextFacet_ get Facets Between Should Give List Of Faceted Pair In Given Range ") {
        val initialDataset = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "Q,E", "Q,R", "W,E"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val facetedPair = textFaceted.getFacetsBetween(2, 3)
        assert(4 == textFaceted.count)

        assert(3 == facetedPair.length)
        assertTrue(facetedPair.contains(("X", 2)))
        assertTrue(facetedPair.contains(("Q", 2)))
        assertTrue(facetedPair.contains(("A", 3)))
    }

    test(" cardinalValues  Should return cardinal values for the text facets "){
        val initialDataset = sparkContext.parallelize(Array("X,Y", "A,B", "X,Z", "A,Q", "A,E", "Q,E", "Q,R", "W,E"))
        val initialRDD: TransformableRDD = new TransformableRDD(initialDataset)
        val textFaceted: TextFacets = initialRDD.listFacets(0)
        val cardinalValues: Array[String] = textFaceted.cardinalValues

        assert(cardinalValues sameElements Array("Q", "A", "X", "W"))
    }
}
{'fraction_non_alphanumeric': 0.13056731583403894, 'fraction_numerical': 0.007281964436917867, 'mean_word_length': 3.6910246227164416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5833457', 'n_tokens_mistral': 1970, 'n_tokens_neox': 1952, 'n_words': 417}