// Path: src/test/java/com/thoughtworks/datacommons/prepbuddy/api/JavaSparkTestCase.java
// public class JavaSparkTestCase implements Serializable {
//     protected transient JavaSparkContext javaSparkContext;
// 
//     @Before
//     public void setUp() throws Exception {
//         SparkConf sparkConf = new SparkConf().setAppName(getClass().getName()).setMaster("local");
//         Logger.getLogger("org").setLevel(Level.OFF);
//         Logger.getLogger("akka").setLevel(Level.OFF);
//         javaSparkContext = new JavaSparkContext(sparkConf);
//     }
// 
//     @After
//     public void tearDown() throws Exception {
//         javaSparkContext.close();
//     }
// }
// 
// Path: src/main/scala/com/thoughtworks/datacommons/prepbuddy/api/java/types/FileType.java
// public class FileType {
//     public static final com.thoughtworks.datacommons.prepbuddy.types.FileType CSV = CSV$.MODULE$;
//     public static final com.thoughtworks.datacommons.prepbuddy.types.FileType TSV = TSV$.MODULE$;
// }


import com.thoughtworks.datacommons.prepbuddy.api.JavaSparkTestCase;
import com.thoughtworks.datacommons.prepbuddy.api.java.types.FileType;
import com.thoughtworks.datacommons.prepbuddy.clusterers.SimpleFingerprintAlgorithm;
import com.thoughtworks.datacommons.prepbuddy.utils.PivotTable;
import com.thoughtworks.datacommons.prepbuddy.utils.RowRecord;
import org.apache.spark.api.java.JavaDoubleRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.junit.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.*;
        assertTrue(listOfDuplicates.contains("John,Male,India,12343"));
    }

    @Test
    public void shouldBeAbleToDetectDeduplicateRecordsByConsideringAllTheColumns() {
        JavaRDD<String> initialDataset = javaSparkContext.parallelize(Arrays.asList(
                "Smith,Male,USA,12345",
                "John,Male,USA,12343",
                "John,Male,India,12343",
                "Smith,Male,USA,12345"
        ));
        JavaTransformableRDD initialRDD = new JavaTransformableRDD(initialDataset);
        JavaTransformableRDD duplicates = initialRDD.duplicates();
        assertEquals(1, duplicates.count());
    }

    @Test
    public void shouldBeAbleToSelectAColumnFromTheDataSet() {
        JavaRDD<String> initialDataset = javaSparkContext.parallelize(Arrays.asList(
                "Smith,Male,USA,12345",
                "John,Male,USA,12343",
                "John,Male,India,12343",
                "Smith,Male,USA,12342"
        ));
        JavaTransformableRDD initialRDD = new JavaTransformableRDD(initialDataset, FileType.CSV);
        JavaRDD<String> columnValues = initialRDD.select(0);
        List<String> expectedValues = Arrays.asList("Smith", "John", "John", "Smith");
        assertEquals(expectedValues, columnValues.collect());
    }

// Path: src/test/java/com/thoughtworks/datacommons/prepbuddy/api/JavaSparkTestCase.java
// public class JavaSparkTestCase implements Serializable {
//     protected transient JavaSparkContext javaSparkContext;
// 
//     @Before
//     public void setUp() throws Exception {
//         SparkConf sparkConf = new SparkConf().setAppName(getClass().getName()).setMaster("local");
//         Logger.getLogger("org").setLevel(Level.OFF);
//         Logger.getLogger("akka").setLevel(Level.OFF);
//         javaSparkContext = new JavaSparkContext(sparkConf);
//     }
// 
//     @After
//     public void tearDown() throws Exception {
//         javaSparkContext.close();
//     }
// }
// 
// Path: src/main/scala/com/thoughtworks/datacommons/prepbuddy/api/java/types/FileType.java
// public class FileType {
//     public static final com.thoughtworks.datacommons.prepbuddy.types.FileType CSV = CSV$.MODULE$;
//     public static final com.thoughtworks.datacommons.prepbuddy.types.FileType TSV = TSV$.MODULE$;
// }


// Path: src/test/java/com/thoughtworks/datacommons/prepbuddy/api/java/JavaTransformableRDDTest.java
import com.thoughtworks.datacommons.prepbuddy.api.JavaSparkTestCase;
import com.thoughtworks.datacommons.prepbuddy.api.java.types.FileType;
import com.thoughtworks.datacommons.prepbuddy.clusterers.SimpleFingerprintAlgorithm;
import com.thoughtworks.datacommons.prepbuddy.utils.PivotTable;
import com.thoughtworks.datacommons.prepbuddy.utils.RowRecord;
import org.apache.spark.api.java.JavaDoubleRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.junit.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.*;

        assertTrue(listOfDuplicates.contains("John,Male,India,12343"));
    }

    @Test
    public void shouldBeAbleToDetectDeduplicateRecordsByConsideringAllTheColumns() {
        JavaRDD<String> initialDataset = javaSparkContext.parallelize(Arrays.asList(
                "Smith,Male,USA,12345",
                "John,Male,USA,12343",
                "John,Male,India,12343",
                "Smith,Male,USA,12345"
        ));
        JavaTransformableRDD initialRDD = new JavaTransformableRDD(initialDataset);
        JavaTransformableRDD duplicates = initialRDD.duplicates();
        assertEquals(1, duplicates.count());
    }

    @Test
    public void shouldBeAbleToSelectAColumnFromTheDataSet() {
        JavaRDD<String> initialDataset = javaSparkContext.parallelize(Arrays.asList(
                "Smith,Male,USA,12345",
                "John,Male,USA,12343",
                "John,Male,India,12343",
                "Smith,Male,USA,12342"
        ));
        JavaTransformableRDD initialRDD = new JavaTransformableRDD(initialDataset, FileType.CSV);
        JavaRDD<String> columnValues = initialRDD.select(0);
        List<String> expectedValues = Arrays.asList("Smith", "John", "John", "Smith");
        assertEquals(expectedValues, columnValues.collect());
    }

    @Test