<commit_before>package com.github.sparkcaller;

import org.apache.avro.generic.GenericData;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import picard.vcf.MergeVcfs;

import java.io.*;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

public class Utils {
    /*
    Get all the files in the folder 'pathToFolder' ending with 'fileExtension', and return them as an ArrayList.
    */
    public static ArrayList<File> getFilesInFolder(String pathToFolder, String fileExtension) {
        File folder = new File(pathToFolder);
        File[] listOfFiles = folder.listFiles();

        if (listOfFiles != null) {
            ArrayList<File> bamFiles = new ArrayList<File>();

            for (File file : listOfFiles) {
                if (file.isFile() && file.getName().endsWith(fileExtension)) {
                    bamFiles.add(file);
                }
            }
            return bamFiles;
        }

        System.err.println(String.format("The folder: %s is either empty or was not found!", folder.getName()));
        return null;
    }

    public static String removeExtenstion(String filepath, String ext) {
        return filepath.substring(0, filepath.length() - (ext.length()+1));
    }

    /*
    Merges all the VCF files specified in the list 'vcfFiles' and returns the filename of the output file.
    */
    public static String mergeVCFFiles(List<File> vcfFiles, String outputFileName) {
        MergeVcfs mergeEngine = new MergeVcfs();
        mergeEngine.INPUT = vcfFiles;

        // Only pass the output file as an argument,
        // as it is more efficient to set the input files directly in the object.
        String outputArgs[] = {"O=" + outputFileName};

        mergeEngine.instanceMain(outputArgs);
        return outputFileName;
    }

    public static Properties loadConfigFile(String configFilepath) {
        Properties prop = new Properties();
        InputStream input = null;

        try {
            input = new FileInputStream(configFilepath);
            prop.load(input);
        } catch (IOException ex) {
            ex.printStackTrace();
        } finally {
            if (input != null) {
                try {
                    input.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

        return prop;
    }

    public static List<File> splitVcf(JavaSparkContext sparkContext, String pathToInputVcf) throws IOException {
        JavaRDD<String> vcfFile = sparkContext.textFile(pathToInputVcf);
        JavaRDD<String> vcfHeader = vcfFile.filter(line -> line.startsWith("#"));
        JavaRDD<String> vcfLines = vcfFile.subtract(vcfHeader);


        // Get the first word in the line, as this maps to the chromosome.
        JavaPairRDD<String, Iterable<String>> vcfChromosomesRDD = vcfLines.groupBy(line -> line.split("\t", 0)[0]);
        List<String> chromosomes = vcfChromosomesRDD.keys().distinct().collect();
        List<File> vcfFilesByChromosome = new ArrayList<>();

        for (String chromosome: chromosomes) {
            JavaRDD<String> chromosomeRDD = getRddByChromosome(vcfChromosomesRDD, chromosome).sortBy(line -> Long.parseLong(line.split("\t")[1]), true, 1);
            List<String> chromosomeVcfLines = vcfHeader.union(chromosomeRDD).collect();

            File chromosomeOutputFile = new File(Utils.removeExtenstion(pathToInputVcf, "vcf") + "-" + chromosome + ".vcf");
            FileWriter newVcfFile = new FileWriter(chromosomeOutputFile);

            int vcfFileSize = chromosomeVcfLines.size();
            for (int i = 0; i < vcfFileSize; i++) {
                newVcfFile.write(chromosomeVcfLines.get(i) + "\n");
            }

            newVcfFile.close();
            vcfFilesByChromosome.add(chromosomeOutputFile);
        }

        return vcfFilesByChromosome;
    }

    public static JavaRDD<String> getRddByChromosome(JavaPairRDD<String, Iterable<String>> chromosomes, String chromosome) {
        return chromosomes.filter(rddChromosome -> rddChromosome._1().equals(chromosome)).values().flatMap(line -> line);
    }

    public static ArrayList<String> possibleStringToArgs(String maybeString) {
        if (maybeString == null) {
            return null;
        }

        String[] args = maybeString.split("\\s+");
        return new ArrayList<>(Arrays.asList(args));
    }
}<commit_msg>[Genomics]: Explain what some of the less obvious parts of the code actually does.<commit_after>package com.github.sparkcaller;

import org.apache.avro.generic.GenericData;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import picard.vcf.MergeVcfs;

import java.io.*;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

public class Utils {
    /*
    Get all the files in the folder 'pathToFolder' ending with 'fileExtension', and return them as an ArrayList.
    */
    public static ArrayList<File> getFilesInFolder(String pathToFolder, String fileExtension) {
        File folder = new File(pathToFolder);
        File[] listOfFiles = folder.listFiles();

        if (listOfFiles != null) {
            ArrayList<File> bamFiles = new ArrayList<File>();

            for (File file : listOfFiles) {
                if (file.isFile() && file.getName().endsWith(fileExtension)) {
                    bamFiles.add(file);
                }
            }
            return bamFiles;
        }

        System.err.println(String.format("The folder: %s is either empty or was not found!", folder.getName()));
        return null;
    }

    public static String removeExtenstion(String filepath, String ext) {
        return filepath.substring(0, filepath.length() - (ext.length()+1));
    }

    /*
    Merges all the VCF files specified in the list 'vcfFiles' and returns the filename of the output file.
    */
    public static String mergeVCFFiles(List<File> vcfFiles, String outputFileName) {
        MergeVcfs mergeEngine = new MergeVcfs();
        mergeEngine.INPUT = vcfFiles;

        // Only pass the output file as an argument,
        // as it is more efficient to set the input files directly in the object.
        String outputArgs[] = {"O=" + outputFileName};

        mergeEngine.instanceMain(outputArgs);
        return outputFileName;
    }

    public static Properties loadConfigFile(String configFilepath) {
        Properties prop = new Properties();
        InputStream input = null;

        try {
            input = new FileInputStream(configFilepath);
            prop.load(input);
        } catch (IOException ex) {
            ex.printStackTrace();
        } finally {
            if (input != null) {
                try {
                    input.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

        return prop;
    }

    public static List<File> splitVcf(JavaSparkContext sparkContext, String pathToInputVcf) throws IOException {
        JavaRDD<String> vcfFile = sparkContext.textFile(pathToInputVcf);
        JavaRDD<String> vcfHeader = vcfFile.filter(line -> line.startsWith("#"));
        JavaRDD<String> vcfLines = vcfFile.subtract(vcfHeader);


        // Get the first word in the line, as this maps to the chromosome.
        JavaPairRDD<String, Iterable<String>> vcfChromosomesRDD = vcfLines.groupBy(line -> line.split("\t", 0)[0]);
        List<String> chromosomes = vcfChromosomesRDD.keys().distinct().collect();
        List<File> vcfFilesByChromosome = new ArrayList<>();

        for (String chromosome: chromosomes) {
            JavaRDD<String> chromosomeRDD = getRddByChromosome(vcfChromosomesRDD, chromosome);

            // Sort the variants by position.
            chromosomeRDD = chromosomeRDD.sortBy(line -> Long.parseLong(line.split("\t")[1]), true, 1);

            // Make sure that each file has a header.
            List<String> chromosomeVcfLines = vcfHeader.union(chromosomeRDD).collect();

            File chromosomeOutputFile = new File(Utils.removeExtenstion(pathToInputVcf, "vcf") + "-" + chromosome + ".vcf");
            FileWriter newVcfFile = new FileWriter(chromosomeOutputFile);

            int vcfFileSize = chromosomeVcfLines.size();
            for (int i = 0; i < vcfFileSize; i++) {
                newVcfFile.write(chromosomeVcfLines.get(i) + "\n");
            }

            newVcfFile.close();
            vcfFilesByChromosome.add(chromosomeOutputFile);
        }

        return vcfFilesByChromosome;
    }

    public static JavaRDD<String> getRddByChromosome(JavaPairRDD<String, Iterable<String>> chromosomes, String chromosome) {
        // Get the RDD which has the desired 'chromosome' as the key,
        // then get all variants connected to this chromosome and return it as a single RDD.
        return chromosomes.filter(rddChromosome -> rddChromosome._1().equals(chromosome)).values().flatMap(line -> line);
    }

    public static ArrayList<String> possibleStringToArgs(String maybeString) {
        if (maybeString == null) {
            return null;
        }

        String[] args = maybeString.split("\\s+");
        return new ArrayList<>(Arrays.asList(args));
    }
}
{'fraction_non_alphanumeric': 0.10665825181445251, 'fraction_numerical': 0.0016829704428315978, 'mean_word_length': 2.5504107542942496, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5936921', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2622, 'n_words': 666}